diff --git a/crates/common/src/config/inner.rs b/crates/common/src/config/inner.rs index 4f304bdc..56c10112 100644 --- a/crates/common/src/config/inner.rs +++ b/crates/common/src/config/inner.rs @@ -15,6 +15,7 @@ use mail_auth::{MX, Parameters, Txt}; use mail_send::smtp::tls::build_tls_connector; use nlp::bayes::{TokenHash, Weights}; use parking_lot::RwLock; +use store::write::BatchBuilder; use utils::{ cache::{Cache, CacheWithTtl}, config::Config, @@ -42,11 +43,15 @@ impl Data { subject_names.insert("localhost".to_string()); } - // Parse id generator - let id_generator = config + // Build and test snowflake id generator + let node_id = config .property::("cluster.node-id") - .map(SnowflakeIdGenerator::with_node_id) - .unwrap_or_default(); + .unwrap_or_else(store::rand::random); + let id_generator = SnowflakeIdGenerator::with_node_id(node_id); + BatchBuilder::init_id_generator(node_id as u16); + if !id_generator.is_valid() { + panic!("Invalid system time, panicking to avoid data corruption"); + } Data { tls_certificates: ArcSwap::from_pointee(certificates), diff --git a/crates/common/src/core.rs b/crates/common/src/core.rs index 60524144..b37fd3f2 100644 --- a/crates/common/src/core.rs +++ b/crates/common/src/core.rs @@ -18,8 +18,8 @@ use store::{ dispatch::DocumentSet, roaring::RoaringBitmap, write::{ - AlignedBytes, Archive, BatchBuilder, BitmapClass, BlobOp, DirectoryClass, QueueClass, - TagValue, ValueClass, key::DeserializeBigEndian, log::ChangeLogBuilder, now, + AlignedBytes, Archive, AssignedIds, BatchBuilder, BitmapClass, BlobOp, DirectoryClass, + QueueClass, TagValue, ValueClass, key::DeserializeBigEndian, now, }, }; use trc::AddContext; @@ -245,7 +245,7 @@ impl Server { .clear(DirectoryClass::UsedQuota(account_id)) .add(DirectoryClass::UsedQuota(account_id), quota); self.store() - .write(batch) + .write(batch.build_all()) .await .caused_by(trc::location!()) .map(|_| ()) @@ -349,22 +349,44 @@ impl Server { }) } - pub async fn get_property( + #[inline(always)] + pub async fn get_archive( + &self, + account_id: u32, + collection: Collection, + document_id: u32, + ) -> trc::Result>> { + self.core + .storage + .data + .get_value(ValueKey { + account_id, + collection: collection.into(), + document_id, + class: ValueClass::Property(Property::Value.into()), + }) + .await + .add_context(|err| { + err.caused_by(trc::location!()) + .account_id(account_id) + .collection(collection) + .document_id(document_id) + }) + } + + #[inline(always)] + pub async fn get_archive_by_property( &self, account_id: u32, collection: Collection, document_id: u32, property: impl AsRef + Sync + Send, - ) -> trc::Result> - where - U: Deserialize + 'static, - { + ) -> trc::Result>> { let property = property.as_ref(); - self.core .storage .data - .get_value::(ValueKey { + .get_value(ValueKey { account_id, collection: collection.into(), document_id, @@ -376,7 +398,6 @@ impl Server { .account_id(account_id) .collection(collection) .document_id(document_id) - .id(property.to_string()) }) } @@ -385,14 +406,12 @@ impl Server { account_id: u32, collection: Collection, documents: &I, - property: Property, mut cb: CB, ) -> trc::Result<()> where I: DocumentSet + Send + Sync, CB: FnMut(u32, Archive) -> trc::Result + Send + Sync, { - let property: u8 = property.as_ref().into(); let collection: u8 = collection.into(); self.core @@ -404,13 +423,13 @@ impl Server { account_id, collection, document_id: documents.min(), - class: ValueClass::Property(property), + class: ValueClass::Property(Property::Value.into()), }, ValueKey { account_id, collection, document_id: documents.max(), - class: ValueClass::Property(property), + class: ValueClass::Property(Property::Value.into()), }, ), |key, value| { @@ -428,10 +447,10 @@ impl Server { err.caused_by(trc::location!()) .account_id(account_id) .collection(collection) - .id(property.to_string()) }) } + #[inline(always)] pub async fn get_document_ids( &self, account_id: u32, @@ -454,7 +473,7 @@ impl Server { account_id: u32, collection: Collection, property: impl AsRef + Sync + Send, - value: impl Into> + Sync + Send, + value: impl Into + Sync + Send, ) -> trc::Result> { let property = property.as_ref(); self.core @@ -478,6 +497,7 @@ impl Server { }) } + #[inline(always)] pub fn notify_task_queue(&self) { self.inner.ipc.index_tx.notify_one(); } @@ -502,47 +522,37 @@ impl Server { .map(|_| total) } - pub fn begin_changes(&self, account_id: u32) -> trc::Result { - self.assign_change_id(account_id) - .map(ChangeLogBuilder::with_change_id) - } - #[inline(always)] - pub fn assign_change_id(&self, _: u32) -> trc::Result { - self.generate_snowflake_id() + pub fn generate_snowflake_id(&self) -> u64 { + self.inner.data.jmap_id_gen.generate() } - pub fn generate_snowflake_id(&self) -> trc::Result { - self.inner.data.jmap_id_gen.generate().ok_or_else(|| { - trc::StoreEvent::UnexpectedError - .into_err() - .caused_by(trc::location!()) - .ctx(trc::Key::Reason, "Failed to generate snowflake id.") - }) - } + pub async fn commit_batch(&self, mut builder: BatchBuilder) -> trc::Result { + let mut assigned_ids = AssignedIds::default(); + let change_id = builder.last_change_id(); - pub async fn commit_changes( - &self, - account_id: u32, - mut changes: ChangeLogBuilder, - ) -> trc::Result { - if changes.change_id == u64::MAX || changes.change_id == 0 { - changes.change_id = self.assign_change_id(account_id)?; + for batch in builder.build() { + assigned_ids = self.store().write(batch).await?; } - let state = changes.change_id; - let mut builder = BatchBuilder::new(); - builder - .with_account_id(account_id) - .custom(changes) - .caused_by(trc::location!())?; - self.core - .storage - .data - .write(builder.build()) - .await - .caused_by(trc::location!()) - .map(|_| state) + if builder.has_logs() { + let change_id = change_id.unwrap(); + for (account_id, changed_collections) in builder.changed_collections() { + let mut state_change = StateChange::new(*account_id); + for changed_collection in *changed_collections { + if let Ok(data_type) = DataType::try_from(changed_collection) { + state_change.set_change(data_type, change_id); + } + } + if state_change.has_changes() { + self.broadcast_state_change(state_change).await; + } + } + + assigned_ids.change_id = change_id.into(); + } + + Ok(assigned_ids) } pub async fn delete_changes(&self, account_id: u32, before: Duration) -> trc::Result<()> { @@ -602,17 +612,6 @@ impl Server { } } - #[inline] - pub async fn broadcast_single_state_change( - &self, - account_id: u32, - change_id: u64, - data_type: DataType, - ) { - self.broadcast_state_change(StateChange::new(account_id).with_change(data_type, change_id)) - .await; - } - #[allow(clippy::blocks_in_conditions)] pub async fn put_blob( &self, @@ -635,7 +634,7 @@ impl Server { self.core .storage .data - .write(batch.build()) + .write(batch.build_all()) .await .caused_by(trc::location!())?; @@ -661,7 +660,7 @@ impl Server { self.core .storage .data - .write(batch.build()) + .write(batch.build_all()) .await .caused_by(trc::location!())?; } diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 44f4e04b..4b2d3330 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -42,6 +42,7 @@ use manager::webadmin::{Resource, WebAdminManager}; use nlp::bayes::{TokenHash, Weights}; use parking_lot::{Mutex, RwLock}; use rustls::sign::CertifiedKey; +use store::roaring::RoaringBitmap; use tokio::sync::{Notify, Semaphore, mpsc}; use tokio_rustls::TlsConnector; use utils::{ @@ -571,3 +572,37 @@ impl std::borrow::Borrow for DavResource { &self.document_id } } + +impl Threads { + pub fn assign_thread_id(&self, thread_name: &[u8], message_id: &[u8]) -> u32 { + let mut bytes = Vec::with_capacity(thread_name.len() + message_id.len()); + bytes.extend_from_slice(thread_name); + bytes.extend_from_slice(message_id); + let mut hash = store::gxhash::gxhash32(&bytes, 791120); + + if self.threads.is_empty() { + return hash; + } + + // Naive pass, assume hash is unique + let mut threads_ids = RoaringBitmap::new(); + let mut is_unique_hash = true; + for &thread_id in self.threads.keys() { + if is_unique_hash && thread_id != hash { + is_unique_hash = false; + } + threads_ids.insert(thread_id); + } + + if is_unique_hash { + hash + } else { + loop { + hash = hash.wrapping_add(1); + if !threads_ids.contains(hash) { + return hash; + } + } + } + } +} diff --git a/crates/common/src/listener/mod.rs b/crates/common/src/listener/mod.rs index 9cbf7b6a..87a282e7 100644 --- a/crates/common/src/listener/mod.rs +++ b/crates/common/src/listener/mod.rs @@ -113,8 +113,7 @@ pub trait SessionManager: Sync + Send + 'static + Clone { TcpAcceptorResult::Tls(accept) => match accept.await { Ok(stream) => { // Generate sessionId - session.session_id = - session.instance.span_id_gen.generate().unwrap_or_default(); + session.session_id = session.instance.span_id_gen.generate(); session_id = session.session_id; // Send span @@ -159,8 +158,7 @@ pub trait SessionManager: Sync + Send + 'static + Clone { }, TcpAcceptorResult::Plain(stream) => { // Generate sessionId - session.session_id = - session.instance.span_id_gen.generate().unwrap_or_default(); + session.session_id = session.instance.span_id_gen.generate(); session_id = session.session_id; // Send span @@ -183,7 +181,7 @@ pub trait SessionManager: Sync + Send + 'static + Clone { } } else { // Generate sessionId - session.session_id = session.instance.span_id_gen.generate().unwrap_or_default(); + session.session_id = session.instance.span_id_gen.generate(); session_id = session.session_id; // Send span diff --git a/crates/common/src/manager/backup.rs b/crates/common/src/manager/backup.rs index b2b452b4..06de03d5 100644 --- a/crates/common/src/manager/backup.rs +++ b/crates/common/src/manager/backup.rs @@ -864,7 +864,7 @@ impl Core { .send(Op::Family(Family::Bitmap)) .failed("Failed to send family"); - let mut bitmaps: AHashMap<(u32, u8), AHashSet>> = AHashMap::new(); + let mut bitmaps: AHashMap<(u32, u8), AHashSet> = AHashMap::new(); for subspace in [ SUBSPACE_BITMAP_ID, diff --git a/crates/common/src/manager/config.rs b/crates/common/src/manager/config.rs index c5e2c9a9..69a1545c 100644 --- a/crates/common/src/manager/config.rs +++ b/crates/common/src/manager/config.rs @@ -194,7 +194,7 @@ impl ConfigManager { } if !batch.is_empty() { - self.cfg_store.write(batch.build()).await?; + self.cfg_store.write(batch.build_all()).await?; } if !local_batch.is_empty() { @@ -236,7 +236,7 @@ impl ConfigManager { } else { let mut batch = BatchBuilder::new(); batch.clear(ValueClass::Config(key.to_string().into_bytes())); - self.cfg_store.write(batch.build()).await.map(|_| ()) + self.cfg_store.write(batch.build_all()).await.map(|_| ()) } } diff --git a/crates/common/src/manager/console.rs b/crates/common/src/manager/console.rs index 9e149225..59b2f6e9 100644 --- a/crates/common/src/manager/console.rs +++ b/crates/common/src/manager/console.rs @@ -160,7 +160,7 @@ pub async fn store_console(store: Store) { subspace: key.next().unwrap(), key: key.collect(), })); - if let Err(err) = store.write(batch.build()).await { + if let Err(err) = store.write(batch.build_all()).await { println!("Failed to delete key: {}", err); } } @@ -210,7 +210,7 @@ pub async fn store_console(store: Store) { }), value, ); - if let Err(err) = store.write(batch.build()).await { + if let Err(err) = store.write(batch.build_all()).await { println!("Failed to insert key: {}", err); } } diff --git a/crates/common/src/manager/restore.rs b/crates/common/src/manager/restore.rs index 7861b51e..b1a0530d 100644 --- a/crates/common/src/manager/restore.rs +++ b/crates/common/src/manager/restore.rs @@ -12,12 +12,11 @@ use std::{ use crate::Core; use jmap_proto::types::{collection::Collection, property::Property}; use store::{ - BlobStore, SerializeInfallible, Store, U32_LEN, + BlobStore, Key, LogKey, SUBSPACE_LOGS, SerializeInfallible, Store, U32_LEN, roaring::RoaringBitmap, write::{ - BatchBuilder, BitmapClass, BitmapHash, BlobOp, DirectoryClass, InMemoryClass, - MaybeDynamicId, MaybeDynamicValue, Operation, TagValue, TaskQueueClass, ValueClass, - key::DeserializeBigEndian, + AnyClass, BatchBuilder, BitmapClass, BitmapHash, BlobOp, DirectoryClass, InMemoryClass, + Operation, TagValue, TaskQueueClass, ValueClass, ValueOp, key::DeserializeBigEndian, }, }; use store::{ @@ -177,7 +176,7 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { } Family::Directory => { let key = key.as_slice(); - let class: DirectoryClass = + let class: DirectoryClass = match key.first().expect("Failed to read directory key type") { 0 => DirectoryClass::NameToId( key.get(1..) @@ -189,12 +188,12 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { .expect("Failed to read directory string") .to_vec(), ), - 2 => DirectoryClass::Principal(MaybeDynamicId::Static( + 2 => DirectoryClass::Principal( key.get(1..) .expect("Failed to read range for principal id") .deserialize_leb128::() .expect("Failed to deserialize principal id"), - )), + ), /*3 => DirectoryClass::Domain( key.get(1..) .expect("Failed to read directory string") @@ -215,24 +214,22 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { continue; } 5 => DirectoryClass::MemberOf { - principal_id: MaybeDynamicId::Static( - key.deserialize_be_u32(1) - .expect("Failed to read principal id"), - ), - member_of: MaybeDynamicId::Static( - key.deserialize_be_u32(1 + U32_LEN) - .expect("Failed to read principal id"), - ), + principal_id: key + .deserialize_be_u32(1) + .expect("Failed to read principal id"), + + member_of: key + .deserialize_be_u32(1 + U32_LEN) + .expect("Failed to read principal id"), }, 6 => DirectoryClass::Members { - principal_id: MaybeDynamicId::Static( - key.deserialize_be_u32(1) - .expect("Failed to read principal id"), - ), - has_member: MaybeDynamicId::Static( - key.deserialize_be_u32(1 + U32_LEN) - .expect("Failed to read principal id"), - ), + principal_id: key + .deserialize_be_u32(1) + .expect("Failed to read principal id"), + + has_member: key + .deserialize_be_u32(1 + U32_LEN) + .expect("Failed to read principal id"), }, _ => failed("Invalid directory key"), @@ -268,21 +265,23 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { _ => failed("Invalid queue key"), } } - Family::Index => batch.ops.push(Operation::Index { - field: key.first().copied().expect("Failed to read index field"), - key: key.get(1..).expect("Failed to read index key").to_vec(), - set: true, - }), + Family::Index => { + batch.any_op(Operation::Index { + field: key.first().copied().expect("Failed to read index field"), + key: key.get(1..).expect("Failed to read index key").to_vec(), + set: true, + }); + } Family::Bitmap => { let key = key.as_slice(); - let class: BitmapClass = + let class: BitmapClass = match key.first().expect("Failed to read bitmap class") { 0 => BitmapClass::DocumentIds, 1 => BitmapClass::Tag { field: key.get(1).copied().expect("Failed to read field"), - value: TagValue::Id(MaybeDynamicId::Static( + value: TagValue::Id( key.deserialize_be_u32(2).expect("Failed to read tag id"), - )), + ), }, 2 => BitmapClass::Tag { field: key.get(1).copied().expect("Failed to read field"), @@ -292,12 +291,12 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { }, 3 => BitmapClass::Tag { field: key.get(1).copied().expect("Failed to read field"), - value: TagValue::Id(MaybeDynamicId::Static( + value: TagValue::Id( key.get(2) .copied() .expect("Failed to read tag static id") .into(), - )), + ), }, 4 => { if reader.version == 1 && collection == email_collection { @@ -325,15 +324,15 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { .expect("Failed to deserialize bitmap"); for document_id in document_ids { - batch.ops.push(Operation::DocumentId { document_id }); - batch.ops.push(Operation::Bitmap { + batch.any_op(Operation::DocumentId { document_id }); + batch.any_op(Operation::Bitmap { class: class.clone(), set: true, }); - if batch.ops.len() >= 1000 { + if batch.len() >= 1000 { store - .write(batch.build()) + .write(batch.build_all()) .await .failed("Failed to write batch"); batch = BatchBuilder::new(); @@ -344,14 +343,20 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { } } Family::Log => { - batch.ops.push(Operation::ChangeId { - change_id: key - .as_slice() - .deserialize_be_u64(0) - .expect("Failed to deserialize change id"), - }); - batch.ops.push(Operation::Log { - set: MaybeDynamicValue::Static(value), + batch.any_op(Operation::Value { + class: ValueClass::Any(AnyClass { + subspace: SUBSPACE_LOGS, + key: LogKey { + account_id, + collection, + change_id: key + .as_slice() + .deserialize_be_u64(0) + .expect("Failed to deserialize change id"), + } + .serialize(0), + }), + op: ValueOp::Set(value), }); } Family::None => failed("No family specified in file"), @@ -359,9 +364,9 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { } } - if batch.ops.len() >= 1000 || batch_size >= 5_000_000 { + if batch.len() >= 1000 || batch_size >= 5_000_000 { store - .write(batch.build()) + .write(batch.build_all()) .await .failed("Failed to write batch"); batch = BatchBuilder::new(); @@ -375,7 +380,7 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { if !batch.is_empty() { store - .write(batch.build()) + .write(batch.build_all()) .await .failed("Failed to write batch"); } diff --git a/crates/common/src/storage/index.rs b/crates/common/src/storage/index.rs index 58d0923f..67dfa18a 100644 --- a/crates/common/src/storage/index.rs +++ b/crates/common/src/storage/index.rs @@ -5,7 +5,7 @@ */ use ahash::AHashSet; -use jmap_proto::types::{property::Property, value::AclGrant}; +use jmap_proto::types::{collection::Collection, property::Property, value::AclGrant}; use rkyv::{ option::ArchivedOption, primitive::{ArchivedU32, ArchivedU64}, @@ -14,10 +14,7 @@ use rkyv::{ use std::{borrow::Cow, fmt::Debug}; use store::{ Serialize, SerializeInfallible, SerializedVersion, - write::{ - Archive, Archiver, BatchBuilder, BitmapClass, BlobOp, DirectoryClass, IntoOperations, - MaybeDynamicId, Operation, TagValue, - }, + write::{Archive, Archiver, BatchBuilder, BlobOp, DirectoryClass, IntoOperations, TagValue}, }; use utils::BlobHash; @@ -35,7 +32,7 @@ pub enum IndexValue<'x> { }, Tag { field: u8, - value: Vec>, + value: Vec, }, Blob { value: BlobHash, @@ -43,6 +40,13 @@ pub enum IndexValue<'x> { Quota { used: u32, }, + LogChild { + prefix: Option, + }, + LogParent { + collection: Collection, + ids: Vec, + }, Acl { value: Cow<'x, [AclGrant]>, }, @@ -289,6 +293,8 @@ impl IntoOperations for (current, change) in current.inner.index_values().zip(changes.index_values()) { if current != change { merge_index(batch, current, change, self.tenant_id)?; + } else if let IndexValue::LogChild { prefix } = current { + batch.log_update(prefix); } } batch.set(Property::Value, Archiver::new(changes).serialize()?); @@ -313,28 +319,29 @@ fn build_index(batch: &mut BatchBuilder, item: IndexValue<'_>, tenant_id: Option match item { IndexValue::Index { field, value } => { if !value.is_empty() { - batch.ops.push(Operation::Index { - field, - key: value.into_owned(), - set, - }); + if set { + batch.index(field, value.into_owned()); + } else { + batch.unindex(field, value.into_owned()); + } } } IndexValue::IndexList { field, value } => { for key in value { - batch.ops.push(Operation::Index { - field, - key: key.into_owned(), - set, - }); + if set { + batch.index(field, key.into_owned()); + } else { + batch.unindex(field, key.into_owned()); + } } } IndexValue::Tag { field, value } => { for item in value { - batch.ops.push(Operation::Bitmap { - class: BitmapClass::Tag { field, value: item }, - set, - }); + if set { + batch.tag(field, item); + } else { + batch.untag(field, item); + } } } IndexValue::Blob { value } => { @@ -346,14 +353,11 @@ fn build_index(batch: &mut BatchBuilder, item: IndexValue<'_>, tenant_id: Option } IndexValue::Acl { value } => { for item in value.as_ref() { - batch.ops.push(Operation::acl( - item.account_id, - if set { - item.grants.bitmap.serialize().into() - } else { - None - }, - )); + if set { + batch.acl_grant(item.account_id, item.grants.bitmap.serialize()); + } else { + batch.acl_revoke(item.account_id); + } } } IndexValue::Quota { used } => { @@ -367,6 +371,18 @@ fn build_index(batch: &mut BatchBuilder, item: IndexValue<'_>, tenant_id: Option batch.add(DirectoryClass::UsedQuota(tenant_id), value); } } + IndexValue::LogChild { prefix } => { + if set { + batch.log_insert(prefix); + } else { + batch.log_delete(prefix); + } + } + IndexValue::LogParent { collection, ids } => { + for parent_id in ids { + batch.log_child_update(collection, parent_id); + } + } } } @@ -387,19 +403,11 @@ fn merge_index( }, ) => { if !old_value.is_empty() { - batch.ops.push(Operation::Index { - field, - key: old_value.into_owned(), - set: false, - }); + batch.unindex(field, old_value.into_owned()); } if !new_value.is_empty() { - batch.ops.push(Operation::Index { - field, - key: new_value.into_owned(), - set: true, - }); + batch.index(field, new_value.into_owned()); } } ( @@ -415,20 +423,12 @@ fn merge_index( for value in new_value { if !remove_values.remove(&value) { - batch.ops.push(Operation::Index { - field, - key: value.into_owned(), - set: true, - }); + batch.index(field, value.into_owned()); } } for value in remove_values { - batch.ops.push(Operation::Index { - field, - key: value.into_owned(), - set: false, - }); + batch.unindex(field, value.into_owned()); } } ( @@ -442,25 +442,13 @@ fn merge_index( ) => { for old_tag in &old_value { if !new_value.contains(old_tag) { - batch.ops.push(Operation::Bitmap { - class: BitmapClass::Tag { - field, - value: old_tag.clone(), - }, - set: false, - }); + batch.untag(field, old_tag.clone()); } } for new_tag in new_value { if !old_value.contains(&new_tag) { - batch.ops.push(Operation::Bitmap { - class: BitmapClass::Tag { - field, - value: new_tag, - }, - set: true, - }); + batch.tag(field, new_tag); } } } @@ -477,9 +465,7 @@ fn merge_index( .iter() .any(|item| item.account_id == current_item.account_id) { - batch - .ops - .push(Operation::acl(current_item.account_id, None)); + batch.acl_revoke(current_item.account_id); } } @@ -495,26 +481,20 @@ fn merge_index( } } if add_item { - batch.ops.push(Operation::acl( - item.account_id, - item.grants.bitmap.serialize().into(), - )); + batch.acl_grant(item.account_id, item.grants.bitmap.serialize()); } } } (false, true) => { // Add all ACLs for item in new_acl.as_ref() { - batch.ops.push(Operation::acl( - item.account_id, - item.grants.bitmap.serialize().into(), - )); + batch.acl_grant(item.account_id, item.grants.bitmap.serialize()); } } (true, false) => { // Remove all ACLs for item in old_acl.as_ref() { - batch.ops.push(Operation::acl(item.account_id, None)); + batch.acl_revoke(item.account_id); } } _ => {} @@ -530,6 +510,31 @@ fn merge_index( batch.add(DirectoryClass::UsedQuota(tenant_id), value); } } + ( + IndexValue::LogChild { prefix: old_prefix }, + IndexValue::LogChild { prefix: new_prefix }, + ) => { + batch.log_delete(old_prefix); + batch.log_insert(new_prefix); + } + ( + IndexValue::LogParent { + collection, + ids: old_ids, + }, + IndexValue::LogParent { ids: new_ids, .. }, + ) => { + for parent_id in &old_ids { + if !new_ids.contains(parent_id) { + batch.log_child_update(collection, *parent_id); + } + } + for parent_id in new_ids { + if !old_ids.contains(&parent_id) { + batch.log_child_update(collection, parent_id); + } + } + } _ => unreachable!(), } diff --git a/crates/common/src/storage/tag.rs b/crates/common/src/storage/tag.rs index 5ec8a729..3be73ca0 100644 --- a/crates/common/src/storage/tag.rs +++ b/crates/common/src/storage/tag.rs @@ -9,11 +9,11 @@ use std::slice::IterMut; use jmap_proto::types::property::Property; use store::{ Serialize, SerializedVersion, - write::{Archive, Archiver, BatchBuilder, MaybeDynamicId, TagValue, ValueClass}, + write::{Archive, Archiver, BatchBuilder, TagValue, ValueClass}, }; pub struct TagManager< - T: Into> + T: Into + PartialEq + Clone + Sync @@ -40,7 +40,7 @@ enum LastTag { } impl< - T: Into> + T: Into + PartialEq + Clone + Sync diff --git a/crates/common/src/telemetry/metrics/store.rs b/crates/common/src/telemetry/metrics/store.rs index e42e9753..91cb7a17 100644 --- a/crates/common/src/telemetry/metrics/store.rs +++ b/crates/common/src/telemetry/metrics/store.rs @@ -186,7 +186,7 @@ impl MetricsStore for Store { } if !batch.is_empty() { - self.write(batch.build()) + self.write(batch.build_all()) .await .caused_by(trc::location!())?; } diff --git a/crates/common/src/telemetry/tracers/store.rs b/crates/common/src/telemetry/tracers/store.rs index a3f160d5..66a4867e 100644 --- a/crates/common/src/telemetry/tracers/store.rs +++ b/crates/common/src/telemetry/tracers/store.rs @@ -13,7 +13,7 @@ use std::{future::Future, time::Duration}; use ahash::{AHashMap, AHashSet}; use store::{ Deserialize, IterateParams, Store, U64_LEN, ValueKey, - write::{BatchBuilder, MaybeDynamicId, TelemetryClass, ValueClass, key::DeserializeBigEndian}, + write::{BatchBuilder, TelemetryClass, ValueClass, key::DeserializeBigEndian}, }; use trc::{ AddContext, AuthEvent, Event, EventDetails, EventType, Key, MessageIngestEvent, @@ -123,7 +123,7 @@ pub(crate) fn spawn_store_tracer(builder: SubscriberBuilder, settings: StoreTrac } if !batch.is_empty() { - if let Err(err) = store.write(batch.build()).await { + if let Err(err) = store.write(batch.build_all()).await { trc::error!(err.caused_by(trc::location!())); } batch = BatchBuilder::new(); @@ -261,7 +261,7 @@ impl TracingStore for Store { .await .caused_by(trc::location!())?; - let mut delete_keys: Vec> = Vec::new(); + let mut delete_keys: Vec = Vec::new(); self.iterate( IterateParams::new( ValueKey::from(ValueClass::Telemetry(TelemetryClass::Index { @@ -296,15 +296,15 @@ impl TracingStore for Store { let mut batch = BatchBuilder::new(); for key in delete_keys { - if batch.ops.len() >= 1000 { - self.write(batch.build()).await?; + if batch.len() >= 1000 { + self.write(batch.build_all()).await?; batch = BatchBuilder::new(); } batch.clear(key); } if !batch.is_empty() { - self.write(batch.build()).await?; + self.write(batch.build_all()).await?; } } diff --git a/crates/dav/src/card/copy_move.rs b/crates/dav/src/card/copy_move.rs index 46632d9f..7c20ffd7 100644 --- a/crates/dav/src/card/copy_move.rs +++ b/crates/dav/src/card/copy_move.rs @@ -5,10 +5,26 @@ */ use common::{Server, auth::AccessToken}; -use dav_proto::RequestHeaders; +use dav_proto::{Depth, RequestHeaders, schema::response::CardCondition}; +use groupware::{ + DavName, + contact::{AddressBook, ContactCard}, + hierarchy::DavHierarchy, +}; use http_proto::HttpResponse; +use hyper::StatusCode; +use jmap_proto::types::{acl::Acl, collection::Collection}; +use store::write::BatchBuilder; +use trc::AddContext; -use crate::common::uri::DavUriResource; +use crate::{ + DavError, DavErrorCondition, + card::{insert_card, update_card}, + common::uri::DavUriResource, + file::DavFileResource, +}; + +use super::{delete::delete_card, update_addressbook}; pub(crate) trait CardCopyMoveRequestHandler: Sync + Send { fn handle_card_copy_move_request( @@ -26,12 +42,507 @@ impl CardCopyMoveRequestHandler for Server { headers: RequestHeaders<'_>, is_move: bool, ) -> crate::Result { - // Validate URI - let resource_ = self + // Validate source + let from_resource_ = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; + let from_account_id = from_resource_.account_id; + let from_resources = self + .fetch_dav_resources(from_account_id, Collection::AddressBook) + .await + .caused_by(trc::location!())?; + let from_resource_name = from_resource_ + .resource + .ok_or(DavError::Code(StatusCode::FORBIDDEN))?; + let from_resource = from_resources + .paths + .by_name(from_resource_name) + .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; - todo!() + // Validate ACL + if !access_token.is_member(from_account_id) + && !self + .has_access_to_document( + access_token, + from_account_id, + Collection::AddressBook, + if from_resource.is_container { + from_resource.document_id + } else { + from_resource.parent_id.unwrap() + }, + Acl::ReadItems, + ) + .await + .caused_by(trc::location!())? + { + return Err(DavError::Code(StatusCode::FORBIDDEN)); + } + + // Validate destination + let destination = self + .validate_uri( + access_token, + headers + .destination + .ok_or(DavError::Code(StatusCode::BAD_GATEWAY))?, + ) + .await?; + if destination.collection != Collection::AddressBook { + return Err(DavError::Code(StatusCode::BAD_GATEWAY)); + } + let to_account_id = destination + .account_id + .ok_or(DavError::Code(StatusCode::BAD_GATEWAY))?; + let to_resources = if to_account_id == from_account_id { + from_resources.clone() + } else { + self.fetch_dav_resources(to_account_id, Collection::AddressBook) + .await + .caused_by(trc::location!())? + }; + + // Map destination + let destination_resource_name = destination + .resource + .ok_or(DavError::Code(StatusCode::BAD_GATEWAY))?; + if let Some(to_resource) = to_resources.paths.by_name(destination_resource_name) { + if from_resource.name == to_resource.name { + // Same resource + return Err(DavError::Code(StatusCode::BAD_GATEWAY)); + } + let new_name = destination_resource_name + .rsplit_once('/') + .map(|(_, name)| name) + .unwrap_or(destination_resource_name); + + match (from_resource.is_container, to_resource.is_container) { + (true, true) => { + // Overwrite container + if is_move { + move_container( + self, + access_token, + from_account_id, + from_resource.document_id, + from_resources + .subtree(from_resource_name) + .filter(|r| !r.is_container) + .map(|r| r.document_id) + .collect::>(), + to_account_id, + to_resource.document_id.into(), + to_resources + .subtree(destination_resource_name) + .filter(|r| !r.is_container) + .map(|r| r.document_id) + .collect::>(), + new_name.into(), + ) + .await + } else { + copy_container( + self, + access_token, + from_account_id, + from_resource.document_id, + from_resources + .subtree(from_resource_name) + .filter(|r| !r.is_container) + .map(|r| r.document_id) + .collect::>(), + to_account_id, + to_resource.document_id.into(), + to_resources + .subtree(destination_resource_name) + .filter(|r| !r.is_container) + .map(|r| r.document_id) + .collect::>(), + new_name.into(), + ) + .await + } + } + (false, false) => { + // Overwrite card + let from_addressbook_id = from_resource.parent_id.unwrap(); + let to_addressbook_id = to_resource.parent_id.unwrap(); + + if is_move { + move_card( + self, + access_token, + from_account_id, + from_resource.document_id, + from_addressbook_id, + to_account_id, + to_resource.document_id.into(), + to_addressbook_id, + new_name.into(), + ) + .await + } else { + copy_card( + self, + access_token, + from_account_id, + from_resource.document_id, + from_addressbook_id, + to_resource.document_id.into(), + to_addressbook_id, + headers.format_to_base_uri( + destination_resource_name + .rsplit_once('/') + .map(|(base, _)| base) + .unwrap_or(destination_resource_name), + ), + new_name.into(), + ) + .await + } + } + _ => Err(DavError::Code(StatusCode::BAD_GATEWAY)), + } + } else if let Some((parent_resource, new_name)) = + to_resources.map_parent(destination_resource_name) + { + if let Some(parent_resource) = parent_resource { + // Creating items under a card is not allowed + // Copying/moving containers under a container is not allowed + if !parent_resource.is_container || from_resource.is_container { + return Err(DavError::Code(StatusCode::BAD_GATEWAY)); + } + + let todo = "check acls"; + + // Copy/move card + let from_addressbook_id = from_resource.parent_id.unwrap(); + let to_addressbook_id = parent_resource.document_id; + if is_move { + if from_account_id != to_account_id + || parent_resource.document_id != from_addressbook_id + { + move_card( + self, + access_token, + from_account_id, + from_resource.document_id, + from_addressbook_id, + to_account_id, + None, + to_addressbook_id, + new_name.into(), + ) + .await + } else { + rename_card( + self, + access_token, + from_account_id, + from_resource.document_id, + from_addressbook_id, + new_name, + ) + .await + } + } else { + copy_card( + self, + access_token, + from_account_id, + from_resource.document_id, + from_addressbook_id, + None, + to_addressbook_id, + headers.format_to_base_uri(&parent_resource.name), + new_name, + ) + .await + } + } else { + // Copying/moving cards to the root is not allowed + if !from_resource.is_container { + return Err(DavError::Code(StatusCode::BAD_GATEWAY)); + } + + // Shared users cannot create containers + if !access_token.is_member(to_account_id) { + return Err(DavError::Code(StatusCode::FORBIDDEN)); + } + + // Copy/move container + let from_children_ids = from_resources + .subtree(from_resource_name) + .filter(|r| !r.is_container) + .map(|r| r.document_id) + .collect::>(); + if is_move { + if from_account_id != to_account_id { + move_container( + self, + access_token, + from_account_id, + from_resource.document_id, + if headers.depth != Depth::Zero { + from_children_ids + } else { + return Err(DavError::Code(StatusCode::BAD_GATEWAY)); + }, + to_account_id, + None, + vec![], + new_name.into(), + ) + .await + } else { + rename_container( + self, + access_token, + from_account_id, + from_resource.document_id, + new_name, + ) + .await + } + } else { + copy_container( + self, + access_token, + from_account_id, + from_resource.document_id, + if headers.depth != Depth::Zero { + from_children_ids + } else { + vec![] + }, + to_account_id, + None, + vec![], + new_name.into(), + ) + .await + } + } + } else { + Err(DavError::Code(StatusCode::CONFLICT)) + } } } + +#[allow(clippy::too_many_arguments)] +async fn copy_card( + server: &Server, + access_token: &AccessToken, + from_account_id: u32, + from_document_id: u32, + to_account_id: u32, + to_document_id: Option, + to_addressbook_id: u32, + to_base_path: String, + new_name: &str, +) -> crate::Result { + // Fetch card + let card_ = server + .get_archive(from_account_id, Collection::ContactCard, from_document_id) + .await + .caused_by(trc::location!())? + .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + let card = card_ + .to_unarchived::() + .caused_by(trc::location!())?; + let mut batch = BatchBuilder::new(); + + if from_account_id == to_account_id { + if let Some(name) = card + .inner + .names + .iter() + .find(|n| n.parent_id == to_addressbook_id) + { + return Err(DavError::Condition(DavErrorCondition::new( + StatusCode::PRECONDITION_FAILED, + CardCondition::NoUidConflict(format!("{}/{}", to_base_path, name.name).into()), + ))); + } + let mut new_card = card + .deserialize::() + .caused_by(trc::location!())?; + new_card.names.push(DavName { + name: new_name.to_string(), + parent_id: to_addressbook_id, + }); + update_card( + access_token, + card.clone(), + new_card, + from_account_id, + from_document_id, + false, + &mut batch, + ) + .caused_by(trc::location!())?; + } else { + let todo = "check uid"; + let mut new_card = card + .deserialize::() + .caused_by(trc::location!())?; + new_card.names = vec![DavName { + name: new_name.to_string(), + parent_id: to_addressbook_id, + }]; + //insert_card(access_token, new_card, to_account_id, false, &mut batch) + // .caused_by(trc::location!())?; + } + + if let Some(to_document_id) = to_document_id { + delete_card( + access_token, + to_account_id, + to_document_id, + to_addressbook_id, + card, + &mut batch, + ) + .await + .caused_by(trc::location!())?; + Ok(HttpResponse::new(StatusCode::NO_CONTENT)) + } else { + Ok(HttpResponse::new(StatusCode::CREATED)) + } +} + +#[allow(clippy::too_many_arguments)] +async fn move_card( + server: &Server, + access_token: &AccessToken, + from_account_id: u32, + from_document_id: u32, + from_addressbook_id: u32, + to_account_id: u32, + to_document_id: Option, + to_addressbook_id: u32, + new_name: Option<&str>, +) -> crate::Result { + todo!() +} + +#[allow(clippy::too_many_arguments)] +async fn rename_card( + server: &Server, + access_token: &AccessToken, + account_id: u32, + document_id: u32, + addressbook_id: u32, + new_name: &str, +) -> crate::Result { + // Fetch card + let card_ = server + .get_archive(account_id, Collection::ContactCard, document_id) + .await + .caused_by(trc::location!())? + .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + let card = card_ + .to_unarchived::() + .caused_by(trc::location!())?; + + let name_idx = card + .inner + .names + .iter() + .position(|n| n.parent_id == addressbook_id) + .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + let mut new_card = card + .deserialize::() + .caused_by(trc::location!())?; + new_card.names[name_idx].name = new_name.to_string(); + + let mut batch = BatchBuilder::new(); + update_card( + access_token, + card, + new_card, + account_id, + document_id, + false, + &mut batch, + ) + .caused_by(trc::location!())?; + server + .commit_batch(batch) + .await + .caused_by(trc::location!())?; + + Ok(HttpResponse::new(StatusCode::CREATED)) +} + +#[allow(clippy::too_many_arguments)] +async fn copy_container( + server: &Server, + access_token: &AccessToken, + from_account_id: u32, + from_document_id: u32, + from_children_ids: Vec, + to_account_id: u32, + to_document_id: Option, + to_children_ids: Vec, + new_name: Option<&str>, +) -> crate::Result { + todo!() +} + +#[allow(clippy::too_many_arguments)] +async fn move_container( + server: &Server, + access_token: &AccessToken, + from_account_id: u32, + from_document_id: u32, + from_children_ids: Vec, + to_account_id: u32, + to_document_id: Option, + to_children_ids: Vec, + new_name: Option<&str>, +) -> crate::Result { + todo!() +} + +#[allow(clippy::too_many_arguments)] +async fn rename_container( + server: &Server, + access_token: &AccessToken, + account_id: u32, + document_id: u32, + new_name: &str, +) -> crate::Result { + // Fetch book + let book_ = server + .get_archive(account_id, Collection::AddressBook, document_id) + .await + .caused_by(trc::location!())? + .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + let book = book_ + .to_unarchived::() + .caused_by(trc::location!())?; + let mut new_book = book + .deserialize::() + .caused_by(trc::location!())?; + new_book.name = new_name.to_string(); + + let mut batch = BatchBuilder::new(); + update_addressbook( + access_token, + book, + new_book, + account_id, + document_id, + false, + &mut batch, + ) + .caused_by(trc::location!())?; + server + .commit_batch(batch) + .await + .caused_by(trc::location!())?; + + Ok(HttpResponse::new(StatusCode::CREATED)) +} diff --git a/crates/dav/src/card/delete.rs b/crates/dav/src/card/delete.rs index 6ca1d6a5..c328b82b 100644 --- a/crates/dav/src/card/delete.rs +++ b/crates/dav/src/card/delete.rs @@ -4,11 +4,28 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{Server, auth::AccessToken}; +use common::{ + Server, auth::AccessToken, sharing::EffectiveAcl, storage::index::ObjectIndexBuilder, +}; use dav_proto::RequestHeaders; +use groupware::{ + contact::{AddressBook, ArchivedAddressBook, ArchivedContactCard, ContactCard}, + hierarchy::DavHierarchy, +}; use http_proto::HttpResponse; +use hyper::StatusCode; +use jmap_proto::types::{acl::Acl, collection::Collection}; +use store::write::{Archive, BatchBuilder}; +use trc::AddContext; -use crate::common::uri::DavUriResource; +use crate::{ + DavError, DavMethod, + common::{ + ETag, + lock::{LockRequestHandler, ResourceState}, + uri::DavUriResource, + }, +}; pub(crate) trait CardDeleteRequestHandler: Sync + Send { fn handle_card_delete_request( @@ -25,11 +42,239 @@ impl CardDeleteRequestHandler for Server { headers: RequestHeaders<'_>, ) -> crate::Result { // Validate URI - let resource_ = self + let resource = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; + let account_id = resource.account_id; + let delete_path = resource + .resource + .filter(|r| !r.is_empty()) + .ok_or(DavError::Code(StatusCode::FORBIDDEN))?; + let resources = self + .fetch_dav_resources(account_id, Collection::AddressBook) + .await + .caused_by(trc::location!())?; - todo!() + // Check resource type + let delete_resource = resources + .paths + .by_name(delete_path) + .ok_or(DavError::Code(StatusCode::FORBIDDEN))?; + let document_id = delete_resource.document_id; + + // Fetch entry + let mut batch = BatchBuilder::new(); + if delete_resource.is_container { + let book_ = self + .get_archive(account_id, Collection::AddressBook, document_id) + .await + .caused_by(trc::location!())? + .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + + let book = book_ + .to_unarchived::() + .caused_by(trc::location!())?; + + // Validate ACL + if !access_token.is_member(account_id) + && !book + .inner + .acls + .effective_acl(access_token) + .contains_all([Acl::Delete, Acl::RemoveItems].into_iter()) + { + return Err(DavError::Code(StatusCode::FORBIDDEN)); + } + + // Validate headers + self.validate_headers( + access_token, + &headers, + vec![ResourceState { + account_id, + collection: Collection::AddressBook, + document_id: document_id.into(), + etag: book.etag().into(), + path: delete_path, + ..Default::default() + }], + Default::default(), + DavMethod::DELETE, + ) + .await?; + + // Delete addressbook and cards + delete_address_book( + self, + access_token, + account_id, + document_id, + resources + .subtree(delete_path) + .filter(|r| !r.is_container) + .map(|r| r.document_id) + .collect::>(), + book, + &mut batch, + ) + .await + .caused_by(trc::location!())?; + } else { + // Validate ACL + let addressbook_id = delete_resource.parent_id.unwrap(); + if !access_token.is_member(account_id) + && !self + .has_access_to_document( + access_token, + account_id, + Collection::AddressBook, + addressbook_id, + Acl::RemoveItems, + ) + .await + .caused_by(trc::location!())? + { + return Err(DavError::Code(StatusCode::FORBIDDEN)); + } + + let card_ = self + .get_archive(account_id, Collection::ContactCard, document_id) + .await + .caused_by(trc::location!())? + .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; + + // Validate headers + self.validate_headers( + access_token, + &headers, + vec![ResourceState { + account_id, + collection: Collection::ContactCard, + document_id: document_id.into(), + etag: card_.etag().into(), + path: delete_path, + ..Default::default() + }], + Default::default(), + DavMethod::DELETE, + ) + .await?; + + // Delete card + delete_card( + access_token, + account_id, + document_id, + addressbook_id, + card_ + .to_unarchived::() + .caused_by(trc::location!())?, + &mut batch, + ) + .await + .caused_by(trc::location!())?; + } + + self.commit_batch(batch).await.caused_by(trc::location!())?; + + Ok(HttpResponse::new(StatusCode::NO_CONTENT)) } } + +pub(crate) async fn delete_address_book( + server: &Server, + access_token: &AccessToken, + account_id: u32, + document_id: u32, + children_ids: Vec, + book: Archive<&ArchivedAddressBook>, + batch: &mut BatchBuilder, +) -> trc::Result<()> { + // Process deletions + let addressbook_id = document_id; + for document_id in children_ids { + if let Some(card_) = server + .get_archive(account_id, Collection::ContactCard, document_id) + .await? + { + delete_card( + access_token, + account_id, + document_id, + addressbook_id, + card_ + .to_unarchived::() + .caused_by(trc::location!())?, + batch, + ) + .await?; + } + } + + // Delete addressbook + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .with_collection(Collection::AddressBook) + .delete_document(document_id) + .custom( + ObjectIndexBuilder::<_, ()>::new() + .with_tenant_id(access_token) + .with_current(book), + ) + .caused_by(trc::location!())?; + + Ok(()) +} + +pub(crate) async fn delete_card( + access_token: &AccessToken, + account_id: u32, + document_id: u32, + addressbook_id: u32, + card: Archive<&ArchivedContactCard>, + batch: &mut BatchBuilder, +) -> trc::Result<()> { + if let Some(delete_idx) = card + .inner + .names + .iter() + .position(|name| name.parent_id == addressbook_id) + { + batch + .with_account_id(account_id) + .with_collection(Collection::ContactCard); + + if card.inner.names.len() > 1 { + // Unlink addressbook id from card + let mut new_card = card + .deserialize::() + .caused_by(trc::location!())?; + new_card.names.swap_remove(delete_idx); + batch + .update_document(document_id) + .custom( + ObjectIndexBuilder::new() + .with_tenant_id(access_token) + .with_current(card) + .with_changes(new_card), + ) + .caused_by(trc::location!())?; + } else { + // Delete card + batch + .delete_document(document_id) + .custom( + ObjectIndexBuilder::<_, ()>::new() + .with_tenant_id(access_token) + .with_current(card), + ) + .caused_by(trc::location!())?; + } + + batch.commit_point(); + } + + Ok(()) +} diff --git a/crates/dav/src/card/get.rs b/crates/dav/src/card/get.rs index 30d39f57..f5f34c75 100644 --- a/crates/dav/src/card/get.rs +++ b/crates/dav/src/card/get.rs @@ -9,8 +9,7 @@ use dav_proto::{RequestHeaders, schema::property::Rfc1123DateTime}; use groupware::{contact::ContactCard, hierarchy::DavHierarchy}; use http_proto::HttpResponse; use hyper::StatusCode; -use jmap_proto::types::{acl::Acl, collection::Collection, property::Property}; -use store::write::{AlignedBytes, Archive}; +use jmap_proto::types::{acl::Acl, collection::Collection}; use trc::AddContext; use crate::{ @@ -78,12 +77,7 @@ impl CardGetRequestHandler for Server { // Fetch card let card_ = self - .get_property::>( - account_id, - Collection::ContactCard, - resource.document_id, - Property::Value, - ) + .get_archive(account_id, Collection::ContactCard, resource.document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/card/mkcol.rs b/crates/dav/src/card/mkcol.rs index 04b2b2e5..5b94fcaa 100644 --- a/crates/dav/src/card/mkcol.rs +++ b/crates/dav/src/card/mkcol.rs @@ -12,8 +12,8 @@ use dav_proto::{ use groupware::{contact::AddressBook, hierarchy::DavHierarchy}; use http_proto::HttpResponse; use hyper::StatusCode; -use jmap_proto::types::{collection::Collection, type_state::DataType}; -use store::write::{BatchBuilder, log::LogInsert, now}; +use jmap_proto::types::collection::Collection; +use store::write::{BatchBuilder, now}; use trc::AddContext; use crate::{ @@ -81,7 +81,6 @@ impl CardMkColRequestHandler for Server { .await?; // Build file container - let change_id = self.generate_snowflake_id().caused_by(trc::location!())?; let now = now(); let mut book = AddressBook { name: name.to_string(), @@ -108,22 +107,19 @@ impl CardMkColRequestHandler for Server { // Prepare write batch let mut batch = BatchBuilder::new(); - batch - .with_change_id(change_id) - .with_account_id(account_id) - .with_collection(Collection::AddressBook) - .create_document() - .log(LogInsert()) - .custom(ObjectIndexBuilder::<(), _>::new().with_changes(book)) - .caused_by(trc::location!())?; - self.store() - .write(batch) + let document_id = self + .store() + .assign_document_ids(account_id, Collection::AddressBook, 1) .await .caused_by(trc::location!())?; + batch + .with_account_id(account_id) + .with_collection(Collection::AddressBook) + .create_document(document_id) + .custom(ObjectIndexBuilder::<(), _>::new().with_changes(book)) + .caused_by(trc::location!())?; + self.commit_batch(batch).await.caused_by(trc::location!())?; - // Broadcast state change - self.broadcast_single_state_change(account_id, change_id, DataType::AddressBook) - .await; if let Some(prop_stat) = return_prop_stat { Ok(HttpResponse::new(StatusCode::CREATED).with_xml_body( MkColResponse::new(prop_stat) diff --git a/crates/dav/src/card/mod.rs b/crates/dav/src/card/mod.rs index 8624d509..5006021b 100644 --- a/crates/dav/src/card/mod.rs +++ b/crates/dav/src/card/mod.rs @@ -4,6 +4,13 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use common::{auth::AccessToken, storage::index::ObjectIndexBuilder}; +use groupware::contact::{AddressBook, ArchivedAddressBook, ArchivedContactCard, ContactCard}; +use jmap_proto::types::collection::Collection; +use store::write::{Archive, BatchBuilder, now}; + +use crate::common::ExtractETag; + pub mod acl; pub mod copy_move; pub mod delete; @@ -13,3 +20,88 @@ pub mod propfind; pub mod proppatch; pub mod query; pub mod update; + +pub(crate) fn update_card( + access_token: &AccessToken, + card: Archive<&ArchivedContactCard>, + mut new_card: ContactCard, + account_id: u32, + document_id: u32, + with_etag: bool, + batch: &mut BatchBuilder, +) -> trc::Result> { + // Build card + new_card.modified = now() as i64; + + // Prepare write batch + batch + .with_account_id(account_id) + .with_collection(Collection::ContactCard) + .update_document(document_id) + .custom( + ObjectIndexBuilder::new() + .with_current(card) + .with_changes(new_card) + .with_tenant_id(access_token), + )? + .commit_point(); + + Ok(if with_etag { batch.etag() } else { None }) +} + +pub(crate) fn insert_card( + access_token: &AccessToken, + mut card: ContactCard, + account_id: u32, + document_id: u32, + with_etag: bool, + batch: &mut BatchBuilder, +) -> trc::Result> { + // Build card + let now = now() as i64; + card.modified = now; + card.created = now; + + // Prepare write batch + batch + .with_account_id(account_id) + .with_collection(Collection::ContactCard) + .create_document(document_id) + .custom( + ObjectIndexBuilder::<(), _>::new() + .with_changes(card) + .with_tenant_id(access_token), + )? + .commit_point(); + + Ok(if with_etag { batch.etag() } else { None }) +} + +pub(crate) fn update_addressbook( + access_token: &AccessToken, + book: Archive<&ArchivedAddressBook>, + mut new_book: AddressBook, + account_id: u32, + document_id: u32, + with_etag: bool, + batch: &mut BatchBuilder, +) -> trc::Result> { + // Build card + new_book.modified = now() as i64; + + // Prepare write batch + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .with_collection(Collection::AddressBook) + .update_document(document_id) + .custom( + ObjectIndexBuilder::new() + .with_current(book) + .with_changes(new_book) + .with_tenant_id(access_token), + )? + .commit_point(); + + Ok(if with_etag { batch.etag() } else { None }) +} diff --git a/crates/dav/src/card/update.rs b/crates/dav/src/card/update.rs index e88eab32..a4771826 100644 --- a/crates/dav/src/card/update.rs +++ b/crates/dav/src/card/update.rs @@ -13,16 +13,10 @@ use dav_proto::{ use groupware::{DavName, IDX_CARD_UID, contact::ContactCard, hierarchy::DavHierarchy}; use http_proto::HttpResponse; use hyper::StatusCode; -use jmap_proto::types::{ - acl::Acl, collection::Collection, property::Property, state::StateChange, type_state::DataType, -}; +use jmap_proto::types::{acl::Acl, collection::Collection}; use store::{ query::Filter, - write::{ - AlignedBytes, Archive, BatchBuilder, - log::{Changes, LogInsert}, - now, - }, + write::{BatchBuilder, now}, }; use trc::AddContext; @@ -33,6 +27,7 @@ use crate::{ lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, + file::DavFileResource, }; pub(crate) trait CardUpdateRequestHandler: Sync + Send { @@ -115,12 +110,7 @@ impl CardUpdateRequestHandler for Server { // Update let card_ = self - .get_property::>( - account_id, - Collection::FileNode, - document_id, - Property::Value, - ) + .get_archive(account_id, Collection::FileNode, document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -188,7 +178,6 @@ impl CardUpdateRequestHandler for Server { } // Build node - let change_id = self.generate_snowflake_id().caused_by(trc::location!())?; let mut new_card = card .deserialize::() .caused_by(trc::location!())?; @@ -199,15 +188,9 @@ impl CardUpdateRequestHandler for Server { // Prepare write batch let mut batch = BatchBuilder::new(); batch - .with_change_id(change_id) .with_account_id(account_id) - .with_collection(Collection::AddressBook) - .log(Changes::child_update( - card.inner.names.iter().map(|n| n.parent_id.to_native()), - )) .with_collection(Collection::ContactCard) .update_document(document_id) - .log(Changes::update([document_id])) .custom( ObjectIndexBuilder::new() .with_current(card) @@ -216,24 +199,10 @@ impl CardUpdateRequestHandler for Server { ) .caused_by(trc::location!())?; let etag = batch.etag(); - self.store() - .write(batch) - .await - .caused_by(trc::location!())?; - - // Broadcast state change - self.broadcast_state_change( - StateChange::new(account_id) - .with_change(DataType::ContactCard, change_id) - .with_change(DataType::AddressBook, change_id), - ) - .await; + self.commit_batch(batch).await.caused_by(trc::location!())?; Ok(HttpResponse::new(StatusCode::NO_CONTENT).with_etag_opt(etag)) - } else if let Some((parent, name)) = resource_name - .rsplit_once('/') - .and_then(|(parent, name)| resources.paths.by_name(parent).map(|parent| (parent, name))) - { + } else if let Some((Some(parent), name)) = resources.map_parent(resource_name) { if !parent.is_container { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); } @@ -308,7 +277,6 @@ impl CardUpdateRequestHandler for Server { } // Build node - let change_id = self.generate_snowflake_id().caused_by(trc::location!())?; let now = now(); let card = ContactCard { names: vec![DavName { @@ -324,14 +292,15 @@ impl CardUpdateRequestHandler for Server { // Prepare write batch let mut batch = BatchBuilder::new(); + let document_id = self + .store() + .assign_document_ids(account_id, Collection::ContactCard, 1) + .await + .caused_by(trc::location!())?; batch - .with_change_id(change_id) .with_account_id(account_id) - .with_collection(Collection::AddressBook) - .log(Changes::child_update([parent.document_id])) .with_collection(Collection::ContactCard) - .create_document() - .log(LogInsert()) + .create_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(card) @@ -339,18 +308,7 @@ impl CardUpdateRequestHandler for Server { ) .caused_by(trc::location!())?; let etag = batch.etag(); - self.store() - .write(batch) - .await - .caused_by(trc::location!())?; - - // Broadcast state change - self.broadcast_state_change( - StateChange::new(account_id) - .with_change(DataType::ContactCard, change_id) - .with_change(DataType::AddressBook, change_id), - ) - .await; + self.commit_batch(batch).await.caused_by(trc::location!())?; Ok(HttpResponse::new(StatusCode::CREATED).with_etag_opt(etag)) } else { diff --git a/crates/dav/src/common/acl.rs b/crates/dav/src/common/acl.rs index 09f27cc2..d7883e01 100644 --- a/crates/dav/src/common/acl.rs +++ b/crates/dav/src/common/acl.rs @@ -20,15 +20,10 @@ use hyper::StatusCode; use jmap_proto::types::{ acl::Acl, collection::Collection, - property::Property, value::{AclGrant, ArchivedAclGrant}, }; use rkyv::vec::ArchivedVec; -use store::{ - ahash::AHashSet, - roaring::RoaringBitmap, - write::{AlignedBytes, Archive}, -}; +use store::{ahash::AHashSet, roaring::RoaringBitmap}; use trc::AddContext; use utils::map::bitmap::Bitmap; @@ -102,12 +97,7 @@ impl DavAclHandler for Server { } let archive = self - .get_property::>( - uri.account_id, - uri.collection, - uri.resource, - Property::Value, - ) + .get_archive(uri.account_id, uri.collection, uri.resource) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/common/lock.rs b/crates/dav/src/common/lock.rs index 15e4bcae..8174cf83 100644 --- a/crates/dav/src/common/lock.rs +++ b/crates/dav/src/common/lock.rs @@ -14,7 +14,6 @@ use dav_proto::{RequestHeaders, schema::request::LockInfo}; use http_proto::HttpResponse; use hyper::StatusCode; use jmap_proto::types::collection::Collection; -use jmap_proto::types::property::Property; use std::collections::HashMap; use store::dispatch::lookup::KeyValue; use store::write::serialize::rkyv_deserialize; @@ -492,11 +491,10 @@ impl LockRequestHandler for Server { resource_state.document_id.filter(|&id| id != u32::MAX) { if let Some(archive) = self - .get_property::>( + .get_archive( resource_state.account_id, resource_state.collection, document_id, - Property::Value, ) .await .caused_by(trc::location!())? diff --git a/crates/dav/src/common/mod.rs b/crates/dav/src/common/mod.rs index 2610169f..b9324c3f 100644 --- a/crates/dav/src/common/mod.rs +++ b/crates/dav/src/common/mod.rs @@ -11,7 +11,7 @@ use dav_proto::{ use jmap_proto::types::property::Property; use store::{ U32_LEN, - write::{Archive, BatchBuilder, MaybeDynamicValue, Operation, ValueClass, ValueOp}, + write::{Archive, BatchBuilder, Operation, ValueClass, ValueOp}, }; use uri::{OwnedUri, Urn}; @@ -48,11 +48,11 @@ impl ETag for Archive { impl ExtractETag for BatchBuilder { fn etag(&self) -> Option { let p_value = u8::from(Property::Value); - for op in self.ops.iter().rev() { + for op in self.ops().iter().rev() { match op { Operation::Value { class: ValueClass::Property(p_id), - op: ValueOp::Set(MaybeDynamicValue::Static(value)), + op: ValueOp::Set(value), } if *p_id == p_value => { return value .get(value.len() - U32_LEN..) diff --git a/crates/dav/src/file/acl.rs b/crates/dav/src/file/acl.rs index 020b903c..3c735294 100644 --- a/crates/dav/src/file/acl.rs +++ b/crates/dav/src/file/acl.rs @@ -9,8 +9,8 @@ use dav_proto::RequestHeaders; use groupware::{file::FileNode, hierarchy::DavHierarchy}; use http_proto::HttpResponse; use hyper::StatusCode; -use jmap_proto::types::{acl::Acl, collection::Collection, property::Property}; -use store::write::{AlignedBytes, Archive}; +use jmap_proto::types::{acl::Acl, collection::Collection}; +use store::write::BatchBuilder; use trc::AddContext; use crate::{ @@ -49,12 +49,7 @@ impl FileAclRequestHandler for Server { // Fetch node let node_ = self - .get_property::>( - account_id, - Collection::FileNode, - resource.resource, - Property::Value, - ) + .get_archive(account_id, Collection::FileNode, resource.resource) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -87,17 +82,18 @@ impl FileAclRequestHandler for Server { { let mut new_node = node.deserialize().caused_by(trc::location!())?; new_node.acls = grants; + let mut batch = BatchBuilder::new(); update_file_node( - self, access_token, node, new_node, account_id, resource.resource, false, + &mut batch, ) - .await .caused_by(trc::location!())?; + self.commit_batch(batch).await.caused_by(trc::location!())?; } Ok(HttpResponse::new(StatusCode::OK)) diff --git a/crates/dav/src/file/copy_move.rs b/crates/dav/src/file/copy_move.rs index 80f0beb6..b8835b13 100644 --- a/crates/dav/src/file/copy_move.rs +++ b/crates/dav/src/file/copy_move.rs @@ -11,12 +11,10 @@ use dav_proto::{Depth, RequestHeaders}; use groupware::{file::FileNode, hierarchy::DavHierarchy}; use http_proto::HttpResponse; use hyper::StatusCode; -use jmap_proto::types::{ - acl::Acl, collection::Collection, property::Property, type_state::DataType, -}; +use jmap_proto::types::{acl::Acl, collection::Collection}; use store::{ ahash::AHashMap, - write::{AlignedBytes, Archive, BatchBuilder, log::ChangeLogBuilder, now}, + write::{BatchBuilder, now}, }; use trc::AddContext; use utils::map::bitmap::Bitmap; @@ -113,28 +111,29 @@ impl FileCopyMoveRequestHandler for Server { .ok_or(DavError::Code(StatusCode::BAD_GATEWAY))?; let mut delete_destination = None; // Check if the resource exists - let mut destination = if let Some((destination, new_name)) = - to_files.map_parent::(destination_resource_name) - { - if let Some(mut existing_destination) = to_files - .paths - .by_name(destination_resource_name) - .map(Destination::from_dav_resource) - { - if !headers.overwrite_fail { - existing_destination.account_id = to_account_id; - delete_destination = Some(existing_destination); - } else { - return Ok(HttpResponse::new(StatusCode::PRECONDITION_FAILED)); + let mut destination = + if let Some((destination, new_name)) = to_files.map_parent(destination_resource_name) { + if let Some(mut existing_destination) = to_files + .paths + .by_name(destination_resource_name) + .map(Destination::from_dav_resource) + { + if !headers.overwrite_fail { + existing_destination.account_id = to_account_id; + delete_destination = Some(existing_destination); + } else { + return Ok(HttpResponse::new(StatusCode::PRECONDITION_FAILED)); + } } - } - let mut destination = destination.unwrap_or_default(); - destination.new_name = Some(new_name.to_string()); - destination - } else { - return Err(DavError::Code(StatusCode::CONFLICT)); - }; + let mut destination = destination + .map(Destination::from_dav_resource) + .unwrap_or_default(); + destination.new_name = Some(new_name.to_string()); + destination + } else { + return Err(DavError::Code(StatusCode::CONFLICT)); + }; destination.account_id = to_account_id; if from_account_id == destination.account_id && delete_destination.is_none() { @@ -334,12 +333,7 @@ async fn move_container( return Err(DavError::Code(StatusCode::BAD_GATEWAY)); } let node_ = server - .get_property::>( - from_account_id, - Collection::FileNode, - from_document_id, - Property::Value, - ) + .get_archive(from_account_id, Collection::FileNode, from_document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -351,17 +345,21 @@ async fn move_container( if let Some(new_name) = destination.new_name { new_node.name = new_name; } + let mut batch = BatchBuilder::new(); let etag = update_file_node( - server, access_token, node, new_node, from_account_id, from_document_id, true, + &mut batch, ) - .await .caused_by(trc::location!())?; + server + .commit_batch(batch) + .await + .caused_by(trc::location!())?; Ok(HttpResponse::new(StatusCode::CREATED).with_etag_opt(etag)) } else { @@ -418,6 +416,7 @@ async fn copy_container( }; // Top-down copy + let mut batch = BatchBuilder::new(); let mut id_map = AHashMap::with_capacity(copy_files.len()); let mut delete_files = if delete_source { Vec::with_capacity(copy_files.len()) @@ -425,17 +424,15 @@ async fn copy_container( Vec::new() }; copy_files.sort_unstable_by(|a, b| a.1.cmp(&b.1)); - let change_id = server.generate_snowflake_id()?; - let mut changes = ChangeLogBuilder::with_change_id(change_id); let now = now() as i64; + let mut next_document_id = server + .store() + .assign_document_ids(to_account_id, Collection::FileNode, copy_files.len() as u64) + .await + .caused_by(trc::location!())?; for (document_id, _) in copy_files.into_iter() { let node_ = server - .get_property::>( - from_account_id, - Collection::FileNode, - document_id, - Property::Value, - ) + .get_archive(from_account_id, Collection::FileNode, document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))? @@ -462,46 +459,26 @@ async fn copy_container( }; // Prepare write batch - let mut batch = BatchBuilder::new(); + let new_document_id = next_document_id; + next_document_id -= 1; batch - .with_change_id(change_id) .with_account_id(to_account_id) .with_collection(Collection::FileNode) - .create_document() + .create_document(new_document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(node) .with_tenant_id(access_token), ) - .caused_by(trc::location!())?; - let new_document_id = server - .store() - .write(batch) - .await .caused_by(trc::location!())? - .last_document_id() - .caused_by(trc::location!())?; - changes.log_insert(Collection::FileNode, new_document_id); + .commit_point(); id_map.insert(document_id + 1, new_document_id + 1); } - // Write changes - if !changes.is_empty() { - server - .commit_changes(to_account_id, changes) - .await - .caused_by(trc::location!())?; - server - .broadcast_single_state_change(to_account_id, change_id, DataType::FileNode) - .await; - } - // Delete nodes if !delete_files.is_empty() { - let mut changes = ChangeLogBuilder::with_change_id(change_id); for (document_id, node) in delete_files.into_iter().rev() { // Delete record - let mut batch = BatchBuilder::new(); batch .with_account_id(from_account_id) .with_collection(Collection::FileNode) @@ -511,25 +488,17 @@ async fn copy_container( .with_tenant_id(access_token) .with_current(node), ) - .caused_by(trc::location!())?; - server - .store() - .write(batch) - .await - .caused_by(trc::location!())?; - changes.log_delete(Collection::FileNode, document_id); + .caused_by(trc::location!())? + .commit_point(); } + } - // Write changes - if !changes.is_empty() { - server - .commit_changes(from_account_id, changes) - .await - .caused_by(trc::location!())?; - server - .broadcast_single_state_change(from_account_id, change_id, DataType::FileNode) - .await; - } + // Write changes + if !batch.is_empty() { + server + .commit_batch(batch) + .await + .caused_by(trc::location!())?; } Ok(HttpResponse::new(StatusCode::CREATED)) @@ -549,12 +518,7 @@ async fn overwrite_and_delete_item( // dest_node is the current file at the destination let dest_node_ = server - .get_property::>( - to_account_id, - Collection::FileNode, - to_document_id, - Property::Value, - ) + .get_archive(to_account_id, Collection::FileNode, to_document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -565,12 +529,7 @@ async fn overwrite_and_delete_item( // source_node is the file to be copied let source_node__ = server - .get_property::>( - from_account_id, - Collection::FileNode, - from_document_id, - Property::Value, - ) + .get_archive(from_account_id, Collection::FileNode, from_document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -585,27 +544,30 @@ async fn overwrite_and_delete_item( }; source_node.parent_id = dest_node.inner.parent_id.into(); + let mut batch = BatchBuilder::new(); let etag = update_file_node( - server, access_token, dest_node, source_node, to_account_id, to_document_id, true, + &mut batch, ) - .await .caused_by(trc::location!())?; delete_file_node( - server, access_token, source_node_, from_account_id, from_document_id, + &mut batch, ) - .await .caused_by(trc::location!())?; + server + .commit_batch(batch) + .await + .caused_by(trc::location!())?; Ok(HttpResponse::new(StatusCode::NO_CONTENT).with_etag_opt(etag)) } @@ -624,12 +586,7 @@ async fn overwrite_item( // dest_node is the current file at the destination let dest_node_ = server - .get_property::>( - to_account_id, - Collection::FileNode, - to_document_id, - Property::Value, - ) + .get_archive(to_account_id, Collection::FileNode, to_document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -640,12 +597,7 @@ async fn overwrite_item( // source_node is the file to be copied let mut source_node = server - .get_property::>( - from_account_id, - Collection::FileNode, - from_document_id, - Property::Value, - ) + .get_archive(from_account_id, Collection::FileNode, from_document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))? @@ -657,18 +609,21 @@ async fn overwrite_item( dest_node.inner.name.to_string() }; source_node.parent_id = dest_node.inner.parent_id.into(); - + let mut batch = BatchBuilder::new(); let etag = update_file_node( - server, access_token, dest_node, source_node, to_account_id, to_document_id, true, + &mut batch, ) - .await .caused_by(trc::location!())?; + server + .commit_batch(batch) + .await + .caused_by(trc::location!())?; Ok(HttpResponse::new(StatusCode::NO_CONTENT).with_etag_opt(etag)) } @@ -686,12 +641,7 @@ async fn move_item( let parent_id = destination.document_id.map(|id| id + 1).unwrap_or(0); let node_ = server - .get_property::>( - from_account_id, - Collection::FileNode, - from_document_id, - Property::Value, - ) + .get_archive(from_account_id, Collection::FileNode, from_document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -704,35 +654,49 @@ async fn move_item( new_node.name = new_name; } + let mut batch = BatchBuilder::new(); let etag = if from_account_id == to_account_id { // Destination is in the same account: just update the parent id update_file_node( - server, access_token, node, new_node, from_account_id, from_document_id, true, + &mut batch, ) - .await .caused_by(trc::location!())? } else { // Destination is in a different account: insert a new node, then delete the old one - let etag = insert_file_node(server, access_token, new_node, to_account_id, true) + let to_document_id = server + .store() + .assign_document_ids(to_account_id, Collection::FileNode, 1) .await .caused_by(trc::location!())?; + let etag = insert_file_node( + access_token, + new_node, + to_account_id, + to_document_id, + true, + &mut batch, + ) + .caused_by(trc::location!())?; delete_file_node( - server, access_token, node, from_account_id, from_document_id, + &mut batch, ) - .await .caused_by(trc::location!())?; etag }; + server + .commit_batch(batch) + .await + .caused_by(trc::location!())?; Ok(HttpResponse::new(StatusCode::CREATED).with_etag_opt(etag)) } @@ -750,12 +714,7 @@ async fn copy_item( let parent_id = destination.document_id.map(|id| id + 1).unwrap_or(0); let mut node = server - .get_property::>( - from_account_id, - Collection::FileNode, - from_document_id, - Property::Value, - ) + .get_archive(from_account_id, Collection::FileNode, from_document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))? @@ -765,7 +724,23 @@ async fn copy_item( if let Some(new_name) = destination.new_name { node.name = new_name; } - let etag = insert_file_node(server, access_token, node, to_account_id, true) + let mut batch = BatchBuilder::new(); + let to_document_id = server + .store() + .assign_document_ids(to_account_id, Collection::FileNode, 1) + .await + .caused_by(trc::location!())?; + let etag = insert_file_node( + access_token, + node, + to_account_id, + to_document_id, + true, + &mut batch, + ) + .caused_by(trc::location!())?; + server + .commit_batch(batch) .await .caused_by(trc::location!())?; @@ -783,12 +758,7 @@ async fn rename_item( let from_document_id = from_resource.resource.document_id; let node_ = server - .get_property::>( - from_account_id, - Collection::FileNode, - from_document_id, - Property::Value, - ) + .get_archive(from_account_id, Collection::FileNode, from_document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -799,17 +769,21 @@ async fn rename_item( if let Some(new_name) = destination.new_name { new_node.name = new_name; } + let mut batch = BatchBuilder::new(); let etag = update_file_node( - server, access_token, node, new_node, from_account_id, from_document_id, true, + &mut batch, ) - .await .caused_by(trc::location!())?; + server + .commit_batch(batch) + .await + .caused_by(trc::location!())?; Ok(HttpResponse::new(StatusCode::CREATED).with_etag_opt(etag)) } diff --git a/crates/dav/src/file/delete.rs b/crates/dav/src/file/delete.rs index c6afb742..849816f7 100644 --- a/crates/dav/src/file/delete.rs +++ b/crates/dav/src/file/delete.rs @@ -9,10 +9,8 @@ use dav_proto::RequestHeaders; use groupware::{file::FileNode, hierarchy::DavHierarchy}; use http_proto::HttpResponse; use hyper::StatusCode; -use jmap_proto::types::{ - acl::Acl, collection::Collection, property::Property, type_state::DataType, -}; -use store::write::{AlignedBytes, Archive, BatchBuilder, log::ChangeLogBuilder}; +use jmap_proto::types::{acl::Acl, collection::Collection}; +use store::write::BatchBuilder; use trc::AddContext; use crate::{ @@ -106,23 +104,17 @@ pub(crate) async fn delete_files( ids: Vec, ) -> trc::Result<()> { // Process deletions - let mut changes = ChangeLogBuilder::new(); - + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .with_collection(Collection::FileNode); for document_id in ids { if let Some(node) = server - .get_property::>( - account_id, - Collection::FileNode, - document_id, - Property::Value, - ) + .get_archive(account_id, Collection::FileNode, document_id) .await? { // Delete record - let mut batch = BatchBuilder::new(); batch - .with_account_id(account_id) - .with_collection(Collection::FileNode) .delete_document(document_id) .custom( ObjectIndexBuilder::<_, ()>::new() @@ -132,25 +124,17 @@ pub(crate) async fn delete_files( .caused_by(trc::location!())?, ), ) - .caused_by(trc::location!())?; - server - .store() - .write(batch) - .await - .caused_by(trc::location!())?; - changes.log_delete(Collection::FileNode, document_id); + .caused_by(trc::location!())? + .commit_point(); } } // Write changes - if !changes.is_empty() { - let change_id = server - .commit_changes(account_id, changes) + if !batch.is_empty() { + server + .commit_batch(batch) .await .caused_by(trc::location!())?; - server - .broadcast_single_state_change(account_id, change_id, DataType::FileNode) - .await; } Ok(()) diff --git a/crates/dav/src/file/get.rs b/crates/dav/src/file/get.rs index 782a9571..b4882dc1 100644 --- a/crates/dav/src/file/get.rs +++ b/crates/dav/src/file/get.rs @@ -9,8 +9,7 @@ use dav_proto::{RequestHeaders, schema::property::Rfc1123DateTime}; use groupware::{file::FileNode, hierarchy::DavHierarchy}; use http_proto::HttpResponse; use hyper::StatusCode; -use jmap_proto::types::{acl::Acl, collection::Collection, property::Property}; -use store::write::{AlignedBytes, Archive}; +use jmap_proto::types::{acl::Acl, collection::Collection}; use trc::AddContext; use crate::{ @@ -53,12 +52,7 @@ impl FileGetRequestHandler for Server { // Fetch node let node_ = self - .get_property::>( - account_id, - Collection::FileNode, - resource.resource, - Property::Value, - ) + .get_archive(account_id, Collection::FileNode, resource.resource) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/file/mkcol.rs b/crates/dav/src/file/mkcol.rs index c0fd49e2..059cf2df 100644 --- a/crates/dav/src/file/mkcol.rs +++ b/crates/dav/src/file/mkcol.rs @@ -12,8 +12,8 @@ use dav_proto::{ use groupware::{file::FileNode, hierarchy::DavHierarchy}; use http_proto::HttpResponse; use hyper::StatusCode; -use jmap_proto::types::{acl::Acl, collection::Collection, type_state::DataType}; -use store::write::{BatchBuilder, log::LogInsert, now}; +use jmap_proto::types::{acl::Acl, collection::Collection}; +use store::write::{BatchBuilder, now}; use trc::AddContext; use crate::{ @@ -84,7 +84,6 @@ impl FileMkColRequestHandler for Server { .await?; // Build file container - let change_id = self.generate_snowflake_id().caused_by(trc::location!())?; let now = now(); let mut node = FileNode { parent_id, @@ -114,23 +113,20 @@ impl FileMkColRequestHandler for Server { } // Prepare write batch - let mut batch = BatchBuilder::new(); - batch - .with_change_id(change_id) - .with_account_id(account_id) - .with_collection(Collection::FileNode) - .create_document() - .log(LogInsert()) - .custom(ObjectIndexBuilder::<(), _>::new().with_changes(node)) - .caused_by(trc::location!())?; - self.store() - .write(batch) + let document_id = self + .store() + .assign_document_ids(account_id, Collection::FileNode, 1) .await .caused_by(trc::location!())?; + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .with_collection(Collection::FileNode) + .create_document(document_id) + .custom(ObjectIndexBuilder::<(), _>::new().with_changes(node)) + .caused_by(trc::location!())?; + self.commit_batch(batch).await.caused_by(trc::location!())?; - // Broadcast state change - self.broadcast_single_state_change(account_id, change_id, DataType::FileNode) - .await; if let Some(prop_stat) = return_prop_stat { Ok(HttpResponse::new(StatusCode::CREATED).with_xml_body( MkColResponse::new(prop_stat) diff --git a/crates/dav/src/file/mod.rs b/crates/dav/src/file/mod.rs index 059d7520..04fc688b 100644 --- a/crates/dav/src/file/mod.rs +++ b/crates/dav/src/file/mod.rs @@ -4,17 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{ - DavResource, DavResources, Server, auth::AccessToken, storage::index::ObjectIndexBuilder, -}; +use common::{DavResource, DavResources, auth::AccessToken, storage::index::ObjectIndexBuilder}; use groupware::file::{ArchivedFileNode, FileNode}; use hyper::StatusCode; -use jmap_proto::types::{collection::Collection, type_state::DataType}; -use store::write::{ - Archive, BatchBuilder, - log::{Changes, LogInsert}, - now, -}; +use jmap_proto::types::collection::Collection; +use store::write::{Archive, BatchBuilder, now}; use crate::{ DavError, @@ -49,8 +43,7 @@ pub(crate) trait DavFileResource { resource: &OwnedUri<'_>, ) -> crate::Result>; - fn map_parent<'x, T: FromDavResource>(&self, resource: &'x str) - -> Option<(Option, &'x str)>; + fn map_parent<'x>(&self, resource: &'x str) -> Option<(Option<&DavResource>, &'x str)>; #[allow(clippy::type_complexity)] fn map_parent_resource<'x, T: FromDavResource>( @@ -75,15 +68,9 @@ impl DavFileResource for DavResources { .ok_or(DavError::Code(StatusCode::NOT_FOUND)) } - fn map_parent<'x, T: FromDavResource>( - &self, - resource: &'x str, - ) -> Option<(Option, &'x str)> { + fn map_parent<'x>(&self, resource: &'x str) -> Option<(Option<&DavResource>, &'x str)> { let (parent, child) = if let Some((parent, child)) = resource.rsplit_once('/') { - ( - Some(self.paths.by_name(parent).map(T::from_dav_resource)?), - child, - ) + (Some(self.paths.by_name(parent)?), child) } else { (None, resource) }; @@ -98,10 +85,10 @@ impl DavFileResource for DavResources { if let Some(r) = resource.resource { if self.paths.by_name(r).is_none() { self.map_parent(r) - .map(|r| UriResource { + .map(|(parent, child)| UriResource { collection: resource.collection, account_id: resource.account_id, - resource: r, + resource: (parent.map(T::from_dav_resource), child), }) .ok_or(DavError::Code(StatusCode::CONFLICT)) } else { @@ -129,50 +116,39 @@ impl FromDavResource for FileItemId { } } -pub(crate) async fn update_file_node( - server: &Server, +pub(crate) fn update_file_node( access_token: &AccessToken, node: Archive<&ArchivedFileNode>, mut new_node: FileNode, account_id: u32, document_id: u32, with_etag: bool, + batch: &mut BatchBuilder, ) -> trc::Result> { // Build node new_node.modified = now() as i64; - let change_id = server.generate_snowflake_id()?; - - // Prepare write batch - let mut batch = BatchBuilder::new(); batch - .with_change_id(change_id) .with_account_id(account_id) .with_collection(Collection::FileNode) .update_document(document_id) - .log(Changes::update([document_id])) .custom( ObjectIndexBuilder::new() .with_current(node) .with_changes(new_node) .with_tenant_id(access_token), - )?; - let etag = if with_etag { batch.etag() } else { None }; - server.store().write(batch).await?; + )? + .commit_point(); - // Broadcast state change - server - .broadcast_single_state_change(account_id, change_id, DataType::FileNode) - .await; - - Ok(etag) + Ok(if with_etag { batch.etag() } else { None }) } -pub(crate) async fn insert_file_node( - server: &Server, +pub(crate) fn insert_file_node( access_token: &AccessToken, mut node: FileNode, account_id: u32, + document_id: u32, with_etag: bool, + batch: &mut BatchBuilder, ) -> trc::Result> { // Build node let now = now() as i64; @@ -180,57 +156,37 @@ pub(crate) async fn insert_file_node( node.created = now; // Prepare write batch - let mut batch = BatchBuilder::new(); - let change_id = server.generate_snowflake_id()?; batch - .with_change_id(change_id) .with_account_id(account_id) .with_collection(Collection::FileNode) - .create_document() - .log(LogInsert()) + .create_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(node) .with_tenant_id(access_token), - )?; - let etag = if with_etag { batch.etag() } else { None }; + )? + .commit_point(); - server.store().write(batch).await?; - - // Broadcast state change - server - .broadcast_single_state_change(account_id, change_id, DataType::FileNode) - .await; - - Ok(etag) + Ok(if with_etag { batch.etag() } else { None }) } -pub(crate) async fn delete_file_node( - server: &Server, +pub(crate) fn delete_file_node( access_token: &AccessToken, node: Archive<&ArchivedFileNode>, account_id: u32, document_id: u32, + batch: &mut BatchBuilder, ) -> trc::Result<()> { // Prepare write batch - let mut batch = BatchBuilder::new(); - let change_id = server.generate_snowflake_id()?; batch - .with_change_id(change_id) .with_account_id(account_id) .with_collection(Collection::FileNode) .delete_document(document_id) - .log(Changes::delete([document_id])) .custom( ObjectIndexBuilder::<_, ()>::new() .with_current(node) .with_tenant_id(access_token), - )?; - server.store().write(batch).await?; - - // Broadcast state change - server - .broadcast_single_state_change(account_id, change_id, DataType::FileNode) - .await; + )? + .commit_point(); Ok(()) } diff --git a/crates/dav/src/file/propfind.rs b/crates/dav/src/file/propfind.rs index 86c116ce..905810f1 100644 --- a/crates/dav/src/file/propfind.rs +++ b/crates/dav/src/file/propfind.rs @@ -19,7 +19,7 @@ use dav_proto::schema::{ use groupware::{file::FileNode, hierarchy::DavHierarchy}; use http_proto::HttpResponse; use hyper::StatusCode; -use jmap_proto::types::{acl::Acl, collection::Collection, property::Property}; +use jmap_proto::types::{acl::Acl, collection::Collection}; use store::{ ahash::AHashMap, dispatch::DocumentSet, @@ -268,7 +268,6 @@ impl HandleFilePropFindRequest for Server { account_id, Collection::FileNode, &paths, - Property::Value, |document_id, node_| { let node = node_.unarchive::().caused_by(trc::location!())?; let item = paths.items.get(&document_id).unwrap(); diff --git a/crates/dav/src/file/proppatch.rs b/crates/dav/src/file/proppatch.rs index b4c07652..0f4c5390 100644 --- a/crates/dav/src/file/proppatch.rs +++ b/crates/dav/src/file/proppatch.rs @@ -16,8 +16,8 @@ use dav_proto::{ use groupware::{file::FileNode, hierarchy::DavHierarchy}; use http_proto::HttpResponse; use hyper::StatusCode; -use jmap_proto::types::{acl::Acl, collection::Collection, property::Property}; -use store::write::{AlignedBytes, Archive}; +use jmap_proto::types::{acl::Acl, collection::Collection}; +use store::write::BatchBuilder; use trc::AddContext; use crate::{ @@ -75,12 +75,7 @@ impl FilePropPatchRequestHandler for Server { // Fetch node let node_ = self - .get_property::>( - account_id, - Collection::FileNode, - resource.resource, - Property::Value, - ) + .get_archive(account_id, Collection::FileNode, resource.resource) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -138,17 +133,19 @@ impl FilePropPatchRequestHandler for Server { } let etag = if is_success { - update_file_node( - self, + let mut batch = BatchBuilder::new(); + let etag = update_file_node( access_token, node, new_node, account_id, resource.resource, true, + &mut batch, ) - .await - .caused_by(trc::location!())? + .caused_by(trc::location!())?; + self.commit_batch(batch).await.caused_by(trc::location!())?; + etag } else { node_.etag().into() }; diff --git a/crates/dav/src/file/update.rs b/crates/dav/src/file/update.rs index 7a82e04c..fed2d91c 100644 --- a/crates/dav/src/file/update.rs +++ b/crates/dav/src/file/update.rs @@ -14,14 +14,8 @@ use groupware::{ }; use http_proto::HttpResponse; use hyper::StatusCode; -use jmap_proto::types::{ - acl::Acl, collection::Collection, property::Property, type_state::DataType, -}; -use store::write::{ - AlignedBytes, Archive, BatchBuilder, - log::{Changes, LogInsert}, - now, -}; +use jmap_proto::types::{acl::Acl, collection::Collection}; +use store::write::{BatchBuilder, now}; use trc::AddContext; use utils::BlobHash; @@ -71,12 +65,7 @@ impl FileUpdateRequestHandler for Server { if let Some(document_id) = files.paths.by_name(resource_name).map(|r| r.document_id) { // Update let node_ = self - .get_property::>( - account_id, - Collection::FileNode, - document_id, - Property::Value, - ) + .get_archive(account_id, Collection::FileNode, document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -170,7 +159,6 @@ impl FileUpdateRequestHandler for Server { .hash; // Build node - let change_id = self.generate_snowflake_id().caused_by(trc::location!())?; let mut new_node = node.deserialize::().caused_by(trc::location!())?; let new_file = new_node.file.as_mut().unwrap(); new_file.blob_hash = blob_hash; @@ -181,11 +169,9 @@ impl FileUpdateRequestHandler for Server { // Prepare write batch let mut batch = BatchBuilder::new(); batch - .with_change_id(change_id) .with_account_id(account_id) .with_collection(Collection::FileNode) .update_document(document_id) - .log(Changes::update([document_id])) .custom( ObjectIndexBuilder::new() .with_current(node) @@ -194,20 +180,13 @@ impl FileUpdateRequestHandler for Server { ) .caused_by(trc::location!())?; let etag = batch.etag(); - self.store() - .write(batch) - .await - .caused_by(trc::location!())?; - - // Broadcast state change - self.broadcast_single_state_change(account_id, change_id, DataType::FileNode) - .await; + self.commit_batch(batch).await.caused_by(trc::location!())?; Ok(HttpResponse::new(StatusCode::NO_CONTENT).with_etag_opt(etag)) } else { // Insert let orig_resource_name = resource_name; - let (parent_id, resource_name) = files + let (parent, resource_name) = files .map_parent(resource_name) .ok_or(DavError::Code(StatusCode::CONFLICT))?; @@ -217,7 +196,7 @@ impl FileUpdateRequestHandler for Server { access_token, account_id, Collection::FileNode, - parent_id, + parent.map(|r| r.document_id), Acl::AddItems, ) .await?; @@ -225,12 +204,7 @@ impl FileUpdateRequestHandler for Server { // Verify that parent is a collection if parent_id > 0 && self - .get_property::>( - account_id, - Collection::FileNode, - parent_id - 1, - Property::Value, - ) + .get_archive(account_id, Collection::FileNode, parent_id - 1) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))? @@ -275,7 +249,6 @@ impl FileUpdateRequestHandler for Server { .hash; // Build node - let change_id = self.generate_snowflake_id().caused_by(trc::location!())?; let now = now(); let node = FileNode { parent_id, @@ -295,12 +268,15 @@ impl FileUpdateRequestHandler for Server { // Prepare write batch let mut batch = BatchBuilder::new(); + let document_id = self + .store() + .assign_document_ids(account_id, Collection::FileNode, 1) + .await + .caused_by(trc::location!())?; batch - .with_change_id(change_id) .with_account_id(account_id) .with_collection(Collection::FileNode) - .create_document() - .log(LogInsert()) + .create_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(node) @@ -308,14 +284,7 @@ impl FileUpdateRequestHandler for Server { ) .caused_by(trc::location!())?; let etag = batch.etag(); - self.store() - .write(batch) - .await - .caused_by(trc::location!())?; - - // Broadcast state change - self.broadcast_single_state_change(account_id, change_id, DataType::FileNode) - .await; + self.commit_batch(batch).await.caused_by(trc::location!())?; Ok(HttpResponse::new(StatusCode::CREATED).with_etag_opt(etag)) } diff --git a/crates/directory/src/backend/internal/manage.rs b/crates/directory/src/backend/internal/manage.rs index 8c366260..b458830c 100644 --- a/crates/directory/src/backend/internal/manage.rs +++ b/crates/directory/src/backend/internal/manage.rs @@ -7,10 +7,10 @@ use ahash::{AHashMap, AHashSet}; use jmap_proto::types::collection::Collection; use store::{ - Deserialize, IterateParams, Serialize, Store, U32_LEN, ValueKey, + Deserialize, IterateParams, Serialize, SerializeInfallible, Store, U32_LEN, ValueKey, write::{ - AssignedIds, BatchBuilder, DirectoryClass, MaybeDynamicId, MaybeDynamicValue, - SerializeWithId, ValueClass, assert::LegacyHashedValue, key::DeserializeBigEndian, + BatchBuilder, DirectoryClass, ValueClass, assert::LegacyHashedValue, + key::DeserializeBigEndian, }, }; use trc::AddContext; @@ -141,6 +141,7 @@ impl ManageDirectory for Store { async fn get_or_create_principal_id(&self, name: &str, typ: Type) -> trc::Result { let mut try_count = 0; let name = name.to_lowercase(); + let mut principal_id = None; loop { // Try to obtain ID @@ -152,6 +153,17 @@ impl ManageDirectory for Store { return Ok(principal_id); } + let principal_id = if let Some(principal_id) = principal_id { + principal_id + } else { + let principal_id_ = self + .assign_document_ids(u32::MAX, Collection::Principal, 1) + .await + .caused_by(trc::location!())?; + principal_id = Some(principal_id_); + principal_id_ + }; + // Write principal ID let name_key = ValueClass::Directory(DirectoryClass::NameToId(name.as_bytes().to_vec())); @@ -160,15 +172,21 @@ impl ManageDirectory for Store { .with_account_id(u32::MAX) .with_collection(Collection::Principal) .assert_value(name_key.clone(), ()) - .create_document() - .set(name_key, DynamicPrincipalInfo::new(typ, None)) + .create_document(principal_id) .set( - ValueClass::Directory(DirectoryClass::Principal(MaybeDynamicId::Dynamic(0))), + name_key, + PrincipalInfo::new(principal_id, typ, None).serialize(), + ) + .set( + ValueClass::Directory(DirectoryClass::Principal(principal_id)), Principal { + id: principal_id, typ, ..Default::default() } - .with_field(PrincipalField::Name, name.to_string()), + .with_field(PrincipalField::Name, name.to_string()) + .serialize() + .caused_by(trc::location!())?, ); // Add default user role @@ -176,26 +194,22 @@ impl ManageDirectory for Store { batch .set( ValueClass::Directory(DirectoryClass::MemberOf { - principal_id: MaybeDynamicId::Dynamic(0), - member_of: MaybeDynamicId::Static(ROLE_USER), + principal_id, + member_of: ROLE_USER, }), vec![Type::Role as u8], ) .set( ValueClass::Directory(DirectoryClass::Members { - principal_id: MaybeDynamicId::Static(ROLE_USER), - has_member: MaybeDynamicId::Dynamic(0), + principal_id: ROLE_USER, + has_member: principal_id, }), vec![], ); } - match self - .write(batch.build()) - .await - .and_then(|r| r.last_document_id()) - { - Ok(principal_id) => { + match self.write(batch.build_all()).await { + Ok(_) => { return Ok(principal_id); } Err(err) => { @@ -442,13 +456,18 @@ impl ManageDirectory for Store { } // Write principal + let principal_id = self + .assign_document_ids(u32::MAX, Collection::Principal, 1) + .await + .caused_by(trc::location!())?; + principal.id = principal_id; let mut batch = BatchBuilder::new(); - let pinfo_name = DynamicPrincipalInfo::new(principal.typ, tenant_id); - let pinfo_email = DynamicPrincipalInfo::new(principal.typ, None); + let pinfo_name = PrincipalInfo::new(principal_id, principal.typ, tenant_id); + let pinfo_email = PrincipalInfo::new(principal_id, principal.typ, None); batch .with_account_id(u32::MAX) .with_collection(Collection::Principal) - .create_document() + .create_document(principal_id) .assert_value( ValueClass::Directory(DirectoryClass::NameToId( principal.name().to_string().into_bytes(), @@ -456,7 +475,7 @@ impl ManageDirectory for Store { (), ) .set( - ValueClass::Directory(DirectoryClass::Principal(MaybeDynamicId::Dynamic(0))), + ValueClass::Directory(DirectoryClass::Principal(principal_id)), principal.serialize().caused_by(trc::location!())?, ) .set( @@ -466,7 +485,7 @@ impl ManageDirectory for Store { .unwrap() .into_bytes(), )), - pinfo_name, + pinfo_name.serialize(), ); // Write email to id mapping @@ -477,7 +496,7 @@ impl ManageDirectory for Store { for email in emails { batch.set( ValueClass::Directory(DirectoryClass::EmailToId(email.into_bytes())), - pinfo_email, + pinfo_email.serialize(), ); } } @@ -486,15 +505,15 @@ impl ManageDirectory for Store { for member_of in member_of { batch.set( ValueClass::Directory(DirectoryClass::MemberOf { - principal_id: MaybeDynamicId::Dynamic(0), - member_of: MaybeDynamicId::Static(member_of.id), + principal_id, + member_of: member_of.id, }), vec![member_of.typ as u8], ); batch.set( ValueClass::Directory(DirectoryClass::Members { - principal_id: MaybeDynamicId::Static(member_of.id), - has_member: MaybeDynamicId::Dynamic(0), + principal_id: member_of.id, + has_member: principal_id, }), vec![], ); @@ -502,25 +521,24 @@ impl ManageDirectory for Store { for member in members { batch.set( ValueClass::Directory(DirectoryClass::MemberOf { - principal_id: MaybeDynamicId::Static(member.id), - member_of: MaybeDynamicId::Dynamic(0), + principal_id: member.id, + member_of: principal_id, }), vec![principal.typ as u8], ); batch.set( ValueClass::Directory(DirectoryClass::Members { - principal_id: MaybeDynamicId::Dynamic(0), - has_member: MaybeDynamicId::Static(member.id), + principal_id, + has_member: member.id, }), vec![], ); } - self.write(batch.build()) + self.write(batch.build_all()) .await - .and_then(|r| r.last_document_id()) - .map(|id| CreatedPrincipal { - id, + .map(|_| CreatedPrincipal { + id: principal_id, changed_principals, }) } @@ -698,9 +716,7 @@ impl ManageDirectory for Store { .unwrap_or_default() .into_bytes(), )) - .clear(DirectoryClass::Principal(MaybeDynamicId::Static( - principal_id, - ))) + .clear(DirectoryClass::Principal(principal_id)) .clear(DirectoryClass::UsedQuota(principal_id)); if let Some(emails) = principal.take_str_array(PrincipalField::Emails) { @@ -724,12 +740,12 @@ impl ManageDirectory for Store { // Remove memberOf batch.clear(DirectoryClass::MemberOf { - principal_id: MaybeDynamicId::Static(principal_id), - member_of: MaybeDynamicId::Static(member.principal_id), + principal_id, + member_of: member.principal_id, }); batch.clear(DirectoryClass::Members { - principal_id: MaybeDynamicId::Static(member.principal_id), - has_member: MaybeDynamicId::Static(principal_id), + principal_id: member.principal_id, + has_member: principal_id, }); } @@ -754,16 +770,16 @@ impl ManageDirectory for Store { // Remove members batch.clear(DirectoryClass::MemberOf { - principal_id: MaybeDynamicId::Static(member_id), - member_of: MaybeDynamicId::Static(principal_id), + principal_id: member_id, + member_of: principal_id, }); batch.clear(DirectoryClass::Members { - principal_id: MaybeDynamicId::Static(principal_id), - has_member: MaybeDynamicId::Static(member_id), + principal_id, + has_member: member_id, }); } - self.write(batch.build()) + self.write(batch.build_all()) .await .caused_by(trc::location!())?; @@ -816,12 +832,8 @@ impl ManageDirectory for Store { // Prepare changes let mut batch = BatchBuilder::new(); let mut pinfo_name = - PrincipalInfo::new(principal_id, principal_type, principal.inner.tenant()) - .serialize() - .caused_by(trc::location!())?; - let pinfo_email = PrincipalInfo::new(principal_id, principal_type, None) - .serialize() - .caused_by(trc::location!())?; + PrincipalInfo::new(principal_id, principal_type, principal.inner.tenant()).serialize(); + let pinfo_email = PrincipalInfo::new(principal_id, principal_type, None).serialize(); let update_principal = !changes.is_empty() && !changes.iter().all(|c| { matches!( @@ -835,9 +847,7 @@ impl ManageDirectory for Store { if update_principal { batch.assert_value( - ValueClass::Directory(DirectoryClass::Principal(MaybeDynamicId::Static( - principal_id, - ))), + ValueClass::Directory(DirectoryClass::Principal(principal_id)), &principal, ); } @@ -986,8 +996,7 @@ impl ManageDirectory for Store { principal.inner.set(PrincipalField::Tenant, tenant_info.id); pinfo_name = PrincipalInfo::new(principal_id, principal_type, tenant_info.id.into()) - .serialize() - .caused_by(trc::location!())?; + .serialize(); } else if let Some(tenant_id) = principal.inner.tenant() { // Update quota if let Some(used_quota) = used_quota { @@ -998,9 +1007,8 @@ impl ManageDirectory for Store { changed_principals.add_change(principal_id, principal_type, change.field); principal.inner.remove(PrincipalField::Tenant); - pinfo_name = PrincipalInfo::new(principal_id, principal_type, None) - .serialize() - .caused_by(trc::location!())?; + pinfo_name = + PrincipalInfo::new(principal_id, principal_type, None).serialize(); } else { continue; } @@ -1240,15 +1248,15 @@ impl ManageDirectory for Store { batch.set( ValueClass::Directory(DirectoryClass::MemberOf { - principal_id: MaybeDynamicId::Static(principal_id), - member_of: MaybeDynamicId::Static(member_info.id), + principal_id, + member_of: member_info.id, }), vec![member_info.typ as u8], ); batch.set( ValueClass::Directory(DirectoryClass::Members { - principal_id: MaybeDynamicId::Static(member_info.id), - has_member: MaybeDynamicId::Static(principal_id), + principal_id: member_info.id, + has_member: principal_id, }), vec![], ); @@ -1274,12 +1282,12 @@ impl ManageDirectory for Store { ); batch.clear(ValueClass::Directory(DirectoryClass::MemberOf { - principal_id: MaybeDynamicId::Static(principal_id), - member_of: MaybeDynamicId::Static(member.principal_id), + principal_id, + member_of: member.principal_id, })); batch.clear(ValueClass::Directory(DirectoryClass::Members { - principal_id: MaybeDynamicId::Static(member.principal_id), - has_member: MaybeDynamicId::Static(principal_id), + principal_id: member.principal_id, + has_member: principal_id, })); } } @@ -1318,16 +1326,16 @@ impl ManageDirectory for Store { batch.set( ValueClass::Directory(DirectoryClass::MemberOf { - principal_id: MaybeDynamicId::Static(principal_id), - member_of: MaybeDynamicId::Static(member_info.id), + principal_id, + member_of: member_info.id, }), vec![member_info.typ as u8], ); batch.set( ValueClass::Directory(DirectoryClass::Members { - principal_id: MaybeDynamicId::Static(member_info.id), - has_member: MaybeDynamicId::Static(principal_id), + principal_id: member_info.id, + has_member: principal_id, }), vec![], ); @@ -1368,13 +1376,13 @@ impl ManageDirectory for Store { ); batch.clear(ValueClass::Directory(DirectoryClass::MemberOf { - principal_id: MaybeDynamicId::Static(principal_id), - member_of: MaybeDynamicId::Static(member_info.id), + principal_id, + member_of: member_info.id, })); batch.clear(ValueClass::Directory(DirectoryClass::Members { - principal_id: MaybeDynamicId::Static(member_info.id), - has_member: MaybeDynamicId::Static(principal_id), + principal_id: member_info.id, + has_member: principal_id, })); member_of.remove(pos); @@ -1425,15 +1433,15 @@ impl ManageDirectory for Store { batch.set( ValueClass::Directory(DirectoryClass::MemberOf { - principal_id: MaybeDynamicId::Static(member_info.id), - member_of: MaybeDynamicId::Static(principal_id), + principal_id: member_info.id, + member_of: principal_id, }), vec![principal_type as u8], ); batch.set( ValueClass::Directory(DirectoryClass::Members { - principal_id: MaybeDynamicId::Static(principal_id), - has_member: MaybeDynamicId::Static(member_info.id), + principal_id, + has_member: member_info.id, }), vec![], ); @@ -1461,12 +1469,12 @@ impl ManageDirectory for Store { } batch.clear(ValueClass::Directory(DirectoryClass::MemberOf { - principal_id: MaybeDynamicId::Static(*member_id), - member_of: MaybeDynamicId::Static(principal_id), + principal_id: *member_id, + member_of: principal_id, })); batch.clear(ValueClass::Directory(DirectoryClass::Members { - principal_id: MaybeDynamicId::Static(principal_id), - has_member: MaybeDynamicId::Static(*member_id), + principal_id, + has_member: *member_id, })); } } @@ -1511,15 +1519,15 @@ impl ManageDirectory for Store { batch.set( ValueClass::Directory(DirectoryClass::MemberOf { - principal_id: MaybeDynamicId::Static(member_info.id), - member_of: MaybeDynamicId::Static(principal_id), + principal_id: member_info.id, + member_of: principal_id, }), vec![principal_type as u8], ); batch.set( ValueClass::Directory(DirectoryClass::Members { - principal_id: MaybeDynamicId::Static(principal_id), - has_member: MaybeDynamicId::Static(member_info.id), + principal_id, + has_member: member_info.id, }), vec![], ); @@ -1547,12 +1555,12 @@ impl ManageDirectory for Store { ); batch.clear(ValueClass::Directory(DirectoryClass::MemberOf { - principal_id: MaybeDynamicId::Static(member_info.id), - member_of: MaybeDynamicId::Static(principal_id), + principal_id: member_info.id, + member_of: principal_id, })); batch.clear(ValueClass::Directory(DirectoryClass::Members { - principal_id: MaybeDynamicId::Static(principal_id), - has_member: MaybeDynamicId::Static(member_info.id), + principal_id, + has_member: member_info.id, })); members.remove(pos); break; @@ -1726,14 +1734,12 @@ impl ManageDirectory for Store { if update_principal { batch.set( - ValueClass::Directory(DirectoryClass::Principal(MaybeDynamicId::Static( - principal_id, - ))), + ValueClass::Directory(DirectoryClass::Principal(principal_id)), principal.inner.serialize().caused_by(trc::location!())?, ); } - self.write(batch.build()) + self.write(batch.build_all()) .await .caused_by(trc::location!())?; @@ -2150,20 +2156,6 @@ impl PrincipalField { } } -impl SerializeWithId for Principal { - fn serialize_with_id(&self, ids: &AssignedIds) -> trc::Result> { - let mut principal = self.clone(); - principal.id = ids.last_document_id().caused_by(trc::location!())?; - principal.serialize() - } -} - -impl From for MaybeDynamicValue { - fn from(principal: Principal) -> Self { - MaybeDynamicValue::Dynamic(Box::new(principal)) - } -} - impl<'x> UpdatePrincipal<'x> { pub fn by_id(id: u32) -> Self { Self { @@ -2242,18 +2234,6 @@ fn validate_member_of( } } -#[derive(Clone, Copy)] -pub(crate) struct DynamicPrincipalInfo { - typ: Type, - tenant: Option, -} - -impl DynamicPrincipalInfo { - pub fn new(typ: Type, tenant: Option) -> Self { - Self { typ, tenant } - } -} - impl ChangedPrincipals { pub fn new() -> Self { Self::default() @@ -2388,20 +2368,6 @@ impl ChangedPrincipal { } } -impl SerializeWithId for DynamicPrincipalInfo { - fn serialize_with_id(&self, ids: &AssignedIds) -> trc::Result> { - ids.last_document_id().and_then(|principal_id| { - PrincipalInfo::new(principal_id, self.typ, self.tenant).serialize() - }) - } -} - -impl From for MaybeDynamicValue { - fn from(value: DynamicPrincipalInfo) -> Self { - MaybeDynamicValue::Dynamic(Box::new(value)) - } -} - pub fn err_missing(field: impl Into) -> trc::Error { trc::ManageEvent::MissingParameter.ctx(trc::Key::Key, field) } diff --git a/crates/directory/src/backend/internal/mod.rs b/crates/directory/src/backend/internal/mod.rs index 8c74e782..12b961cc 100644 --- a/crates/directory/src/backend/internal/mod.rs +++ b/crates/directory/src/backend/internal/mod.rs @@ -10,16 +10,8 @@ pub mod manage; use std::{fmt::Display, slice::Iter}; use ahash::AHashMap; -use jmap_proto::types::collection::Collection; -use manage::DynamicPrincipalInfo; -use store::{ - Deserialize, IterateParams, SUBSPACE_DIRECTORY, Serialize, Store, U32_LEN, ValueKey, - write::{ - AnyClass, BatchBuilder, DirectoryClass, MaybeDynamicId, ValueClass, key::KeySerializer, - }, -}; -use trc::AddContext; -use utils::codec::leb128::{Leb128Iterator, Leb128Reader}; +use store::{Deserialize, Serialize, SerializeInfallible, U32_LEN, write::key::KeySerializer}; +use utils::codec::leb128::Leb128Iterator; use crate::{Principal, ROLE_ADMIN, ROLE_USER, Type}; @@ -115,9 +107,9 @@ impl PrincipalInfo { } } -impl Serialize for PrincipalInfo { - fn serialize(&self) -> trc::Result> { - Ok(if let Some(tenant) = self.tenant { +impl SerializeInfallible for PrincipalInfo { + fn serialize(&self) -> Vec { + if let Some(tenant) = self.tenant { KeySerializer::new((U32_LEN * 2) + 1) .write_leb128(self.id) .write(self.typ as u8) @@ -128,7 +120,7 @@ impl Serialize for PrincipalInfo { .write_leb128(self.id) .write(self.typ as u8) .finalize() - }) + } } } @@ -240,7 +232,7 @@ fn deserialize(bytes: &[u8]) -> Option { } } -pub trait MigrateDirectory: Sync + Send { +/*pub trait MigrateDirectory: Sync + Send { fn migrate_directory(&self) -> impl std::future::Future> + Send; } @@ -305,9 +297,7 @@ impl MigrateDirectory for Store { .with_account_id(u32::MAX) .with_collection(Collection::Principal) .set( - ValueClass::Directory(DirectoryClass::Principal(MaybeDynamicId::Static( - account_id, - ))), + ValueClass::Directory(DirectoryClass::Principal(account_id)), principal.serialize().caused_by(trc::location!())?, ); @@ -315,21 +305,21 @@ impl MigrateDirectory for Store { batch .set( ValueClass::Directory(DirectoryClass::MemberOf { - principal_id: MaybeDynamicId::Static(account_id), - member_of: MaybeDynamicId::Static(role), + principal_id: account_id, + member_of: role, }), vec![Type::Role as u8], ) .set( ValueClass::Directory(DirectoryClass::Members { - principal_id: MaybeDynamicId::Static(role), - has_member: MaybeDynamicId::Static(account_id), + principal_id: role, + has_member: account_id, }), vec![], ); } - self.write(batch.build()) + self.write(batch.build_all()) .await .caused_by(trc::location!())?; } @@ -365,7 +355,7 @@ impl MigrateDirectory for Store { key: [3u8].iter().chain(domain.as_bytes()).copied().collect(), })); - if let Err(err) = self.write(batch.build()).await { + if let Err(err) = self.write(batch.build_all()).await { trc::error!( err.caused_by(trc::location!()) .details("Failed to migrate domain, probably a principal already exists") @@ -386,6 +376,7 @@ impl MigrateDirectory for Store { Ok(()) } } +*/ #[derive( Debug, Clone, Copy, PartialEq, Hash, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize, diff --git a/crates/email/src/identity/index.rs b/crates/email/src/identity/index.rs new file mode 100644 index 00000000..98289701 --- /dev/null +++ b/crates/email/src/identity/index.rs @@ -0,0 +1,23 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject}; + +use super::{ArchivedIdentity, Identity}; + +impl IndexableObject for Identity { + fn index_values(&self) -> impl Iterator> { + [IndexValue::LogChild { prefix: None }].into_iter() + } +} + +impl IndexableObject for &ArchivedIdentity { + fn index_values(&self) -> impl Iterator> { + [IndexValue::LogChild { prefix: None }].into_iter() + } +} + +impl IndexableAndSerializableObject for Identity {} diff --git a/crates/email/src/identity/mod.rs b/crates/email/src/identity/mod.rs index fc1279ca..fd372741 100644 --- a/crates/email/src/identity/mod.rs +++ b/crates/email/src/identity/mod.rs @@ -4,6 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +pub mod index; + use store::{SERIALIZE_OBJ_03_V1, SerializedVersion}; #[derive( diff --git a/crates/email/src/mailbox/destroy.rs b/crates/email/src/mailbox/destroy.rs index b2c3ed11..77256ca9 100644 --- a/crates/email/src/mailbox/destroy.rs +++ b/crates/email/src/mailbox/destroy.rs @@ -10,14 +10,9 @@ use common::{ use directory::Permission; use jmap_proto::{ error::set::{SetError, SetErrorType}, - types::{acl::Acl, collection::Collection, id::Id, property::Property}, -}; -use store::{ - SerializeInfallible, - query::Filter, - roaring::RoaringBitmap, - write::{AlignedBytes, Archive, BatchBuilder, log::ChangeLogBuilder}, + types::{acl::Acl, collection::Collection, property::Property}, }; +use store::{SerializeInfallible, query::Filter, roaring::RoaringBitmap, write::BatchBuilder}; use trc::AddContext; use crate::message::{delete::EmailDeletion, metadata::MessageData}; @@ -29,10 +24,9 @@ pub trait MailboxDestroy: Sync + Send { &self, account_id: u32, document_id: u32, - changes: &mut ChangeLogBuilder, access_token: &AccessToken, remove_emails: bool, - ) -> impl Future>> + Send; + ) -> impl Future, SetError>>> + Send; } impl MailboxDestroy for Server { @@ -40,10 +34,9 @@ impl MailboxDestroy for Server { &self, account_id: u32, document_id: u32, - changes: &mut ChangeLogBuilder, access_token: &AccessToken, remove_emails: bool, - ) -> trc::Result> { + ) -> trc::Result, SetError>> { // Internal folders cannot be deleted #[cfg(feature = "test_mode")] if [INBOX_ID, TRASH_ID].contains(&document_id) @@ -83,7 +76,10 @@ impl MailboxDestroy for Server { } // Verify that the mailbox is empty - let mut did_remove_emails = false; + let mut batch = BatchBuilder::new(); + + batch.with_account_id(account_id); + if let Some(message_ids) = self .get_tag( account_id, @@ -94,19 +90,14 @@ impl MailboxDestroy for Server { .await? { if remove_emails { - // Flag removal for state change notification - did_remove_emails = true; - // If the message is in multiple mailboxes, untag it from the current mailbox, // otherwise delete it. let mut destroy_ids = RoaringBitmap::new(); - let mut batch = BatchBuilder::new(); self.get_archives( account_id, Collection::Email, &message_ids, - Property::Value, |message_id, message_data_| { // Remove mailbox from list let prev_message_data = message_data_ @@ -130,7 +121,6 @@ impl MailboxDestroy for Server { let mut new_message_data = prev_message_data .deserialize() .caused_by(trc::location!())?; - let thread_id = new_message_data.thread_id; new_message_data .mailboxes @@ -138,7 +128,6 @@ impl MailboxDestroy for Server { // Untag message from mailbox batch - .with_account_id(account_id) .with_collection(Collection::Email) .update_document(message_id) .custom( @@ -146,35 +135,18 @@ impl MailboxDestroy for Server { .with_changes(new_message_data) .with_current(prev_message_data), ) - .caused_by(trc::location!())?; - changes - .log_update(Collection::Email, Id::from_parts(thread_id, message_id)); + .caused_by(trc::location!())? + .commit_point(); Ok(true) }, ) .await .caused_by(trc::location!())?; - if !batch.is_empty() { - match self.core.storage.data.write(batch.build()).await { - Ok(_) => {} - Err(err) if err.is_assertion_failure() => { - return Ok(Err(SetError::forbidden().with_description(concat!( - "Another process modified a message in this mailbox ", - "while deleting it, please try again." - )))); - } - Err(err) => { - return Err(err.caused_by(trc::location!())); - } - } - } - // Bulk delete messages if !destroy_ids.is_empty() { - let (mut change, _) = self.emails_tombstone(account_id, destroy_ids).await?; - change.changes.remove(&(Collection::Mailbox as u8)); - changes.merge(change); + self.emails_tombstone(account_id, &mut batch, destroy_ids) + .await?; } } else { return Ok(Err(SetError::new(SetErrorType::MailboxHasEmail) @@ -184,12 +156,7 @@ impl MailboxDestroy for Server { // Obtain mailbox if let Some(mailbox_) = self - .get_property::>( - account_id, - Collection::Mailbox, - document_id, - Property::Value, - ) + .get_archive(account_id, Collection::Mailbox, document_id) .await .caused_by(trc::location!())? { @@ -210,8 +177,6 @@ impl MailboxDestroy for Server { } } } - - let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Mailbox) @@ -219,21 +184,23 @@ impl MailboxDestroy for Server { .clear(Property::EmailIds) .custom(ObjectIndexBuilder::<_, ()>::new().with_current(mailbox)) .caused_by(trc::location!())?; + } else { + return Ok(Err(SetError::not_found())); + }; - match self.core.storage.data.write(batch.build()).await { - Ok(_) => { - changes.log_delete(Collection::Mailbox, document_id); - Ok(Ok(did_remove_emails)) - } + if !batch.is_empty() { + let change_id = batch.change_id(); + match self.commit_batch(batch).await { + Ok(_) => Ok(Ok(Some(change_id))), Err(err) if err.is_assertion_failure() => Ok(Err(SetError::forbidden() .with_description(concat!( - "Another process modified this mailbox ", + "Another process modified a message in this mailbox ", "while deleting it, please try again." )))), Err(err) => Err(err.caused_by(trc::location!())), } } else { - Ok(Err(SetError::not_found())) + Ok(Ok(None)) } } } diff --git a/crates/email/src/mailbox/index.rs b/crates/email/src/mailbox/index.rs index 6acffb9a..c2e70889 100644 --- a/crates/email/src/mailbox/index.rs +++ b/crates/email/src/mailbox/index.rs @@ -46,6 +46,7 @@ impl IndexableObject for Mailbox { field: Property::IsSubscribed.into(), value: self.subscribers.iter().map(Into::into).collect::>(), }, + IndexValue::LogChild { prefix: None }, IndexValue::Acl { value: (&self.acls).into(), }, @@ -85,6 +86,7 @@ impl IndexableObject for &ArchivedMailbox { field: Property::IsSubscribed.into(), value: self.subscribers.iter().map(Into::into).collect::>(), }, + IndexValue::LogChild { prefix: None }, IndexValue::Acl { value: self .acls diff --git a/crates/email/src/mailbox/manage.rs b/crates/email/src/mailbox/manage.rs index ea354bb8..45cff535 100644 --- a/crates/email/src/mailbox/manage.rs +++ b/crates/email/src/mailbox/manage.rs @@ -31,7 +31,7 @@ pub trait MailboxFnc: Sync + Send { &self, account_id: u32, path: &str, - ) -> impl Future)>>> + Send; + ) -> impl Future>> + Send; fn mailbox_count_threads( &self, @@ -101,26 +101,27 @@ impl MailboxFnc for Server { object.add_subscriber(account_id); } batch - .create_document_with_id(document_id) + .create_document(document_id) .custom(ObjectIndexBuilder::<(), _>::new().with_changes(object)) .caused_by(trc::location!())?; mailbox_ids.insert(document_id); } + self.store() + .assign_document_ids(account_id, Collection::Mailbox, (ARCHIVE_ID + 1) as u64) + .await + .caused_by(trc::location!())?; self.core .storage .data - .write(batch.build()) + .write(batch.build_all()) .await - .caused_by(trc::location!()) - .map(|_| mailbox_ids) + .caused_by(trc::location!())?; + + Ok(mailbox_ids) } - async fn mailbox_create_path( - &self, - account_id: u32, - path: &str, - ) -> trc::Result)>> { + async fn mailbox_create_path(&self, account_id: u32, path: &str) -> trc::Result> { let folders = self .fetch_folders::(account_id, Collection::Mailbox) .await @@ -165,47 +166,38 @@ impl MailboxFnc for Server { // Create missing folders if !create_paths.is_empty() { - let mut changes = self.begin_changes(account_id)?; + if create_paths + .iter() + .any(|name| name.len() > self.core.jmap.mailbox_name_max_len) + { + return Ok(None); + } + let mut next_document_id = self + .store() + .assign_document_ids(account_id, Collection::Mailbox, create_paths.len() as u64) + .await + .caused_by(trc::location!())?; + let mut batch = BatchBuilder::new(); for name in create_paths { - if name.len() > self.core.jmap.mailbox_name_max_len { - return Ok(None); - } - let mut batch = BatchBuilder::new(); + let document_id = next_document_id; + next_document_id -= 1; batch .with_account_id(account_id) .with_collection(Collection::Mailbox) - .create_document() + .create_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(Mailbox::new(name).with_parent_id(next_parent_id)), ) .caused_by(trc::location!())?; - let document_id = self - .store() - .write_expect_id(batch) - .await - .caused_by(trc::location!())?; - changes.log_insert(Collection::Mailbox, document_id); next_parent_id = document_id + 1; } - let change_id = changes.change_id; - let mut batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_collection(Collection::Mailbox) - .custom(changes) - .caused_by(trc::location!())?; - self.store() - .write(batch.build()) - .await - .caused_by(trc::location!())?; - - Ok(Some((next_parent_id - 1, Some(change_id)))) - } else { - Ok(Some((next_parent_id - 1, None))) + self.commit_batch(batch).await.caused_by(trc::location!())?; } + + Ok(Some(next_parent_id - 1)) } async fn mailbox_count_threads( diff --git a/crates/email/src/message/bayes.rs b/crates/email/src/message/bayes.rs index 4e7f912b..522a34f2 100644 --- a/crates/email/src/message/bayes.rs +++ b/crates/email/src/message/bayes.rs @@ -12,7 +12,7 @@ use mail_parser::Message; use spam_filter::{ SpamFilterInput, analysis::init::SpamFilterInit, modules::bayes::BayesClassifier, }; -use store::write::{AlignedBytes, Archive, TaskQueueClass}; +use store::write::TaskQueueClass; use trc::StoreEvent; use utils::BlobHash; @@ -59,7 +59,7 @@ impl EmailBayesTrain for Server { learn_spam: bool, ) -> trc::Result { let metadata = self - .get_property::>( + .get_archive_by_property( account_id, Collection::Email, document_id, @@ -74,7 +74,7 @@ impl EmailBayesTrain for Server { })?; Ok(TaskQueueClass::BayesTrain { - seq: self.generate_snowflake_id()?, + seq: self.generate_snowflake_id(), hash: BlobHash::from(&metadata.unarchive::()?.blob_hash), learn_spam, }) diff --git a/crates/email/src/message/copy.rs b/crates/email/src/message/copy.rs index d96be1a2..85e3335d 100644 --- a/crates/email/src/message/copy.rs +++ b/crates/email/src/message/copy.rs @@ -15,15 +15,15 @@ use jmap_proto::{ use mail_parser::parsers::fields::thread::thread_name; use store::{ BlobClass, - write::{AlignedBytes, Archive, BatchBuilder, TaskQueueClass, ValueClass, log::Changes}, + write::{BatchBuilder, TaskQueueClass, ValueClass}, }; use trc::AddContext; -use crate::mailbox::UidMailbox; +use crate::{mailbox::UidMailbox, thread::cache::ThreadCache}; use super::{ index::{MAX_ID_LENGTH, MAX_SORT_FIELD_LENGTH, TrimTextValue}, - ingest::{EmailIngest, IngestedEmail, LogEmailInsert}, + ingest::{EmailIngest, IngestedEmail, ThreadResult}, metadata::{HeaderName, HeaderValue, MessageData, MessageMetadata}, }; @@ -56,7 +56,7 @@ impl EmailCopy for Server { // Obtain metadata let account_id = resource_token.account_id; let mut metadata = if let Some(metadata) = self - .get_property::>( + .get_archive_by_property( from_account_id, Collection::Email, from_message_id, @@ -100,12 +100,18 @@ impl EmailCopy for Server { // Obtain threadId let mut references = Vec::with_capacity(5); let mut subject = ""; + let mut message_id = ""; for header in &metadata.contents[0].parts[0].headers { match &header.name { - HeaderName::MessageId - | HeaderName::InReplyTo - | HeaderName::References - | HeaderName::ResentMessageId => { + HeaderName::MessageId => { + header.value.visit_text(|id| { + if !id.is_empty() && id.len() < MAX_ID_LENGTH { + references.push(id.as_bytes()); + message_id = id; + } + }); + } + HeaderName::InReplyTo | HeaderName::References | HeaderName::ResentMessageId => { header.value.visit_text(|id| { if !id.is_empty() && id.len() < MAX_ID_LENGTH { references.push(id.as_bytes()); @@ -127,10 +133,21 @@ impl EmailCopy for Server { } // Obtain threadId - let thread_id = self + let (is_new_thread, thread_id) = match self .find_or_merge_thread(account_id, subject, references, None) .await - .caused_by(trc::location!())?; + .caused_by(trc::location!())? + { + ThreadResult::Id(thread_id) => (false, thread_id), + ThreadResult::Create => ( + true, + self.get_cached_thread_ids(account_id) + .await + .caused_by(trc::location!())? + .assign_thread_id(subject.as_bytes(), message_id.as_bytes()), + ), + ThreadResult::Skip => unreachable!(), + }; // Assign id let mut email = IngestedEmail { @@ -152,18 +169,26 @@ impl EmailCopy for Server { } // Prepare batch - let change_id = self.assign_change_id(account_id)?; let mut batch = BatchBuilder::new(); + let change_id = batch.change_id(); + batch.with_account_id(account_id); + + if is_new_thread { + batch + .with_collection(Collection::Thread) + .update_document(thread_id) + .log_insert(None); + } + + let document_id = self + .store() + .assign_document_ids(account_id, Collection::Email, 1) + .await + .caused_by(trc::location!())?; + batch - .with_account_id(account_id) - .with_change_id(change_id) - .with_collection(Collection::Thread) - .log(Changes::update([thread_id])) - .with_collection(Collection::Mailbox) - .log(Changes::child_update(mailboxes.iter().copied())) .with_collection(Collection::Email) - .create_document() - .log(LogEmailInsert::new(thread_id.into())) + .create_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new().with_changes(MessageData { mailboxes: mailbox_ids, @@ -175,7 +200,7 @@ impl EmailCopy for Server { .caused_by(trc::location!())? .set( ValueClass::TaskQueue(TaskQueueClass::IndexEmail { - seq: self.generate_snowflake_id()?, + seq: change_id, hash: metadata.blob_hash.clone(), }), vec![], @@ -190,14 +215,10 @@ impl EmailCopy for Server { .caused_by(trc::location!())?; // Insert and obtain ids - let ids = self - .core - .storage - .data - .write(batch.build()) + self.store() + .write(batch.build_all()) .await .caused_by(trc::location!())?; - let document_id = ids.last_document_id().caused_by(trc::location!())?; // Request FTS index self.notify_task_queue(); diff --git a/crates/email/src/message/delete.rs b/crates/email/src/message/delete.rs index e1fcf166..34dc4bd4 100644 --- a/crates/email/src/message/delete.rs +++ b/crates/email/src/message/delete.rs @@ -7,19 +7,14 @@ use std::time::Duration; use common::{KV_LOCK_PURGE_ACCOUNT, Server, storage::index::ObjectIndexBuilder}; -use jmap_proto::types::{ - collection::Collection, id::Id, property::Property, state::StateChange, type_state::DataType, -}; +use jmap_proto::types::{collection::Collection, property::Property}; use store::{ - BitmapKey, IterateParams, U32_LEN, ValueKey, + BitmapKey, ValueKey, roaring::RoaringBitmap, - write::{ - AlignedBytes, Archive, BatchBuilder, BitmapClass, MaybeDynamicId, TagValue, ValueClass, - log::{ChangeLogBuilder, Changes}, - }, + write::{AlignedBytes, Archive, BatchBuilder, BitmapClass, TagValue, ValueClass}, }; use trc::AddContext; -use utils::{BlobHash, codec::leb128::Leb128Reader}; +use utils::BlobHash; use std::future::Future; use store::rand::prelude::SliceRandom; @@ -32,8 +27,9 @@ pub trait EmailDeletion: Sync + Send { fn emails_tombstone( &self, account_id: u32, + batch: &mut BatchBuilder, document_ids: RoaringBitmap, - ) -> impl Future> + Send; + ) -> impl Future> + Send; fn purge_accounts(&self) -> impl Future + Send; @@ -49,84 +45,46 @@ pub trait EmailDeletion: Sync + Send { &self, account_id: u32, ) -> impl Future> + Send; - - fn emails_purge_threads(&self, account_id: u32) - -> impl Future> + Send; } impl EmailDeletion for Server { async fn emails_tombstone( &self, account_id: u32, + batch: &mut BatchBuilder, document_ids: RoaringBitmap, - ) -> trc::Result<(ChangeLogBuilder, RoaringBitmap)> { - // Create batch - let mut changes = ChangeLogBuilder::with_change_id(0); - + ) -> trc::Result { // Tombstone message and untag it from the mailboxes - let mut batch = BatchBuilder::new(); + let mut deleted_ids = RoaringBitmap::new(); batch .with_account_id(account_id) .with_collection(Collection::Email); - let mut batches = Vec::new(); - let mut deleted_ids = RoaringBitmap::new(); self.get_archives( account_id, Collection::Email, &document_ids, - Property::Value, |document_id, data_| { - let data = data_ - .to_unarchived::() - .caused_by(trc::location!())?; - let thread_id = u32::from(data.inner.thread_id); - - // Log mailbox changes - for mailbox in data.inner.mailboxes.iter() { - changes.log_child_update(Collection::Mailbox, u32::from(mailbox.mailbox_id)); - } - - // Log message deletion - changes.log_delete(Collection::Email, Id::from_parts(thread_id, document_id)); - - // Log thread changes - changes.log_child_update(Collection::Thread, thread_id); - // Add changes to batch batch .update_document(document_id) - .custom(ObjectIndexBuilder::<_, ()>::new().with_current(data)) + .custom( + ObjectIndexBuilder::<_, ()>::new().with_current( + data_ + .to_unarchived::() + .caused_by(trc::location!())?, + ), + ) .caused_by(trc::location!())? - .tag( - Property::MailboxIds, - TagValue::Id(MaybeDynamicId::Static(TOMBSTONE_ID)), - ); + .tag(Property::MailboxIds, TagValue::Id(TOMBSTONE_ID)) + .commit_point(); deleted_ids.insert(document_id); - if batch.ops.len() >= 1000 { - batches.push(std::mem::replace(&mut batch, BatchBuilder::new())); - batch - .with_account_id(account_id) - .with_collection(Collection::Email); - } - Ok(true) }, ) .await?; - for batch in batches.into_iter().chain([batch]) { - if !batch.is_empty() { - self.core - .storage - .data - .write(batch.build()) - .await - .caused_by(trc::location!())?; - } - } - let not_destroyed = if document_ids.len() == deleted_ids.len() { RoaringBitmap::new() } else { @@ -134,7 +92,7 @@ impl EmailDeletion for Server { deleted_ids }; - Ok((changes, not_destroyed)) + Ok(not_destroyed) } async fn purge_accounts(&self) { @@ -248,7 +206,6 @@ impl EmailDeletion for Server { account_id, Collection::Email, &deletion_candidates, - Property::Value, |document_id, data| { if data.unarchive::()?.change_id < reference_cid { destroy_ids.insert(document_id); @@ -270,19 +227,10 @@ impl EmailDeletion for Server { ); // Tombstone messages - let (changes, _) = self.emails_tombstone(account_id, destroy_ids).await?; - - // Write and broadcast changes - if !changes.is_empty() { - let change_id = self.commit_changes(account_id, changes).await?; - self.broadcast_state_change( - StateChange::new(account_id) - .with_change(DataType::Email, change_id) - .with_change(DataType::Mailbox, change_id) - .with_change(DataType::Thread, change_id), - ) - .await; - } + let mut batch = BatchBuilder::new(); + self.emails_tombstone(account_id, &mut batch, destroy_ids) + .await?; + self.commit_batch(batch).await?; Ok(()) } @@ -315,9 +263,6 @@ impl EmailDeletion for Server { Total = tombstoned_ids.len(), ); - // Delete threadIds - self.emails_purge_threads(account_id).await?; - // Delete full-text index self.core .storage @@ -334,17 +279,15 @@ impl EmailDeletion for Server { .map(|t| t.id); // Delete messages + let mut batch = BatchBuilder::new(); + batch.with_account_id(account_id); + for document_id in tombstoned_ids { - let mut batch = BatchBuilder::new(); batch - .with_account_id(account_id) .with_collection(Collection::Email) .delete_document(document_id) .clear(Property::Value) - .untag( - Property::MailboxIds, - TagValue::Id(MaybeDynamicId::Static(TOMBSTONE_ID)), - ); + .untag(Property::MailboxIds, TagValue::Id(TOMBSTONE_ID)); // Remove message metadata if let Some(metadata_) = self @@ -383,8 +326,8 @@ impl EmailDeletion for Server { .index(&mut batch, account_id, tenant_id, false) .caused_by(trc::location!())?; - // Commit batch - self.core.storage.data.write(batch.build()).await?; + // Commit point + batch.commit_point(); } else { trc::event!( Purge(trc::PurgeEvent::Error), @@ -396,79 +339,7 @@ impl EmailDeletion for Server { } } - Ok(()) - } - - async fn emails_purge_threads(&self, account_id: u32) -> trc::Result<()> { - // Delete threadIs without documents - let mut thread_ids = self - .get_document_ids(account_id, Collection::Thread) - .await - .caused_by(trc::location!())? - .unwrap_or_default(); - - if thread_ids.is_empty() { - return Ok(()); - } - - self.core - .storage - .data - .iterate( - IterateParams::new( - BitmapKey { - account_id, - collection: Collection::Email.into(), - class: BitmapClass::Tag { - field: Property::ThreadId.into(), - value: TagValue::Id(0), - }, - document_id: 0, - }, - BitmapKey { - account_id, - collection: Collection::Email.into(), - class: BitmapClass::Tag { - field: Property::ThreadId.into(), - value: TagValue::Id(u32::MAX), - }, - document_id: u32::MAX, - }, - ) - .no_values(), - |key, _| { - let (thread_id, _) = key - .get(U32_LEN + 2..) - .and_then(|bytes| bytes.read_leb128::()) - .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?; - thread_ids.remove(thread_id); - - Ok(!thread_ids.is_empty()) - }, - ) - .await - .caused_by(trc::location!())?; - - if thread_ids.is_empty() { - return Ok(()); - } - - // Create batch - let mut batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_collection(Collection::Thread) - .with_change_id(self.generate_snowflake_id().caused_by(trc::location!())?) - .log(Changes::delete(thread_ids.iter().map(|id| id as u64))); - for thread_id in thread_ids { - batch.delete_document(thread_id); - } - self.core - .storage - .data - .write(batch.build()) - .await - .caused_by(trc::location!())?; + self.commit_batch(batch).await?; Ok(()) } diff --git a/crates/email/src/message/delivery.rs b/crates/email/src/message/delivery.rs index e91c0cd0..d7045e53 100644 --- a/crates/email/src/message/delivery.rs +++ b/crates/email/src/message/delivery.rs @@ -55,51 +55,6 @@ pub trait MailDelivery: Sync + Send { ) -> impl Future + Send; } -/* - -let semaphore = Arc::new(Semaphore::new( - inner - .shared_core - .load() - .smtp - .queue - .throttle - .local_concurrency, - )); - - loop { - let permit = match semaphore.clone().acquire_owned().await { - Ok(permit) => permit, - Err(_) => { - trc::error!(trc::StoreEvent::UnexpectedError - .into_err() - .details("Semaphore error") - .caused_by(trc::location!())); - break; - } - }; - - match delivery_rx.recv().await { - Some(event) => match event { - DeliveryEvent::Ingest { message, result_tx } => { - let server = inner.build_server(); - - tokio::spawn(async move { - result_tx.send(server.deliver_message(message).await).ok(); - - drop(permit); - }); - } - DeliveryEvent::Stop => break, - }, - None => { - break; - } - } - } - -*/ - impl MailDelivery for Server { async fn deliver_message(&self, message: IngestMessage) -> LocalDeliveryResult { // Obtain permit diff --git a/crates/email/src/message/index.rs b/crates/email/src/message/index.rs index eed9b8fd..68320e02 100644 --- a/crates/email/src/message/index.rs +++ b/crates/email/src/message/index.rs @@ -5,7 +5,7 @@ */ use common::storage::index::{IndexValue, IndexableObject, ObjectIndexBuilder}; -use jmap_proto::types::property::Property; +use jmap_proto::types::{collection::Collection, property::Property}; use mail_parser::{ decoders::html::html_to_text, parsers::{fields::thread::thread_name, preview::preview_text}, @@ -16,7 +16,7 @@ use store::{ Serialize, SerializeInfallible, backend::MAX_TOKEN_LENGTH, fts::{Field, index::FtsDocument}, - write::{Archiver, BatchBuilder, BlobOp, DirectoryClass, MaybeDynamicId, TagValue}, + write::{Archiver, BatchBuilder, BlobOp, DirectoryClass, TagValue}, }; use trc::AddContext; use utils::BlobHash; @@ -582,7 +582,7 @@ impl IndexableObject for MessageData { value: self .mailboxes .iter() - .map(|m| TagValue::Id(MaybeDynamicId::Static(m.mailbox_id))) + .map(|m| TagValue::Id(m.mailbox_id)) .collect(), }, IndexValue::Tag { @@ -591,14 +591,25 @@ impl IndexableObject for MessageData { .keywords .iter() .map(|k| match k.id() { - Ok(id) => TagValue::Id(MaybeDynamicId::Static(id)), + Ok(id) => TagValue::Id(id), Err(string) => TagValue::Text(string.into_bytes()), }) .collect(), }, IndexValue::Tag { field: Property::ThreadId.into(), - value: vec![TagValue::Id(MaybeDynamicId::Static(self.thread_id))], + value: vec![TagValue::Id(self.thread_id)], + }, + IndexValue::LogChild { + prefix: self.thread_id.into(), + }, + IndexValue::LogParent { + collection: Collection::Thread, + ids: vec![self.thread_id], + }, + IndexValue::LogParent { + collection: Collection::Mailbox, + ids: self.mailboxes.iter().map(|m| m.mailbox_id).collect(), }, ] .into_iter() @@ -613,7 +624,7 @@ impl IndexableObject for &ArchivedMessageData { value: self .mailboxes .iter() - .map(|m| TagValue::Id(MaybeDynamicId::Static(u32::from(m.mailbox_id)))) + .map(|m| TagValue::Id(u32::from(m.mailbox_id))) .collect(), }, IndexValue::Tag { @@ -622,16 +633,29 @@ impl IndexableObject for &ArchivedMessageData { .keywords .iter() .map(|k| match k.id() { - Ok(id) => TagValue::Id(MaybeDynamicId::Static(id)), + Ok(id) => TagValue::Id(id), Err(string) => TagValue::Text(string.into_bytes()), }) .collect(), }, IndexValue::Tag { field: Property::ThreadId.into(), - value: vec![TagValue::Id(MaybeDynamicId::Static(u32::from( - self.thread_id, - )))], + value: vec![TagValue::Id(u32::from(self.thread_id))], + }, + IndexValue::LogChild { + prefix: self.thread_id.to_native().into(), + }, + IndexValue::LogParent { + collection: Collection::Thread, + ids: vec![self.thread_id.to_native()], + }, + IndexValue::LogParent { + collection: Collection::Mailbox, + ids: self + .mailboxes + .iter() + .map(|m| m.mailbox_id.to_native()) + .collect(), }, ] .into_iter() diff --git a/crates/email/src/message/ingest.rs b/crates/email/src/message/ingest.rs index 80790982..42e7f0c7 100644 --- a/crates/email/src/message/ingest.rs +++ b/crates/email/src/message/ingest.rs @@ -37,13 +37,7 @@ use store::{ BlobClass, IndexKey, IndexKeyPrefix, IterateParams, U32_LEN, ahash::AHashMap, roaring::RoaringBitmap, - write::{ - AlignedBytes, Archive, AssignedIds, BatchBuilder, MaybeDynamicValue, SerializeWithId, - TaskQueueClass, ValueClass, - key::DeserializeBigEndian, - log::{ChangeLogBuilder, Changes, LogInsert}, - now, - }, + write::{BatchBuilder, TaskQueueClass, ValueClass, key::DeserializeBigEndian, now}, }; use store::{SerializeInfallible, rand::Rng}; use trc::{AddContext, MessageIngestEvent}; @@ -106,14 +100,19 @@ pub trait EmailIngest: Sync + Send { thread_name: &str, references: Vec<&[u8]>, skip_duplicate: Option<(&[u8], u32)>, - ) -> impl Future> + Send; + ) -> impl Future> + Send; fn assign_imap_uid( &self, account_id: u32, mailbox_id: u32, ) -> impl Future> + Send; fn email_bayes_can_train(&self, access_token: &AccessToken) -> bool; - fn create_thread_id(&self, account_id: u32) -> impl Future> + Send; +} + +pub enum ThreadResult { + Id(u32), + Create, + Skip, } impl EmailIngest for Server { @@ -160,21 +159,44 @@ impl EmailIngest for Server { && params.mailbox_ids == [INBOX_ID] { // Set the spam filter result - is_spam = self - .core - .spam - .headers - .status - .as_ref() - .and_then(|name| { - message - .root_part() - .headers - .iter() - .find(|h| h.name.as_str().eq_ignore_ascii_case(name.as_str())) - .and_then(|v| v.value.as_text()) - }) - .is_some_and(|v| v.contains("Yes")); + #[cfg(not(feature = "test_mode"))] + { + is_spam = self + .core + .spam + .headers + .status + .as_ref() + .and_then(|name| { + message + .root_part() + .headers + .iter() + .find(|h| h.name.as_str().eq_ignore_ascii_case(name.as_str())) + .and_then(|v| v.value.as_text()) + }) + .is_some_and(|v| v.contains("Yes")); + } + + #[cfg(feature = "test_mode")] + { + is_spam = self + .core + .spam + .headers + .status + .as_ref() + .and_then(|name| { + message + .root_part() + .headers + .iter() + .rev() + .find(|h| h.name.as_str().eq_ignore_ascii_case(name.as_str())) + .and_then(|v| v.value.as_text()) + }) + .is_some_and(|v| v.contains("Yes")); + } // Classify the message with user's model if let Some(bayes_config) = self @@ -258,6 +280,7 @@ impl EmailIngest for Server { // Obtain message references and thread name let mut message_id = None; + let mut log_thread_create = false; let thread_id = { let mut references = Vec::with_capacity(5); let mut subject = ""; @@ -265,9 +288,8 @@ impl EmailIngest for Server { match &header.name { HeaderName::MessageId => header.value.visit_text(|id| { if !id.is_empty() && id.len() < MAX_ID_LENGTH { - // Used by find_or_merge_thread to skip duplicates - if params.source.is_smtp() && message_id.is_none() { - message_id = references.len().into(); + if message_id.is_none() { + message_id = id.to_string().into(); } references.push(id.as_bytes()); } @@ -295,33 +317,48 @@ impl EmailIngest for Server { } } - let skip_duplicate = message_id.map(|idx| { - ( - references[idx], - params.mailbox_ids.first().copied().unwrap_or(INBOX_ID), - ) - }); - let thread_id = self - .find_or_merge_thread(account_id, subject, references, skip_duplicate) - .await?; - if thread_id != u32::MAX { - thread_id + let skip_duplicate = if params.source.is_smtp() { + message_id.as_deref().map(|message_id| { + ( + message_id.as_bytes(), + params.mailbox_ids.first().copied().unwrap_or(INBOX_ID), + ) + }) } else { - // Duplicate message - trc::event!( - MessageIngest(MessageIngestEvent::Duplicate), - SpanId = params.session_id, - AccountId = account_id, - MessageId = message_id, - ); + None + }; + match self + .find_or_merge_thread(account_id, subject, references, skip_duplicate) + .await? + { + ThreadResult::Id(thread_id) => thread_id, + ThreadResult::Create => { + log_thread_create = true; + self.get_cached_thread_ids(account_id) + .await + .caused_by(trc::location!())? + .assign_thread_id( + subject.as_bytes(), + message_id.as_deref().unwrap_or_default().as_bytes(), + ) + } + ThreadResult::Skip => { + // Duplicate message + trc::event!( + MessageIngest(MessageIngestEvent::Duplicate), + SpanId = params.session_id, + AccountId = account_id, + MessageId = message_id, + ); - return Ok(IngestedEmail { - id: Id::default(), - change_id: u64::MAX, - blob_id: BlobId::default(), - imap_uids: Vec::new(), - size: 0, - }); + return Ok(IngestedEmail { + id: Id::default(), + change_id: u64::MAX, + blob_id: BlobId::default(), + imap_uids: Vec::new(), + size: 0, + }); + } } }; @@ -384,12 +421,7 @@ impl EmailIngest for Server { }; if do_encrypt && !message.is_encrypted() { if let Some(encrypt_params_) = self - .get_property::>( - account_id, - Collection::Principal, - 0, - Property::Parameters, - ) + .get_archive_by_property(account_id, Collection::Principal, 0, Property::Parameters) .await .caused_by(trc::location!())? { @@ -440,11 +472,6 @@ impl EmailIngest for Server { } } - // Obtain a documentId and changeId - let change_id = self - .assign_change_id(account_id) - .caused_by(trc::location!())?; - // Store blob let blob_id = self .put_blob(account_id, raw_message.as_ref(), false) @@ -463,24 +490,30 @@ impl EmailIngest for Server { imap_uids.push(uid); } - // Prepare batch - let mut batch = BatchBuilder::new(); - batch - .with_change_id(change_id) - .with_account_id(account_id) - .with_collection(Collection::Thread) - .log(Changes::update([thread_id])); // Build write batch + let mut batch = BatchBuilder::new(); + let change_id = batch.change_id(); let mailbox_ids_event = mailbox_ids .iter() .map(|m| trc::Value::from(m.mailbox_id)) .collect::>(); + batch.with_account_id(account_id); + + if log_thread_create { + batch + .with_collection(Collection::Thread) + .update_document(thread_id) + .log_insert(None); + } + + let document_id = self + .store() + .assign_document_ids(account_id, Collection::Email, 1) + .await + .caused_by(trc::location!())?; batch - .with_collection(Collection::Mailbox) - .log(Changes::child_update(params.mailbox_ids.iter().copied())) .with_collection(Collection::Email) - .create_document() - .log(LogEmailInsert(thread_id.into())) + .create_document(document_id) .index_message( account_id, tenant_id, @@ -497,7 +530,7 @@ impl EmailIngest for Server { .caused_by(trc::location!())? .set( ValueClass::TaskQueue(TaskQueueClass::IndexEmail { - seq: self.generate_snowflake_id().caused_by(trc::location!())?, + seq: change_id, hash: blob_id.hash.clone(), }), vec![], @@ -507,7 +540,7 @@ impl EmailIngest for Server { if let Some(learn_spam) = train_spam { batch.set( ValueClass::TaskQueue(TaskQueueClass::BayesTrain { - seq: self.generate_snowflake_id()?, + seq: change_id, hash: blob_id.hash.clone(), learn_spam, }), @@ -516,15 +549,10 @@ impl EmailIngest for Server { } // Insert and obtain ids - let ids = self - .core - .storage - .data - .write(batch.build()) + self.store() + .write(batch.build_all()) .await .caused_by(trc::location!())?; - - let document_id = ids.last_document_id().caused_by(trc::location!())?; let id = Id::from_parts(thread_id, document_id); // Request FTS index @@ -575,9 +603,9 @@ impl EmailIngest for Server { thread_name: &str, mut references: Vec<&[u8]>, skip_duplicate: Option<(&[u8], u32)>, - ) -> trc::Result { + ) -> trc::Result { if references.is_empty() { - return self.create_thread_id(account_id).await; + return Ok(ThreadResult::Create); } let mut try_count = 0; @@ -632,7 +660,7 @@ impl EmailIngest for Server { // No matching subjects were found, skip early if subj_results.is_empty() { - return self.create_thread_id(account_id).await; + return Ok(ThreadResult::Create); } // Find messages with matching references @@ -691,7 +719,7 @@ impl EmailIngest for Server { // No matching messages if results.is_empty() { - return self.create_thread_id(account_id).await; + return Ok(ThreadResult::Create); } // Skip duplicate messages @@ -707,7 +735,7 @@ impl EmailIngest for Server { .caused_by(trc::location!())? { if found_message_id.iter().any(|id| ids.contains(*id)) { - return Ok(u32::MAX); + return Ok(ThreadResult::Skip); } } } @@ -732,24 +760,19 @@ impl EmailIngest for Server { } if thread_id == u32::MAX { - return self.create_thread_id(account_id).await; + return Ok(ThreadResult::Create); } else if thread_counts.len() == 1 { - return Ok(thread_id); + return Ok(ThreadResult::Id(thread_id)); } // Delete all but the most common threadId let mut batch = BatchBuilder::new(); - let change_id = self - .assign_change_id(account_id) - .caused_by(trc::location!())?; - let mut changes = ChangeLogBuilder::with_change_id(change_id); batch .with_account_id(account_id) .with_collection(Collection::Thread); for &delete_thread_id in thread_counts.keys() { if delete_thread_id != thread_id { - batch.delete_document(delete_thread_id); - changes.log_delete(Collection::Thread, delete_thread_id); + batch.update_document(delete_thread_id).log_delete(None); } } @@ -761,12 +784,7 @@ impl EmailIngest for Server { continue; } if let Some(data_) = self - .get_property::>( - account_id, - Collection::Email, - document_id, - Property::Value, - ) + .get_archive(account_id, Collection::Email, document_id) .await .caused_by(trc::location!())? { @@ -786,18 +804,11 @@ impl EmailIngest for Server { .with_changes(new_data), ) .caused_by(trc::location!())?; - changes.log_move( - Collection::Email, - Id::from_parts(old_thread_id, document_id), - Id::from_parts(thread_id, document_id), - ); } } - batch.custom(changes).caused_by(trc::location!())?; - - match self.core.storage.data.write(batch.build()).await { - Ok(_) => return Ok(thread_id), + match self.commit_batch(batch).await { + Ok(_) => return Ok(ThreadResult::Id(thread_id)), Err(err) if err.is_assertion_failure() && try_count < MAX_RETRIES => { let backoff = store::rand::rng().random_range(50..=300); tokio::time::sleep(Duration::from_millis(backoff)).await; @@ -821,7 +832,7 @@ impl EmailIngest for Server { self.core .storage .data - .write(batch.build()) + .write(batch.build_all()) .await .and_then(|v| v.last_counter_id().map(|id| id as u32)) } @@ -831,28 +842,6 @@ impl EmailIngest for Server { bayes.account_classify && access_token.has_permission(Permission::SpamFilterTrain) }) } - - async fn create_thread_id(&self, account_id: u32) -> trc::Result { - let mut batch = BatchBuilder::new(); - batch - .with_change_id(self.generate_snowflake_id().caused_by(trc::location!())?) - .with_account_id(account_id) - .with_collection(Collection::Thread) - .create_document() - .log(LogInsert()); - self.store() - .write_expect_id(batch) - .await - .caused_by(trc::location!()) - } -} - -pub struct LogEmailInsert(Option); - -impl LogEmailInsert { - pub fn new(thread_id: Option) -> Self { - Self(thread_id) - } } impl IngestSource<'_> { @@ -861,24 +850,6 @@ impl IngestSource<'_> { } } -impl SerializeWithId for LogEmailInsert { - fn serialize_with_id(&self, ids: &AssignedIds) -> trc::Result> { - let thread_id = match self.0 { - Some(thread_id) => thread_id, - None => ids.first_document_id()?, - }; - let document_id = ids.last_document_id()?; - - Ok(Changes::insert([Id::from_parts(thread_id, document_id)]).serialize()) - } -} - -impl From for MaybeDynamicValue { - fn from(log: LogEmailInsert) -> Self { - MaybeDynamicValue::Dynamic(Box::new(log)) - } -} - impl From for Object { fn from(email: IngestedEmail) -> Self { Object::with_capacity(3) @@ -888,19 +859,3 @@ impl From for Object { .with_property(Property::Size, email.size) } } - -/* - - let thread_id = match thread_id { - Some(thread_id) => thread_id, - None => ids.first_document_id().caused_by(trc::location!())?, - }; - - .with_collection(Collection::Thread); - if let Some(thread_id) = thread_id { - batch.log(Changes::update([thread_id])); - } else { - batch.create_document().log(LogInsert()); - - -*/ diff --git a/crates/email/src/sieve/activate.rs b/crates/email/src/sieve/activate.rs index 19d00184..34c0d623 100644 --- a/crates/email/src/sieve/activate.rs +++ b/crates/email/src/sieve/activate.rs @@ -6,10 +6,7 @@ use common::{Server, storage::index::ObjectIndexBuilder}; use jmap_proto::types::{collection::Collection, property::Property}; -use store::{ - query::Filter, - write::{AlignedBytes, Archive, BatchBuilder}, -}; +use store::{query::Filter, write::BatchBuilder}; use trc::AddContext; use super::SieveScript; @@ -19,7 +16,7 @@ pub trait SieveScriptActivate: Sync + Send { &self, account_id: u32, activate_id: Option, - ) -> impl Future>> + Send; + ) -> impl Future)>> + Send; } impl SieveScriptActivate for Server { @@ -27,7 +24,7 @@ impl SieveScriptActivate for Server { &self, account_id: u32, mut activate_id: Option, - ) -> trc::Result> { + ) -> trc::Result<(u64, Vec<(u32, bool)>)> { let mut changed_ids = Vec::new(); // Find the currently active script let mut active_ids = self @@ -43,7 +40,7 @@ impl SieveScriptActivate for Server { // Check if script is already active if activate_id.is_some_and(|id| active_ids.remove(id)) { if active_ids.is_empty() { - return Ok(changed_ids); + return Ok((0, changed_ids)); } else { activate_id = None; } @@ -57,19 +54,14 @@ impl SieveScriptActivate for Server { // Deactivate scripts for document_id in active_ids { - if let Some(sieve) = self - .get_property::>( - account_id, - Collection::SieveScript, - document_id, - Property::Value, - ) + if let Some(sieve_) = self + .get_archive(account_id, Collection::SieveScript, document_id) .await? { - let sieve = sieve - .into_deserialized::() + let sieve = sieve_ + .to_unarchived::() .caused_by(trc::location!())?; - let mut new_sieve = sieve.inner.clone(); + let mut new_sieve = sieve.deserialize().caused_by(trc::location!())?; new_sieve.is_active = false; batch .update_document(document_id) @@ -79,26 +71,22 @@ impl SieveScriptActivate for Server { .with_changes(new_sieve) .with_current(sieve), ) - .caused_by(trc::location!())?; + .caused_by(trc::location!())? + .commit_point(); changed_ids.push((document_id, false)); } } // Activate script if let Some(document_id) = activate_id { - if let Some(sieve) = self - .get_property::>( - account_id, - Collection::SieveScript, - document_id, - Property::Value, - ) + if let Some(sieve_) = self + .get_archive(account_id, Collection::SieveScript, document_id) .await? { - let sieve = sieve - .into_deserialized::() + let sieve = sieve_ + .to_unarchived::() .caused_by(trc::location!())?; - let mut new_sieve = sieve.inner.clone(); + let mut new_sieve = sieve.deserialize().caused_by(trc::location!())?; new_sieve.is_active = true; batch .update_document(document_id) @@ -114,17 +102,14 @@ impl SieveScriptActivate for Server { // Write changes if !changed_ids.is_empty() { - match self.core.storage.data.write(batch.build()).await { - Ok(_) => (), - Err(err) if err.is_assertion_failure() => { - return Ok(vec![]); - } - Err(err) => { - return Err(err.caused_by(trc::location!())); - } + let change_id = batch.change_id(); + match self.commit_batch(batch).await { + Ok(_) => Ok((change_id, changed_ids)), + Err(err) if err.is_assertion_failure() => Ok((0, vec![])), + Err(err) => Err(err.caused_by(trc::location!())), } + } else { + Ok((0, changed_ids)) } - - Ok(changed_ids) } } diff --git a/crates/email/src/sieve/delete.rs b/crates/email/src/sieve/delete.rs index b0817d39..16c32054 100644 --- a/crates/email/src/sieve/delete.rs +++ b/crates/email/src/sieve/delete.rs @@ -6,7 +6,7 @@ use common::{Server, auth::ResourceToken, storage::index::ObjectIndexBuilder}; use jmap_proto::types::{collection::Collection, property::Property}; -use store::write::{AlignedBytes, Archive, BatchBuilder}; +use store::write::BatchBuilder; use trc::AddContext; use super::SieveScript; @@ -17,7 +17,8 @@ pub trait SieveScriptDelete: Sync + Send { resource_token: &ResourceToken, document_id: u32, fail_if_active: bool, - ) -> impl Future> + Send; + batch: &mut BatchBuilder, + ) -> impl Future>> + Send; } impl SieveScriptDelete for Server { @@ -26,34 +27,29 @@ impl SieveScriptDelete for Server { resource_token: &ResourceToken, document_id: u32, fail_if_active: bool, - ) -> trc::Result { + batch: &mut BatchBuilder, + ) -> trc::Result> { // Fetch record let account_id = resource_token.account_id; - let obj_ = self - .get_property::>( - account_id, - Collection::SieveScript, - document_id, - Property::Value, - ) + let obj_ = if let Some(obj) = self + .get_archive(account_id, Collection::SieveScript, document_id) .await? - .ok_or_else(|| { - trc::StoreEvent::NotFound - .into_err() - .caused_by(trc::location!()) - .document_id(document_id) - })?; + { + obj + } else { + return Ok(None); + }; + let obj = obj_ .to_unarchived::() .caused_by(trc::location!())?; // Make sure the script is not active if fail_if_active && obj.inner.is_active { - return Ok(false); + return Ok(Some(false)); } // Delete record - let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::SieveScript) @@ -64,12 +60,9 @@ impl SieveScriptDelete for Server { .with_current(obj) .with_tenant_id(resource_token), ) - .caused_by(trc::location!())?; + .caused_by(trc::location!())? + .commit_point(); - self.store() - .write(batch) - .await - .caused_by(trc::location!())?; - Ok(true) + Ok(Some(true)) } } diff --git a/crates/email/src/sieve/index.rs b/crates/email/src/sieve/index.rs index 1a417162..b70fc765 100644 --- a/crates/email/src/sieve/index.rs +++ b/crates/email/src/sieve/index.rs @@ -25,6 +25,7 @@ impl IndexableObject for SieveScript { IndexValue::Blob { value: self.blob_hash.clone(), }, + IndexValue::LogChild { prefix: None }, IndexValue::Quota { used: self.size }, ] .into_iter() @@ -49,6 +50,7 @@ impl IndexableObject for &ArchivedSieveScript { IndexValue::Blob { value: (&self.blob_hash).into(), }, + IndexValue::LogChild { prefix: None }, IndexValue::Quota { used: u32::from(self.size), }, diff --git a/crates/email/src/sieve/ingest.rs b/crates/email/src/sieve/ingest.rs index 21ace559..6cc7e224 100644 --- a/crates/email/src/sieve/ingest.rs +++ b/crates/email/src/sieve/ingest.rs @@ -25,7 +25,7 @@ use store::{ ahash::AHashMap, dispatch::lookup::KeyValue, query::Filter, - write::{AlignedBytes, Archive, Archiver, BatchBuilder, BlobOp, LegacyBincode}, + write::{Archiver, BatchBuilder, BlobOp, LegacyBincode}, }; use trc::{AddContext, SieveEvent}; use utils::config::utils::ParseValue; @@ -343,13 +343,10 @@ impl SieveScriptIngest for Server { { target_id = document_id; } - } else if let Ok(Some((document_id, changes))) = + } else if let Ok(Some(document_id)) = self.mailbox_create_path(account_id, &folder).await { target_id = document_id; - if let Some(change_id) = changes { - ingested_message.change_id = change_id; - } } } @@ -613,12 +610,7 @@ impl SieveScriptIngest for Server { ) -> trc::Result { // Obtain script object let script_object = self - .get_property::>( - account_id, - Collection::SieveScript, - document_id, - Property::Value, - ) + .get_archive(account_id, Collection::SieveScript, document_id) .await? .ok_or_else(|| { trc::StoreEvent::NotFound @@ -708,7 +700,7 @@ impl SieveScriptIngest for Server { Vec::new(), ); self.store() - .write(batch.build()) + .write(batch.build_all()) .await .caused_by(trc::location!())?; diff --git a/crates/email/src/submission/index.rs b/crates/email/src/submission/index.rs index 42b8d667..dd5c65b6 100644 --- a/crates/email/src/submission/index.rs +++ b/crates/email/src/submission/index.rs @@ -32,6 +32,7 @@ impl IndexableObject for EmailSubmission { field: Property::SendAt.into(), value: self.send_at.into(), }, + IndexValue::LogChild { prefix: None }, ] .into_iter() } @@ -60,6 +61,7 @@ impl IndexableObject for &ArchivedEmailSubmission { field: Property::SendAt.into(), value: self.send_at.into(), }, + IndexValue::LogChild { prefix: None }, ] .into_iter() } diff --git a/crates/http/src/form/mod.rs b/crates/http/src/form/mod.rs index ba01dace..5584af07 100644 --- a/crates/http/src/form/mod.rs +++ b/crates/http/src/form/mod.rs @@ -190,7 +190,7 @@ impl FormHandler for Server { 0u32.serialize(), ); self.store() - .write(batch.build()) + .write(batch.build_all()) .await .caused_by(trc::location!())?; self.blob_store() diff --git a/crates/http/src/management/crypto.rs b/crates/http/src/management/crypto.rs index 41381507..71f19c1e 100644 --- a/crates/http/src/management/crypto.rs +++ b/crates/http/src/management/crypto.rs @@ -39,7 +39,7 @@ pub trait CryptoHandler: Sync + Send { impl CryptoHandler for Server { async fn handle_crypto_get(&self, access_token: Arc) -> trc::Result { let ec = if let Some(params_) = self - .get_property::>( + .get_archive_by_property( access_token.primary_id(), Collection::Principal, 0, @@ -97,7 +97,7 @@ impl CryptoHandler for Server { .with_collection(Collection::Principal) .update_document(0) .clear(Property::Parameters); - self.core.storage.data.write(batch.build()).await?; + self.core.storage.data.write(batch.build_all()).await?; return Ok(JsonResponse::new(json!({ "data": (), })) @@ -147,7 +147,7 @@ impl CryptoHandler for Server { .with_collection(Collection::Principal) .update_document(0) .set(Property::Parameters, params); - self.core.storage.data.write(batch.build()).await?; + self.core.storage.data.write(batch.build_all()).await?; Ok(JsonResponse::new(json!({ "data": num_certs, diff --git a/crates/http/src/management/enterprise/undelete.rs b/crates/http/src/management/enterprise/undelete.rs index e1b4b2f3..645282c8 100644 --- a/crates/http/src/management/enterprise/undelete.rs +++ b/crates/http/src/management/enterprise/undelete.rs @@ -260,7 +260,7 @@ impl UndeleteApi for Server { self.core .storage .data - .write(batch.build()) + .write(batch.build_all()) .await .caused_by(trc::location!())?; } diff --git a/crates/http/src/management/mod.rs b/crates/http/src/management/mod.rs index e54d851b..73997dcf 100644 --- a/crates/http/src/management/mod.rs +++ b/crates/http/src/management/mod.rs @@ -30,6 +30,7 @@ use dns::DnsManagement; use enterprise::telemetry::TelemetryApi; use hyper::{Method, StatusCode, header}; use jmap::api::{ToJmapHttpResponse, ToRequestError}; +use jmap_proto::error::request::RequestError; use log::LogManagement; use mail_parser::DateTime; use principal::PrincipalManager; @@ -276,6 +277,10 @@ impl ToManageHttpResponse for &trc::Error { HttpResponse::new(StatusCode::UNAUTHORIZED) .with_header(header::WWW_AUTHENTICATE, "Bearer realm=\"Stalwart Server\"") .with_header(header::WWW_AUTHENTICATE, "Basic realm=\"Stalwart Server\"") + .with_content_type("application/problem+json") + .with_text_body( + serde_json::to_string(&RequestError::unauthorized()).unwrap_or_default(), + ) } _ => self.to_request_error().into_http_response(), } diff --git a/crates/http/src/management/report.rs b/crates/http/src/management/report.rs index 4e7ef79a..7df9fdd8 100644 --- a/crates/http/src/management/report.rs +++ b/crates/http/src/management/report.rs @@ -213,9 +213,9 @@ impl ManageReports for Server { batch.clear(ValueClass::Report(report_id)); - if batch.ops.len() > 1000 { + if batch.len() > 1000 { if let Err(err) = - server.core.storage.data.write(batch.build()).await + server.core.storage.data.write(batch.build_all()).await { trc::error!(err.caused_by(trc::location!())); } @@ -223,8 +223,10 @@ impl ManageReports for Server { } } - if !batch.ops.is_empty() { - if let Err(err) = server.core.storage.data.write(batch.build()).await { + if !batch.is_empty() { + if let Err(err) = + server.core.storage.data.write(batch.build_all()).await + { trc::error!(err.caused_by(trc::location!())); } } @@ -280,7 +282,7 @@ impl ManageReports for Server { let mut batch = BatchBuilder::new(); batch.clear(ValueClass::Report(report_id)); - self.core.storage.data.write(batch.build()).await?; + self.core.storage.data.write(batch.build_all()).await?; Ok(JsonResponse::new(json!({ "data": true, diff --git a/crates/http/src/management/stores.rs b/crates/http/src/management/stores.rs index 510c7539..2cda8d11 100644 --- a/crates/http/src/management/stores.rs +++ b/crates/http/src/management/stores.rs @@ -23,7 +23,7 @@ use serde_json::json; use services::index::Indexer; use store::{ Serialize, rand, - write::{AlignedBytes, Archive, Archiver, BatchBuilder, ValueClass}, + write::{Archiver, BatchBuilder, ValueClass}, }; use trc::AddContext; use utils::url_params::UrlParams; @@ -332,12 +332,7 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u .unwrap_or_default() { let mailbox = server - .get_property::>( - account_id, - Collection::Mailbox, - mailbox_id, - Property::Value, - ) + .get_archive(account_id, Collection::Mailbox, mailbox_id) .await .caused_by(trc::location!())? .ok_or_else(|| trc::ImapEvent::Error.into_err().caused_by(trc::location!()))? @@ -359,7 +354,7 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u .clear(Property::EmailIds); server .store() - .write(batch) + .write(batch.build_all()) .await .caused_by(trc::location!())?; mailbox_count += 1; @@ -373,12 +368,7 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u .unwrap_or_default() { let data = server - .get_property::>( - account_id, - Collection::Email, - message_id, - Property::Value, - ) + .get_archive(account_id, Collection::Email, message_id) .await .caused_by(trc::location!())?; let data_ = if let Some(data) = data { @@ -415,7 +405,7 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u ); server .store() - .write(batch) + .write(batch.build_all()) .await .caused_by(trc::location!())?; email_count += 1; diff --git a/crates/imap/src/core/mailbox.rs b/crates/imap/src/core/mailbox.rs index dad9598d..8e809341 100644 --- a/crates/imap/src/core/mailbox.rs +++ b/crates/imap/src/core/mailbox.rs @@ -19,10 +19,7 @@ use indexmap::IndexMap; use jmap_proto::types::{acl::Acl, collection::Collection, id::Id, property::Property}; use parking_lot::Mutex; use std::sync::{Arc, atomic::Ordering}; -use store::{ - query::log::{Change, Query}, - write::{AlignedBytes, Archive}, -}; +use store::query::log::{Change, Query}; use trc::AddContext; use utils::topological::TopologicalSort; @@ -164,7 +161,6 @@ impl SessionData { account_id, Collection::Mailbox, &mailbox_ids, - Property::Value, |mailbox_id, mailbox_| { let mailbox = mailbox_ .unarchive::() @@ -645,12 +641,7 @@ impl SessionData { Ok(access_token.is_member(account_id) || self .server - .get_property::>( - account_id, - Collection::Mailbox, - document_id, - Property::Value, - ) + .get_archive(account_id, Collection::Mailbox, document_id) .await .and_then(|mailbox| { if let Some(mailbox) = mailbox { diff --git a/crates/imap/src/core/message.rs b/crates/imap/src/core/message.rs index dda88f1f..e6e66b08 100644 --- a/crates/imap/src/core/message.rs +++ b/crates/imap/src/core/message.rs @@ -11,18 +11,13 @@ use common::{NextMailboxState, listener::SessionStream}; use email::message::metadata::MessageData; use imap_proto::protocol::{Sequence, expunge, select::Exists}; use jmap_proto::types::{collection::Collection, property::Property}; -use store::{ - ValueKey, - write::{AlignedBytes, Archive, ValueClass}, -}; +use store::{ValueKey, write::ValueClass}; use trc::AddContext; use crate::core::ImapId; use super::{ImapUidToId, MailboxId, MailboxState, SelectedMailbox, SessionData}; -pub(crate) const MAX_RETRIES: usize = 10; - impl SessionData { pub async fn fetch_messages(&self, mailbox: &MailboxId) -> trc::Result { // Obtain message ids @@ -58,7 +53,6 @@ impl SessionData { mailbox.account_id, Collection::Email, &message_ids, - Property::Value, |message_id, message_data_| { let message_data = message_data_ .unarchive::() @@ -231,12 +225,7 @@ impl SessionData { pub async fn get_uid_validity(&self, mailbox: &MailboxId) -> trc::Result { self.server - .get_property::>( - mailbox.account_id, - Collection::Mailbox, - mailbox.mailbox_id, - &Property::Value, - ) + .get_archive(mailbox.account_id, Collection::Mailbox, mailbox.mailbox_id) .await? .ok_or_else(|| { trc::ImapEvent::Error diff --git a/crates/imap/src/op/acl.rs b/crates/imap/src/op/acl.rs index a32097e4..c46d59bd 100644 --- a/crates/imap/src/op/acl.rs +++ b/crates/imap/src/op/acl.rs @@ -22,11 +22,8 @@ use imap_proto::{ receiver::Request, }; -use jmap_proto::types::{ - acl::Acl, collection::Collection, property::Property, state::StateChange, type_state::DataType, - value::AclGrant, -}; -use store::write::{AlignedBytes, Archive, BatchBuilder, log::ChangeLogBuilder}; +use jmap_proto::types::{acl::Acl, collection::Collection, value::AclGrant}; +use store::write::{AlignedBytes, Archive, BatchBuilder}; use trc::AddContext; use utils::map::bitmap::Bitmap; @@ -332,25 +329,12 @@ impl Session { .with_current(current_mailbox), ) .imap_ctx(&arguments.tag, trc::location!())?; + if !batch.is_empty() { data.server - .store() - .write(batch) + .commit_batch(batch) .await .imap_ctx(&arguments.tag, trc::location!())?; - let mut changes = ChangeLogBuilder::new(); - changes.log_update(Collection::Mailbox, mailbox_id.mailbox_id); - let change_id = data - .server - .commit_changes(mailbox_id.account_id, changes) - .await - .imap_ctx(&arguments.tag, trc::location!())?; - data.server - .broadcast_state_change( - StateChange::new(mailbox_id.account_id) - .with_change(DataType::Mailbox, change_id), - ) - .await; } // Invalidate ACLs @@ -439,12 +423,7 @@ impl SessionData { if let Some(mailbox) = self.get_mailbox_by_name(&arguments.mailbox_name) { if let Some(values) = self .server - .get_property::>( - mailbox.account_id, - Collection::Mailbox, - mailbox.mailbox_id, - Property::Value, - ) + .get_archive(mailbox.account_id, Collection::Mailbox, mailbox.mailbox_id) .await .caused_by(trc::location!())? { diff --git a/crates/imap/src/op/copy_move.rs b/crates/imap/src/op/copy_move.rs index 1d19edae..313df1e9 100644 --- a/crates/imap/src/op/copy_move.rs +++ b/crates/imap/src/op/copy_move.rs @@ -25,14 +25,11 @@ use crate::{ use common::{MailboxId, listener::SessionStream, storage::index::ObjectIndexBuilder}; use jmap_proto::{ error::set::SetErrorType, - types::{ - acl::Acl, collection::Collection, id::Id, property::Property, state::StateChange, - type_state::DataType, - }, + types::{acl::Acl, collection::Collection, state::StateChange, type_state::DataType}, }; use store::{ roaring::RoaringBitmap, - write::{AlignedBytes, Archive, BatchBuilder, ValueClass, log::ChangeLogBuilder}, + write::{AlignedBytes, Archive, BatchBuilder, ValueClass}, }; use super::ImapContext; @@ -171,7 +168,6 @@ impl SessionData { } else { Command::Copy(is_uid) }); - let mut changelog = ChangeLogBuilder::new(); let mut did_move = false; let mut copied_ids = Vec::with_capacity(ids.len()); let access_token = self @@ -186,6 +182,7 @@ impl SessionData { let dest_mailbox_id = UidMailbox::new_unassigned(dest_mailbox_id); let can_spam_train = self.server.email_bayes_can_train(&access_token); let mut has_spam_train_tasks = false; + let mut batch = BatchBuilder::new(); for (id, imap_id) in ids { // Obtain mailbox tags @@ -223,14 +220,7 @@ impl SessionData { let mut new_data = data .deserialize() .imap_ctx(&arguments.tag, trc::location!())?; - if changelog.change_id == u64::MAX { - changelog.change_id = self - .server - .assign_change_id(account_id) - .imap_ctx(&arguments.tag, trc::location!())?; - } - new_data.change_id = changelog.change_id; - let thread_id = new_data.thread_id; + new_data.change_id = batch.change_id(); // Add destination folder new_data.add_mailbox(dest_mailbox_id); @@ -253,7 +243,6 @@ impl SessionData { } // Prepare write batch - let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Email) @@ -291,23 +280,20 @@ impl SessionData { has_spam_train_tasks = true; } } - - // Write changes - self.server - .store() - .write(batch) - .await - .imap_ctx(&arguments.tag, trc::location!())?; + batch.commit_point(); // Update changelog - changelog.log_update(Collection::Email, Id::from_parts(thread_id, id)); - changelog.log_child_update(Collection::Mailbox, dest_mailbox_id.mailbox_id); if is_move { - changelog.log_child_update(Collection::Mailbox, src_mailbox.id.mailbox_id); did_move = true; } } + // Write changes + self.server + .commit_batch(batch) + .await + .imap_ctx(&arguments.tag, trc::location!())?; + // Trigger Bayes training if has_spam_train_tasks { self.server.notify_task_queue(); @@ -360,14 +346,21 @@ impl SessionData { // Untag or delete emails if !destroy_ids.is_empty() { + let mut batch = BatchBuilder::new(); self.email_untag_or_delete( src_account_id, src_mailbox.id.mailbox_id, &destroy_ids, - &mut changelog, + &mut batch, ) .await .imap_ctx(&arguments.tag, trc::location!())?; + + self.server + .commit_batch(batch) + .await + .imap_ctx(&arguments.tag, trc::location!())?; + did_move = true; } @@ -384,22 +377,6 @@ impl SessionData { } } - // Write changes on source account - if !changelog.is_empty() { - let change_id = self - .server - .commit_changes(src_mailbox.id.account_id, changelog) - .await - .imap_ctx(&arguments.tag, trc::location!())?; - self.server - .broadcast_state_change( - StateChange::new(src_mailbox.id.account_id) - .with_change(DataType::Email, change_id) - .with_change(DataType::Mailbox, change_id), - ) - .await; - } - // Map copied JMAP Ids to IMAP UIDs in the destination folder. if copied_ids.is_empty() { return Err(if response.rtype != ResponseType::Ok { @@ -493,12 +470,7 @@ impl SessionData { ) -> trc::Result>> { if let Some(data) = self .server - .get_property::>( - account_id, - Collection::Email, - id, - Property::Value, - ) + .get_archive(account_id, Collection::Email, id) .await? { Ok(Some(data)) diff --git a/crates/imap/src/op/create.rs b/crates/imap/src/op/create.rs index 1852ba3e..e04077f9 100644 --- a/crates/imap/src/op/create.rs +++ b/crates/imap/src/op/create.rs @@ -21,10 +21,7 @@ use imap_proto::{ protocol::{create::Arguments, list::Attribute}, receiver::Request, }; -use jmap_proto::types::{ - acl::Acl, collection::Collection, id::Id, property::Property, state::StateChange, - type_state::DataType, -}; +use jmap_proto::types::{acl::Acl, collection::Collection, id::Id, property::Property}; use store::{query::Filter, write::BatchBuilder}; use trc::AddContext; @@ -73,13 +70,20 @@ impl SessionData { debug_assert!(!params.path.is_empty()); // Build batch - let mut changes = self - .server - .begin_changes(params.account_id) - .imap_ctx(&arguments.tag, trc::location!())?; - let mut parent_id = params.parent_mailbox_id.map(|id| id + 1).unwrap_or(0); let mut create_ids = Vec::with_capacity(params.path.len()); + let mut change_id = 0; + let mut next_document_id = self + .server + .store() + .assign_document_ids( + params.account_id, + Collection::Mailbox, + params.path.len() as u64, + ) + .await + .caused_by(trc::location!())?; + let mut batch = BatchBuilder::new(); for (pos, &path_item) in params.path.iter().enumerate() { let mut mailbox = email::mailbox::Mailbox::new(path_item).with_parent_id(parent_id); @@ -88,45 +92,25 @@ impl SessionData { mailbox.role = mailbox_role; } } - let mut batch = BatchBuilder::new(); + let mailbox_id = next_document_id; + next_document_id -= 1; + change_id = batch.change_id(); batch .with_account_id(params.account_id) .with_collection(Collection::Mailbox) - .create_document() + .create_document(mailbox_id) .custom(ObjectIndexBuilder::<(), _>::new().with_changes(mailbox)) - .imap_ctx(&arguments.tag, trc::location!())?; - let mailbox_id = self - .server - .store() - .write_expect_id(batch) - .await - .imap_ctx(&arguments.tag, trc::location!())?; - changes.log_insert(Collection::Mailbox, mailbox_id); + .imap_ctx(&arguments.tag, trc::location!())? + .commit_point(); parent_id = mailbox_id + 1; create_ids.push(mailbox_id); } - // Write changes - let change_id = changes.change_id; - let mut batch = BatchBuilder::new(); - batch - .with_account_id(params.account_id) - .with_collection(Collection::Mailbox) - .custom(changes) - .imap_ctx(&arguments.tag, trc::location!())?; self.server - .store() - .write(batch) + .commit_batch(batch) .await .imap_ctx(&arguments.tag, trc::location!())?; - // Broadcast changes - self.server - .broadcast_state_change( - StateChange::new(params.account_id).with_change(DataType::Mailbox, change_id), - ) - .await; - trc::event!( Imap(trc::ImapEvent::CreateMailbox), SpanId = self.session_id, diff --git a/crates/imap/src/op/delete.rs b/crates/imap/src/op/delete.rs index 19f445df..318a385c 100644 --- a/crates/imap/src/op/delete.rs +++ b/crates/imap/src/op/delete.rs @@ -16,8 +16,6 @@ use email::mailbox::destroy::MailboxDestroy; use imap_proto::{ Command, ResponseCode, StatusResponse, protocol::delete::Arguments, receiver::Request, }; -use jmap_proto::types::{state::StateChange, type_state::DataType}; -use store::write::log::ChangeLogBuilder; use super::ImapContext; @@ -75,41 +73,19 @@ impl SessionData { .get_access_token() .await .imap_ctx(&arguments.tag, trc::location!())?; - let mut changelog = ChangeLogBuilder::new(); - let did_remove_emails = match self + + if let Err(err) = self .server - .mailbox_destroy(account_id, mailbox_id, &mut changelog, &access_token, true) + .mailbox_destroy(account_id, mailbox_id, &access_token, true) .await .imap_ctx(&arguments.tag, trc::location!())? { - Ok(did_remove_emails) => did_remove_emails, - Err(err) => { - return Err(trc::ImapEvent::Error - .into_err() - .details(err.description.unwrap_or("Delete failed".into())) - .code(ResponseCode::from(err.type_)) - .id(arguments.tag)); - } - }; - - // Write changes - let change_id = self - .server - .commit_changes(account_id, changelog) - .await - .imap_ctx(&arguments.tag, trc::location!())?; - - // Broadcast changes - self.server - .broadcast_state_change(if did_remove_emails { - StateChange::new(account_id) - .with_change(DataType::Mailbox, change_id) - .with_change(DataType::Email, change_id) - .with_change(DataType::Thread, change_id) - } else { - StateChange::new(account_id).with_change(DataType::Mailbox, change_id) - }) - .await; + return Err(trc::ImapEvent::Error + .into_err() + .details(err.description.unwrap_or("Delete failed".into())) + .code(ResponseCode::from(err.type_)) + .id(arguments.tag)); + } // Update mailbox cache for account in self.mailboxes.lock().iter_mut() { diff --git a/crates/imap/src/op/expunge.rs b/crates/imap/src/op/expunge.rs index 562f0f7d..7a7fcb90 100644 --- a/crates/imap/src/op/expunge.rs +++ b/crates/imap/src/op/expunge.rs @@ -18,14 +18,8 @@ use trc::AddContext; use crate::core::{SavedSearch, SelectedMailbox, Session, SessionData}; use common::{ImapId, listener::SessionStream, storage::index::ObjectIndexBuilder}; -use jmap_proto::types::{ - acl::Acl, collection::Collection, id::Id, keyword::Keyword, property::Property, - state::StateChange, type_state::DataType, -}; -use store::{ - roaring::RoaringBitmap, - write::{BatchBuilder, log::ChangeLogBuilder}, -}; +use jmap_proto::types::{acl::Acl, collection::Collection, keyword::Keyword, property::Property}; +use store::{roaring::RoaringBitmap, write::BatchBuilder}; use super::{ImapContext, ToModSeq}; @@ -146,15 +140,10 @@ impl SessionData { } // Delete ids - let mut changelog = ChangeLogBuilder::new(); - self.email_untag_or_delete( - account_id, - mailbox.id.mailbox_id, - &deleted_ids, - &mut changelog, - ) - .await - .caused_by(trc::location!())?; + let mut batch = BatchBuilder::new(); + self.email_untag_or_delete(account_id, mailbox.id.mailbox_id, &deleted_ids, &mut batch) + .await + .caused_by(trc::location!())?; trc::event!( Imap(trc::ImapEvent::Expunge), @@ -166,16 +155,11 @@ impl SessionData { ); // Write changes on source account - if !changelog.is_empty() { - let change_id = self.server.commit_changes(account_id, changelog).await?; + if !batch.is_empty() { self.server - .broadcast_state_change( - StateChange::new(account_id) - .with_change(DataType::Email, change_id) - .with_change(DataType::Mailbox, change_id) - .with_change(DataType::Thread, change_id), - ) - .await; + .commit_batch(batch) + .await + .caused_by(trc::location!())?; } Ok(()) @@ -186,89 +170,54 @@ impl SessionData { account_id: u32, mailbox_id: u32, deleted_ids: &RoaringBitmap, - changelog: &mut ChangeLogBuilder, + batch: &mut BatchBuilder, ) -> trc::Result<()> { let mut destroy_ids = RoaringBitmap::new(); - let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Email); self.server - .get_archives( - account_id, - Collection::Email, - deleted_ids, - Property::Value, - |id, data_| { - let data = data_ - .to_unarchived::() - .caused_by(trc::location!())?; + .get_archives(account_id, Collection::Email, deleted_ids, |id, data_| { + let data = data_ + .to_unarchived::() + .caused_by(trc::location!())?; - if !data.inner.has_mailbox_id(mailbox_id) { - return Ok(true); - } else if data.inner.mailboxes.len() == 1 { - destroy_ids.insert(id); - return Ok(true); - } + if !data.inner.has_mailbox_id(mailbox_id) { + return Ok(true); + } else if data.inner.mailboxes.len() == 1 { + destroy_ids.insert(id); + return Ok(true); + } - // Prepare changes - let mut new_data = data.deserialize().caused_by(trc::location!())?; - if changelog.change_id == u64::MAX { - changelog.change_id = self.server.assign_change_id(account_id)? - } + // Untag message from this mailbox and remove Deleted flag + let mut new_data = data.deserialize().caused_by(trc::location!())?; + new_data.change_id = batch.change_id(); + new_data.remove_mailbox(mailbox_id); + new_data.remove_keyword(&Keyword::Deleted); - new_data.change_id = changelog.change_id; - let thread_id = new_data.thread_id; + // Write changes + batch + .update_document(id) + .custom( + ObjectIndexBuilder::new() + .with_current(data) + .with_changes(new_data), + ) + .caused_by(trc::location!())? + .commit_point(); - // Untag message from this mailbox and remove Deleted flag - new_data.remove_mailbox(mailbox_id); - new_data.remove_keyword(&Keyword::Deleted); - - changelog.log_update(Collection::Email, Id::from_parts(thread_id, id)); - changelog.log_child_update(Collection::Mailbox, mailbox_id); - - // Write changes - batch - .update_document(id) - .custom( - ObjectIndexBuilder::new() - .with_current(data) - .with_changes(new_data), - ) - .caused_by(trc::location!())?; - - Ok(true) - }, - ) + Ok(true) + }) .await .caused_by(trc::location!())?; - if !batch.is_empty() { - match self - .server - .store() - .write(batch) - .await - .caused_by(trc::location!()) - { - Ok(_) => {} - Err(err) => { - if !err.is_assertion_failure() { - return Err(err.caused_by(trc::location!())); - } - } - } - } - if !destroy_ids.is_empty() { // Delete message from all mailboxes - let (changes, _) = self - .server - .emails_tombstone(account_id, destroy_ids) + self.server + .emails_tombstone(account_id, batch, destroy_ids) .await .caused_by(trc::location!())?; - changelog.merge(changes); } Ok(()) diff --git a/crates/imap/src/op/fetch.rs b/crates/imap/src/op/fetch.rs index c6860c7a..10fb2cf9 100644 --- a/crates/imap/src/op/fetch.rs +++ b/crates/imap/src/op/fetch.rs @@ -37,13 +37,11 @@ use jmap_proto::types::{ id::Id, keyword::{ArchivedKeyword, Keyword}, property::Property, - state::StateChange, - type_state::DataType, }; use store::{ query::log::{Change, Query}, rkyv::rend::u16_le, - write::{AlignedBytes, Archive, BatchBuilder, log::ChangeLogBuilder}, + write::BatchBuilder, }; use super::{FromModSeq, ImapContext}; @@ -288,9 +286,8 @@ impl SessionData { } } - let mut update_batches = Vec::new(); - // Process each message + let mut batch = BatchBuilder::new(); let mut ids = ids .into_iter() .map(|(id, imap_id)| (imap_id.seqnum, imap_id.uid, id)) @@ -300,30 +297,21 @@ impl SessionData { .iter() .map(|id| trc::Value::from(id.2)) .collect::>(); - let change_id = self - .server - .generate_snowflake_id() - .imap_ctx(&arguments.tag, trc::location!())?; for (seqnum, uid, id) in ids { // Obtain attributes and keywords let (metadata_, data_) = if let (Some(email), Some(keywords)) = ( self.server - .get_property::>( + .get_archive_by_property( account_id, Collection::Email, id, - &Property::BodyStructure, + Property::BodyStructure, ) .await .imap_ctx(&arguments.tag, trc::location!())?, self.server - .get_property::>( - account_id, - Collection::Email, - id, - &Property::Value, - ) + .get_archive(account_id, Collection::Email, id) .await .imap_ctx(&arguments.tag, trc::location!())?, ) { @@ -551,10 +539,8 @@ impl SessionData { .deserialize() .imap_ctx(&arguments.tag, trc::location!())?; new_data.keywords.push(Keyword::Seen); - new_data.change_id = change_id; - let thread_id = new_data.thread_id; + new_data.change_id = batch.change_id(); - let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Email) @@ -564,47 +550,29 @@ impl SessionData { .with_current(data) .with_changes(new_data), ) - .imap_ctx(&arguments.tag, trc::location!())?; - - update_batches.push((Id::from_parts(thread_id, id), batch)); + .imap_ctx(&arguments.tag, trc::location!())? + .commit_point(); } } // Set Seen ids - if !update_batches.is_empty() { - let mut changelog = ChangeLogBuilder::with_change_id(change_id); - for (id, batch) in update_batches { - match self - .server - .store() - .write(batch) - .await - .imap_ctx(&arguments.tag, trc::location!()) - { - Ok(_) => { - changelog.log_update(Collection::Email, id); - } - Err(err) => { - if !err.is_assertion_failure() { - return Err(err.id(arguments.tag)); - } + if !batch.is_empty() { + let change_id = batch.change_id(); + match self + .server + .commit_batch(batch) + .await + .imap_ctx(&arguments.tag, trc::location!()) + { + Ok(_) => { + modseq = change_id.into(); + } + Err(err) => { + if !err.is_assertion_failure() { + return Err(err.id(arguments.tag)); } } } - if !changelog.is_empty() { - // Write changes - let change_id = self - .server - .commit_changes(account_id, changelog) - .await - .imap_ctx(&arguments.tag, trc::location!())?; - modseq = change_id.into(); - self.server - .broadcast_state_change( - StateChange::new(account_id).with_change(DataType::Email, change_id), - ) - .await; - } } trc::event!( diff --git a/crates/imap/src/op/rename.rs b/crates/imap/src/op/rename.rs index 85894004..0b87697c 100644 --- a/crates/imap/src/op/rename.rs +++ b/crates/imap/src/op/rename.rs @@ -15,10 +15,8 @@ use directory::Permission; use imap_proto::{ Command, ResponseCode, StatusResponse, protocol::rename::Arguments, receiver::Request, }; -use jmap_proto::types::{ - acl::Acl, collection::Collection, property::Property, state::StateChange, type_state::DataType, -}; -use store::write::{AlignedBytes, Archive, BatchBuilder}; +use jmap_proto::types::{acl::Acl, collection::Collection}; +use store::write::BatchBuilder; use trc::AddContext; use super::ImapContext; @@ -88,12 +86,7 @@ impl SessionData { // Obtain mailbox let mailbox = self .server - .get_property::>( - params.account_id, - Collection::Mailbox, - mailbox_id, - Property::Value, - ) + .get_archive(params.account_id, Collection::Mailbox, mailbox_id) .await .imap_ctx(&arguments.tag, trc::location!())? .ok_or_else(|| { @@ -130,37 +123,38 @@ impl SessionData { let new_mailbox_name = params.path.pop().unwrap(); // Build batch - let mut changes = self - .server - .begin_changes(params.account_id) - .imap_ctx(&arguments.tag, trc::location!())?; - let mut parent_id = params.parent_mailbox_id.map(|id| id + 1).unwrap_or(0); let mut create_ids = Vec::with_capacity(params.path.len()); + let mut next_document_id = self + .server + .store() + .assign_document_ids( + params.account_id, + Collection::Mailbox, + params.path.len() as u64, + ) + .await + .caused_by(trc::location!())?; + let mut batch = BatchBuilder::new(); + for &path_item in params.path.iter() { - let mut batch = BatchBuilder::new(); + let mailbox_id = next_document_id; + next_document_id -= 1; + batch .with_account_id(params.account_id) .with_collection(Collection::Mailbox) - .create_document() + .create_document(mailbox_id) .custom(ObjectIndexBuilder::<(), _>::new().with_changes( email::mailbox::Mailbox::new(path_item).with_parent_id(parent_id), )) - .imap_ctx(&arguments.tag, trc::location!())?; + .imap_ctx(&arguments.tag, trc::location!())? + .commit_point(); - let mailbox_id = self - .server - .store() - .write_expect_id(batch) - .await - .imap_ctx(&arguments.tag, trc::location!())?; - - changes.log_insert(Collection::Mailbox, mailbox_id); parent_id = mailbox_id + 1; create_ids.push(mailbox_id); } - let mut batch = BatchBuilder::new(); let mut new_mailbox = mailbox.inner.clone(); new_mailbox.name = new_mailbox_name.to_string(); new_mailbox.parent_id = parent_id; @@ -175,25 +169,12 @@ impl SessionData { .with_changes(new_mailbox), ) .imap_ctx(&arguments.tag, trc::location!())?; - changes.log_update(Collection::Mailbox, mailbox_id); - - let change_id = changes.change_id; - batch - .custom(changes) - .imap_ctx(&arguments.tag, trc::location!())?; + let change_id = batch.change_id(); self.server - .store() - .write(batch) + .commit_batch(batch) .await .imap_ctx(&arguments.tag, trc::location!())?; - // Broadcast changes - self.server - .broadcast_state_change( - StateChange::new(params.account_id).with_change(DataType::Mailbox, change_id), - ) - .await; - let mut mailboxes = if !create_ids.is_empty() { self.add_created_mailboxes(&mut params, change_id, create_ids) .add_context(|err| err.id(arguments.tag.clone()))? diff --git a/crates/imap/src/op/status.rs b/crates/imap/src/op/status.rs index f278b123..926ef69a 100644 --- a/crates/imap/src/op/status.rs +++ b/crates/imap/src/op/status.rs @@ -20,10 +20,7 @@ use imap_proto::{ receiver::Request, }; use jmap_proto::types::{collection::Collection, id::Id, keyword::Keyword, property::Property}; -use store::{ - Deserialize, U32_LEN, - write::{AlignedBytes, Archive}, -}; +use store::{Deserialize, U32_LEN}; use store::{ IndexKeyPrefix, IterateParams, roaring::RoaringBitmap, write::key::DeserializeBigEndian, }; @@ -255,11 +252,10 @@ impl SessionData { .caused_by(trc::location!())? as u64, Status::UidValidity => u32::from( self.server - .get_property::>( + .get_archive( mailbox.account_id, Collection::Mailbox, mailbox.mailbox_id, - &Property::Value, ) .await? .ok_or_else(|| { diff --git a/crates/imap/src/op/store.rs b/crates/imap/src/op/store.rs index 6dc15b67..657047cc 100644 --- a/crates/imap/src/op/store.rs +++ b/crates/imap/src/op/store.rs @@ -7,7 +7,7 @@ use std::{sync::Arc, time::Instant}; use crate::{ - core::{SelectedMailbox, Session, SessionData, message::MAX_RETRIES}, + core::{SelectedMailbox, Session, SessionData}, spawn_op, }; use ahash::AHashSet; @@ -23,13 +23,10 @@ use imap_proto::{ }, receiver::Request, }; -use jmap_proto::types::{ - acl::Acl, collection::Collection, id::Id, keyword::Keyword, property::Property, - state::StateChange, type_state::DataType, -}; +use jmap_proto::types::{acl::Acl, collection::Collection, keyword::Keyword}; use store::{ query::log::{Change, Query}, - write::{AlignedBytes, Archive, BatchBuilder, ValueClass, log::ChangeLogBuilder}, + write::{BatchBuilder, ValueClass}, }; use trc::AddContext; @@ -187,207 +184,172 @@ impl SessionData { .iter() .map(|k| Keyword::from(k.clone())) .collect::>(); - let mut changelog = ChangeLogBuilder::new(); - let mut changed_mailboxes = AHashSet::new(); let access_token = self .server .get_access_token(account_id) .await .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?; + let mut changed_mailboxes = AHashSet::new(); let can_spam_train = self.server.email_bayes_can_train(&access_token); let mut has_spam_train_tasks = false; + let mut batch = BatchBuilder::new(); - 'outer: for (id, imap_id) in &ids { - let mut try_count = 0; - loop { - // Obtain message data - let data_ = if let Some(data) = self - .server - .get_property::>( - account_id, - Collection::Email, - *id, - Property::Value, - ) - .await - .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())? - { - data - } else { - continue 'outer; - }; + for (id, imap_id) in &ids { + // Obtain message data + let data_ = if let Some(data) = self + .server + .get_archive(account_id, Collection::Email, *id) + .await + .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())? + { + data + } else { + continue; + }; - // Deserialize - let data = data_ - .to_unarchived::() - .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?; - let mut new_data = data - .deserialize() - .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?; - let thread_id = new_data.thread_id; + // Deserialize + let data = data_ + .to_unarchived::() + .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?; + let mut new_data = data + .deserialize() + .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?; - // Apply changes - let mut seen_changed = false; - match arguments.operation { - Operation::Set => { - seen_changed = set_keywords.contains(&Keyword::Seen) - != new_data.has_keyword(&Keyword::Seen); - new_data.set_keywords(set_keywords.clone()); - } - Operation::Add => { - for keyword in &set_keywords { - if new_data.add_keyword(keyword.clone()) && keyword == &Keyword::Seen { - seen_changed = true; - } - } - } - Operation::Clear => { - for keyword in &set_keywords { - if new_data.remove_keyword(keyword) && keyword == &Keyword::Seen { - seen_changed = true; - } + // Apply changes + let mut seen_changed = false; + match arguments.operation { + Operation::Set => { + seen_changed = set_keywords.contains(&Keyword::Seen) + != new_data.has_keyword(&Keyword::Seen); + new_data.set_keywords(set_keywords.clone()); + } + Operation::Add => { + for keyword in &set_keywords { + if new_data.add_keyword(keyword.clone()) && keyword == &Keyword::Seen { + seen_changed = true; } } } - - if new_data.has_keyword_changes(data.inner) { - // Train spam filter - let mut train_spam = None; - if can_spam_train { - for keyword in new_data.added_keywords(data.inner) { - if keyword == &Keyword::Junk { - train_spam = Some(true); - break; - } else if keyword == &Keyword::NotJunk { - train_spam = Some(false); - break; - } + Operation::Clear => { + for keyword in &set_keywords { + if new_data.remove_keyword(keyword) && keyword == &Keyword::Seen { + seen_changed = true; } - if train_spam.is_none() { - for keyword in new_data.removed_keywords(data.inner) { - if keyword == &Keyword::Junk { - train_spam = Some(false); - break; - } - } - } - }; + } + } + } - // Convert keywords to flags - let flags = if !arguments.is_silent { - new_data - .keywords - .iter() - .cloned() - .map(Flag::from) - .collect::>() + if !new_data.has_keyword_changes(data.inner) { + continue; + } + + // Train spam filter + let mut train_spam = None; + if can_spam_train { + for keyword in new_data.added_keywords(data.inner) { + if keyword == &Keyword::Junk { + train_spam = Some(true); + break; + } else if keyword == &Keyword::NotJunk { + train_spam = Some(false); + break; + } + } + if train_spam.is_none() { + for keyword in new_data.removed_keywords(data.inner) { + if keyword == &Keyword::Junk { + train_spam = Some(false); + break; + } + } + } + }; + + // Convert keywords to flags + let flags = if !arguments.is_silent { + new_data + .keywords + .iter() + .cloned() + .map(Flag::from) + .collect::>() + } else { + vec![] + }; + + // Add change id + new_data.change_id = batch.change_id(); + let modseq = new_data.change_id + 1; + + // Set all current mailboxes as changed if the Seen tag changed + if seen_changed { + for mailbox_id in new_data.mailboxes.iter() { + changed_mailboxes.insert(mailbox_id.mailbox_id); + } + } + + // Write changes + batch + .with_account_id(account_id) + .with_collection(Collection::Email) + .update_document(*id) + .custom( + ObjectIndexBuilder::new() + .with_current(data) + .with_changes(new_data), + ) + .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?; + + // Add spam train task + if let Some(learn_spam) = train_spam { + batch.set( + ValueClass::TaskQueue( + self.server + .email_bayes_queue_task_build(account_id, *id, learn_spam) + .await + .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?, + ), + vec![], + ); + has_spam_train_tasks = true; + } + + // Set commit point + batch.commit_point(); + + // Add item to response + if !arguments.is_silent { + let mut data_items = vec![DataItem::Flags { flags }]; + if is_uid { + data_items.push(DataItem::Uid { uid: imap_id.uid }); + } + if is_condstore { + data_items.push(DataItem::ModSeq { modseq }); + } + items.items.push(FetchItem { + id: imap_id.seqnum, + items: data_items, + }); + } else if is_condstore { + items.items.push(FetchItem { + id: imap_id.seqnum, + items: if is_uid { + vec![ + DataItem::ModSeq { modseq }, + DataItem::Uid { uid: imap_id.uid }, + ] } else { - vec![] - }; - - // Add change id - if changelog.change_id == u64::MAX { - changelog.change_id = self - .server - .assign_change_id(account_id) - .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())? - } - new_data.change_id = changelog.change_id; - - // Set all current mailboxes as changed if the Seen tag changed - if seen_changed { - for mailbox_id in new_data.mailboxes.iter() { - changed_mailboxes.insert(mailbox_id.mailbox_id); - } - } - - // Write changes - let mut batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_collection(Collection::Email) - .update_document(*id) - .custom( - ObjectIndexBuilder::new() - .with_current(data) - .with_changes(new_data), - ) - .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?; - - // Add spam train task - if let Some(learn_spam) = train_spam { - batch.set( - ValueClass::TaskQueue( - self.server - .email_bayes_queue_task_build(account_id, *id, learn_spam) - .await - .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?, - ), - vec![], - ); - has_spam_train_tasks = true; - } - - match self - .server - .store() - .write(batch) - .await - .caused_by(trc::location!()) - { - Ok(_) => { - // Update changelog - changelog.log_update(Collection::Email, Id::from_parts(thread_id, *id)); - - // Add item to response - let modseq = changelog.change_id + 1; - if !arguments.is_silent { - let mut data_items = vec![DataItem::Flags { flags }]; - if is_uid { - data_items.push(DataItem::Uid { uid: imap_id.uid }); - } - if is_condstore { - data_items.push(DataItem::ModSeq { modseq }); - } - items.items.push(FetchItem { - id: imap_id.seqnum, - items: data_items, - }); - } else if is_condstore { - items.items.push(FetchItem { - id: imap_id.seqnum, - items: if is_uid { - vec![ - DataItem::ModSeq { modseq }, - DataItem::Uid { uid: imap_id.uid }, - ] - } else { - vec![DataItem::ModSeq { modseq }] - }, - }); - } - } - Err(err) if err.is_assertion_failure() => { - if try_count < MAX_RETRIES { - try_count += 1; - continue; - } else { - response.rtype = ResponseType::No; - response.message = "Some messages could not be updated.".into(); - } - } - Err(err) => { - return Err(err.id(response.tag.unwrap())); - } - } - } - break; + vec![DataItem::ModSeq { modseq }] + }, + }); } } // Log mailbox changes - for mailbox_id in &changed_mailboxes { - changelog.log_child_update(Collection::Mailbox, *mailbox_id); + if !changed_mailboxes.is_empty() { + for parent_id in changed_mailboxes { + batch.log_child_update(Collection::Mailbox, parent_id); + } } // Trigger Bayes training @@ -396,21 +358,23 @@ impl SessionData { } // Write changes - if !changelog.is_empty() { - let change_id = self + if !batch.is_empty() { + match self .server - .commit_changes(account_id, changelog) + .commit_batch(batch) .await - .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?; - self.server - .broadcast_state_change(if !changed_mailboxes.is_empty() { - StateChange::new(account_id) - .with_change(DataType::Email, change_id) - .with_change(DataType::Mailbox, change_id) - } else { - StateChange::new(account_id).with_change(DataType::Email, change_id) - }) - .await; + .caused_by(trc::location!()) + { + Ok(_) => {} + Err(err) if err.is_assertion_failure() => { + items.items.clear(); + response.rtype = ResponseType::No; + response.message = "Some messages were modified by another process.".into(); + } + Err(err) => { + return Err(err.id(response.tag.unwrap())); + } + } } trc::event!( diff --git a/crates/imap/src/op/subscribe.rs b/crates/imap/src/op/subscribe.rs index 6ee13c86..bab49207 100644 --- a/crates/imap/src/op/subscribe.rs +++ b/crates/imap/src/op/subscribe.rs @@ -13,10 +13,8 @@ use crate::{ use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; use imap_proto::{Command, ResponseCode, StatusResponse, receiver::Request}; -use jmap_proto::types::{ - collection::Collection, property::Property, state::StateChange, type_state::DataType, -}; -use store::write::{AlignedBytes, Archive, BatchBuilder}; +use jmap_proto::types::collection::Collection; +use store::write::BatchBuilder; use super::ImapContext; @@ -94,14 +92,9 @@ impl SessionData { } // Obtain mailbox - let mailbox = self + let mailbox_ = self .server - .get_property::>( - account_id, - Collection::Mailbox, - mailbox_id, - Property::Value, - ) + .get_archive(account_id, Collection::Mailbox, mailbox_id) .await .imap_ctx(&tag, trc::location!())? .ok_or_else(|| { @@ -111,19 +104,16 @@ impl SessionData { .code(ResponseCode::NonExistent) .id(tag.clone()) .caused_by(trc::location!()) - })? - .into_deserialized::() + })?; + let mailbox = mailbox_ + .to_unarchived::() .imap_ctx(&tag, trc::location!())?; if (subscribe && !mailbox.inner.is_subscribed(self.account_id)) || (!subscribe && mailbox.inner.is_subscribed(self.account_id)) { // Build batch - let mut changes = self - .server - .begin_changes(account_id) - .imap_ctx(&tag, trc::location!())?; - let mut new_mailbox = mailbox.inner.clone(); + let mut new_mailbox = mailbox.deserialize().imap_ctx(&tag, trc::location!())?; if subscribe { new_mailbox.subscribers.push(self.account_id); } else { @@ -140,23 +130,12 @@ impl SessionData { .with_changes(new_mailbox), ) .imap_ctx(&tag, trc::location!())?; - changes.log_update(Collection::Mailbox, mailbox_id); - - let change_id = changes.change_id; - batch.custom(changes).imap_ctx(&tag, trc::location!())?; + let change_id = batch.change_id(); self.server - .store() - .write(batch) + .commit_batch(batch) .await .imap_ctx(&tag, trc::location!())?; - // Broadcast changes - self.server - .broadcast_state_change( - StateChange::new(account_id).with_change(DataType::Mailbox, change_id), - ) - .await; - // Update mailbox cache for account in self.mailboxes.lock().iter_mut() { if account.account_id == account_id { diff --git a/crates/jmap-proto/src/method/set.rs b/crates/jmap-proto/src/method/set.rs index a51508b5..9f9cc450 100644 --- a/crates/jmap-proto/src/method/set.rs +++ b/crates/jmap-proto/src/method/set.rs @@ -25,7 +25,7 @@ use crate::{ id::Id, keyword::Keyword, property::{HeaderForm, ObjectProperty, Property, SetProperty}, - state::{State, StateChange}, + state::State, value::{Object, SetValue, SetValueMap, Value}, }, }; @@ -90,9 +90,6 @@ pub struct SetResponse { #[serde(rename = "notDestroyed")] #[serde(skip_serializing_if = "VecMap::is_empty")] pub not_destroyed: VecMap, - - #[serde(skip)] - pub state_change: Option, } impl JsonObjectParser for SetRequest { @@ -475,7 +472,6 @@ impl SetResponse { not_created: VecMap::new(), not_updated: VecMap::new(), not_destroyed: VecMap::new(), - state_change: None, }) } else { Err(trc::JmapEvent::RequestTooLarge.into_err()) diff --git a/crates/jmap-proto/src/types/keyword.rs b/crates/jmap-proto/src/types/keyword.rs index 91e88cce..3bfe6dbe 100644 --- a/crates/jmap-proto/src/types/keyword.rs +++ b/crates/jmap-proto/src/types/keyword.rs @@ -6,10 +6,7 @@ use std::fmt::Display; -use store::{ - Serialize, - write::{MaybeDynamicId, TagValue}, -}; +use store::{Serialize, write::TagValue}; use crate::parser::{JsonObjectParser, json::Parser}; @@ -276,7 +273,7 @@ impl ArchivedKeyword { } } -impl From for TagValue { +impl From for TagValue { fn from(value: Keyword) -> Self { match value.into_id() { Ok(id) => TagValue::Id(id), @@ -285,7 +282,7 @@ impl From for TagValue { } } -impl From<&Keyword> for TagValue { +impl From<&Keyword> for TagValue { fn from(value: &Keyword) -> Self { match value.id() { Ok(id) => TagValue::Id(id), @@ -294,10 +291,10 @@ impl From<&Keyword> for TagValue { } } -impl From<&ArchivedKeyword> for TagValue { +impl From<&ArchivedKeyword> for TagValue { fn from(value: &ArchivedKeyword) -> Self { match value.id() { - Ok(id) => TagValue::Id(MaybeDynamicId::Static(id)), + Ok(id) => TagValue::Id(id), Err(string) => TagValue::Text(string.into_bytes()), } } diff --git a/crates/jmap-proto/src/types/property.rs b/crates/jmap-proto/src/types/property.rs index d0279463..fdcdc1ac 100644 --- a/crates/jmap-proto/src/types/property.rs +++ b/crates/jmap-proto/src/types/property.rs @@ -1226,7 +1226,7 @@ impl AsRef for Property { } } -impl From for ValueClass { +impl From for ValueClass { fn from(value: Property) -> Self { ValueClass::Property(value.into()) } diff --git a/crates/jmap-proto/src/types/state.rs b/crates/jmap-proto/src/types/state.rs index 3788d447..a19160d2 100644 --- a/crates/jmap-proto/src/types/state.rs +++ b/crates/jmap-proto/src/types/state.rs @@ -42,6 +42,10 @@ impl StateChange { } } + pub fn set_change(&mut self, type_state: DataType, change_id: u64) { + self.types.push((type_state, change_id)); + } + pub fn with_change(mut self, type_state: DataType, change_id: u64) -> Self { if let Some((_, last_change_id)) = self.types.iter_mut().find(|(ts, _)| ts == &type_state) { *last_change_id = change_id; diff --git a/crates/jmap-proto/src/types/type_state.rs b/crates/jmap-proto/src/types/type_state.rs index 53bd7e80..04bc498f 100644 --- a/crates/jmap-proto/src/types/type_state.rs +++ b/crates/jmap-proto/src/types/type_state.rs @@ -7,7 +7,7 @@ use std::fmt::Display; use serde::Serialize; -use utils::map::bitmap::BitmapItem; +use utils::map::bitmap::{BitmapItem, ShortId}; use crate::parser::{JsonObjectParser, json::Parser}; @@ -172,6 +172,29 @@ impl TryFrom<&str> for DataType { } } +impl TryFrom for DataType { + type Error = (); + + fn try_from(value: ShortId) -> Result { + match value.0 { + 0 => Ok(DataType::Email), + 1 => Ok(DataType::Mailbox), + 2 => Ok(DataType::Thread), + 3 => Ok(DataType::Identity), + 4 => Ok(DataType::EmailSubmission), + 5 => Ok(DataType::SieveScript), + 6 => Ok(DataType::PushSubscription), + 8 => Ok(DataType::Calendar), + 9 => Ok(DataType::CalendarEvent), + 10 => Ok(DataType::CalendarEventNotification), + 11 => Ok(DataType::AddressBook), + 12 => Ok(DataType::ContactCard), + 13 => Ok(DataType::FileNode), + _ => Err(()), + } + } +} + impl DataType { pub fn as_str(&self) -> &'static str { match self { diff --git a/crates/jmap/src/api/request.rs b/crates/jmap/src/api/request.rs index 9fdd3928..5fa6332e 100644 --- a/crates/jmap/src/api/request.rs +++ b/crates/jmap/src/api/request.rs @@ -105,11 +105,6 @@ impl RequestHandler for Server { ResponseMethod::Set(set_response) => { // Add created ids set_response.update_created_ids(&mut response); - - // Publish state changes - if let Some(state_change) = set_response.state_change.take() { - self.broadcast_state_change(state_change).await; - } } ResponseMethod::ImportEmail(import_response) => { // Add created ids diff --git a/crates/jmap/src/blob/copy.rs b/crates/jmap/src/blob/copy.rs index b90c07a4..0d9b5b23 100644 --- a/crates/jmap/src/blob/copy.rs +++ b/crates/jmap/src/blob/copy.rs @@ -55,7 +55,7 @@ impl BlobCopy for Server { 0u32.serialize(), ); self.store() - .write(batch) + .write(batch.build_all()) .await .caused_by(trc::location!())?; let dest_blob_id = BlobId { diff --git a/crates/jmap/src/blob/get.rs b/crates/jmap/src/blob/get.rs index 20af554d..563a31db 100644 --- a/crates/jmap/src/blob/get.rs +++ b/crates/jmap/src/blob/get.rs @@ -24,10 +24,7 @@ use jmap_proto::{ use mail_builder::encoders::base64::base64_encode; use sha1::{Digest, Sha1}; use sha2::{Sha256, Sha512}; -use store::{ - BlobClass, - write::{AlignedBytes, Archive}, -}; +use store::BlobClass; use trc::AddContext; use utils::map::vec_map::VecMap; @@ -217,12 +214,7 @@ impl BlobOperations for Server { let collection = Collection::from(*collection); if collection == Collection::Email { if let Some(data_) = self - .get_property::>( - req_account_id, - Collection::Email, - *document_id, - Property::Value, - ) + .get_archive(req_account_id, Collection::Email, *document_id) .await? { let data = data_ diff --git a/crates/jmap/src/email/get.rs b/crates/jmap/src/email/get.rs index 1539eadf..e82ed853 100644 --- a/crates/jmap/src/email/get.rs +++ b/crates/jmap/src/email/get.rs @@ -27,10 +27,7 @@ use jmap_proto::{ }, }; -use store::{ - BlobClass, - write::{AlignedBytes, Archive}, -}; +use store::BlobClass; use trc::{AddContext, StoreEvent}; use utils::BlobHash; @@ -157,7 +154,7 @@ impl EmailGet for Server { continue; } let metadata_ = match self - .get_property::>( + .get_archive_by_property( account_id, Collection::Email, id.document_id(), @@ -177,12 +174,7 @@ impl EmailGet for Server { // Obtain message data let data_ = match self - .get_property::>( - account_id, - Collection::Email, - id.document_id(), - &Property::Value, - ) + .get_archive(account_id, Collection::Email, id.document_id()) .await? { Some(data) => data, diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index 27ed9dc5..189e9376 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -39,14 +39,10 @@ use mail_builder::{ mime::{BodyPart, MimePart}, }; use mail_parser::MessageParser; -use store::{ - ahash::AHashSet, - roaring::RoaringBitmap, - write::{AlignedBytes, Archive, BatchBuilder, log::ChangeLogBuilder}, -}; +use store::{ahash::AHashSet, roaring::RoaringBitmap, write::BatchBuilder}; use trc::AddContext; -use crate::{JmapMethods, blob::download::BlobDownload, changes::state::StateManager}; +use crate::{JmapMethods, blob::download::BlobDownload}; use std::future::Future; use super::headers::{BuildHeader, ValueToHeader}; @@ -110,6 +106,7 @@ impl EmailSet for Server { (None, None, None) }; + let mut last_change_id = None; let will_destroy = request.unwrap_destroy(); // Obtain quota @@ -743,6 +740,7 @@ impl EmailSet for Server { .await { Ok(message) => { + last_change_id = message.change_id.into(); response.created.insert(id, message.into()); } Err(err) if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) => { @@ -757,7 +755,9 @@ impl EmailSet for Server { } // Process updates - let mut changes = ChangeLogBuilder::new(); + let mut batch = BatchBuilder::new(); + let mut changed_mailboxes = AHashSet::new(); + let mut will_update = Vec::with_capacity(request.update.as_ref().map_or(0, |u| u.len())); 'update: for (id, object) in request.unwrap_update() { // Make sure id won't be destroyed if will_destroy.contains(&id) { @@ -768,12 +768,7 @@ impl EmailSet for Server { // Obtain message data let document_id = id.document_id(); let data_ = match self - .get_property::>( - account_id, - Collection::Email, - document_id, - &Property::Value, - ) + .get_archive(account_id, Collection::Email, document_id) .await? { Some(data) => data, @@ -854,19 +849,6 @@ impl EmailSet for Server { continue 'update; } - // Prepare write batch - let mut batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_collection(Collection::Email) - .update_document(document_id); - if changes.change_id == u64::MAX { - changes.change_id = self.assign_change_id(account_id)?; - } - new_data.change_id = changes.change_id; - let mut changed_mailboxes = AHashSet::new(); - changes.log_update(Collection::Email, id); - // Process keywords if has_keyword_changes { // Verify permissions on shared accounts @@ -966,32 +948,47 @@ impl EmailSet for Server { } } - // Log mailbox changes - for mailbox_id in changed_mailboxes { - changes.log_child_update(Collection::Mailbox, mailbox_id); - } + // Update change id + new_data.change_id = batch.change_id(); + last_change_id = new_data.change_id.into(); // Write changes batch + .with_account_id(account_id) + .with_collection(Collection::Email) + .update_document(document_id) .custom( ObjectIndexBuilder::new() .with_current(data) .with_changes(new_data), ) - .caused_by(trc::location!())?; + .caused_by(trc::location!())? + .commit_point(); + will_update.push(id); + } - match self.core.storage.data.write(batch.build()).await { + if !batch.is_empty() { + // Log mailbox changes + for parent_id in changed_mailboxes { + batch.log_child_update(Collection::Mailbox, parent_id); + } + + match self.commit_batch(batch).await { Ok(_) => { // Add to updated list - response.updated.append(id, None); + for id in will_update { + response.updated.append(id, None); + } } Err(err) if err.is_assertion_failure() => { - response.not_updated.append( - id, - SetError::forbidden().with_description( - "Another process modified this message, please try again.", - ), - ); + for id in will_update { + response.not_updated.append( + id, + SetError::forbidden().with_description( + "Another process modified this message, please try again.", + ), + ); + } } Err(err) => { return Err(err.caused_by(trc::location!())); @@ -1044,11 +1041,14 @@ impl EmailSet for Server { if !destroy_ids.is_empty() { // Batch delete (tombstone) messages - let (change, not_destroyed) = - self.emails_tombstone(account_id, destroy_ids).await?; - - // Merge changes - changes.merge(change); + let mut batch = BatchBuilder::new(); + let not_destroyed = self + .emails_tombstone(account_id, &mut batch, destroy_ids) + .await?; + if !batch.is_empty() { + last_change_id = batch.change_id().into(); + self.commit_batch(batch).await.caused_by(trc::location!())?; + } // Mark messages that were not found as not destroyed (this should not occur in practice) if !not_destroyed.is_empty() { @@ -1070,21 +1070,19 @@ impl EmailSet for Server { } // Update state - if !changes.is_empty() || !response.created.is_empty() { - let new_state = if !changes.is_empty() { - self.commit_changes(account_id, changes).await?.into() - } else { - self.get_state(account_id, Collection::Email).await? - }; - if let State::Exact(change_id) = &new_state { - response.state_change = StateChange::new(account_id) - .with_change(DataType::Email, *change_id) - .with_change(DataType::Mailbox, *change_id) - .with_change(DataType::Thread, *change_id) - .into(); + if let Some(change_id) = last_change_id { + if response.updated.is_empty() && response.destroyed.is_empty() { + // Message ingest does not broadcast state changes + self.broadcast_state_change( + StateChange::new(account_id) + .with_change(DataType::Email, change_id) + .with_change(DataType::Mailbox, change_id) + .with_change(DataType::Thread, change_id), + ) + .await; } - response.new_state = new_state.into(); + response.new_state = State::Exact(change_id).into(); } Ok(response) diff --git a/crates/jmap/src/email/snippet.rs b/crates/jmap/src/email/snippet.rs index 5b99d4a5..8955e598 100644 --- a/crates/jmap/src/email/snippet.rs +++ b/crates/jmap/src/email/snippet.rs @@ -18,10 +18,7 @@ use jmap_proto::{ }; use mail_parser::decoders::html::html_to_text; use nlp::language::{Language, search_snippet::generate_snippet, stemmer::Stemmer}; -use store::{ - backend::MAX_TOKEN_LENGTH, - write::{AlignedBytes, Archive}, -}; +use store::backend::MAX_TOKEN_LENGTH; use trc::AddContext; use utils::BlobHash; @@ -125,11 +122,11 @@ impl EmailSearchSnippet for Server { continue; } let metadata_ = match self - .get_property::>( + .get_archive_by_property( account_id, Collection::Email, document_id, - &Property::BodyStructure, + Property::BodyStructure, ) .await? { diff --git a/crates/jmap/src/identity/get.rs b/crates/jmap/src/identity/get.rs index 05fb10f6..0a62df45 100644 --- a/crates/jmap/src/identity/get.rs +++ b/crates/jmap/src/identity/get.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::Server; +use common::{Server, storage::index::ObjectIndexBuilder}; use directory::{QueryBy, backend::internal::PrincipalField}; use email::identity::{ArchivedEmailAddress, Identity}; use jmap_proto::{ @@ -16,10 +16,9 @@ use jmap_proto::{ }, }; use store::{ - Serialize, rkyv::{option::ArchivedOption, vec::ArchivedVec}, roaring::RoaringBitmap, - write::{AlignedBytes, Archive, Archiver, BatchBuilder}, + write::BatchBuilder, }; use trc::AddContext; use utils::sanitize_email; @@ -85,12 +84,7 @@ impl IdentityGet for Server { continue; } let _identity = if let Some(identity) = self - .get_property::>( - account_id, - Collection::Identity, - document_id, - Property::Value, - ) + .get_archive(account_id, Collection::Identity, document_id) .await? { identity @@ -174,8 +168,12 @@ impl IdentityGet for Server { .trim() .to_string(); let has_many = num_emails > 1; - for (idx, email) in principal.iter_str(PrincipalField::Emails).enumerate() { - let document_id = idx as u32; + let mut next_document_id = self + .store() + .assign_document_ids(account_id, Collection::Identity, num_emails as u64) + .await + .caused_by(trc::location!())?; + for email in principal.iter_str(PrincipalField::Emails) { let email = sanitize_email(email).unwrap_or_default(); if email.is_empty() { continue; @@ -187,24 +185,19 @@ impl IdentityGet for Server { } else { name.clone() }; - batch.create_document_with_id(document_id).set( - Property::Value, - Archiver::new(Identity { + let document_id = next_document_id; + next_document_id -= 1; + batch + .create_document(document_id) + .custom(ObjectIndexBuilder::<(), _>::new().with_changes(Identity { name, email, ..Default::default() - }) - .serialize() - .caused_by(trc::location!())?, - ); + })) + .caused_by(trc::location!())?; identity_ids.insert(document_id); } - self.core - .storage - .data - .write(batch.build()) - .await - .caused_by(trc::location!())?; + self.commit_batch(batch).await.caused_by(trc::location!())?; Ok(identity_ids) } diff --git a/crates/jmap/src/identity/set.rs b/crates/jmap/src/identity/set.rs index 97ac1b72..7eedd668 100644 --- a/crates/jmap/src/identity/set.rs +++ b/crates/jmap/src/identity/set.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::Server; +use common::{Server, storage::index::ObjectIndexBuilder}; use directory::{QueryBy, backend::internal::PrincipalField}; use email::identity::{EmailAddress, Identity}; use jmap_proto::{ @@ -14,12 +14,12 @@ use jmap_proto::{ types::{ collection::Collection, property::Property, + state::State, value::{MaybePatchValue, Value}, }, }; use std::future::Future; -use store::write::{AlignedBytes, Archive, BatchBuilder, log::ChangeLogBuilder}; -use store::{Serialize, write::Archiver}; +use store::write::BatchBuilder; use trc::AddContext; use utils::sanitize_email; @@ -36,7 +36,7 @@ impl IdentitySet for Server { mut request: SetRequest, ) -> trc::Result { let account_id = request.account_id.document_id(); - let mut identity_ids = self + let identity_ids = self .get_document_ids(account_id, Collection::Identity) .await? .unwrap_or_default(); @@ -44,7 +44,7 @@ impl IdentitySet for Server { let will_destroy = request.unwrap_destroy(); // Process creates - let mut changes = ChangeLogBuilder::new(); + let mut batch = BatchBuilder::new(); 'create: for (id, object) in request.unwrap_create() { let mut identity = Identity::default(); @@ -89,24 +89,18 @@ impl IdentitySet for Server { } // Insert record - let mut batch = BatchBuilder::new(); + let document_id = self + .store() + .assign_document_ids(account_id, Collection::Identity, 1) + .await + .caused_by(trc::location!())?; batch .with_account_id(account_id) .with_collection(Collection::Identity) - .create_document() - .set( - Property::Value, - Archiver::new(identity) - .serialize() - .caused_by(trc::location!())?, - ); - let document_id = self - .store() - .write_expect_id(batch) - .await - .caused_by(trc::location!())?; - identity_ids.insert(document_id); - changes.log_insert(Collection::Identity, document_id); + .create_document(document_id) + .custom(ObjectIndexBuilder::<(), _>::new().with_changes(identity)) + .caused_by(trc::location!())? + .commit_point(); response.created(id, document_id); } @@ -120,26 +114,25 @@ impl IdentitySet for Server { // Obtain identity let document_id = id.document_id(); - let mut identity = if let Some(identity) = self - .get_property::>( - account_id, - Collection::Identity, - document_id, - Property::Value, - ) + let identity_ = if let Some(identity_) = self + .get_archive(account_id, Collection::Identity, document_id) .await? { - identity - .deserialize::() - .caused_by(trc::location!())? + identity_ } else { response.not_updated.append(id, SetError::not_found()); continue 'update; }; + let identity = identity_ + .to_unarchived::() + .caused_by(trc::location!())?; + let mut new_identity = identity + .deserialize::() + .caused_by(trc::location!())?; for (property, value) in object.0 { if let Err(err) = response.eval_object_references(value).and_then(|value| { - validate_identity_value(&property, value, &mut identity, false) + validate_identity_value(&property, value, &mut new_identity, false) }) { response.not_updated.append(id, err); continue 'update; @@ -147,22 +140,17 @@ impl IdentitySet for Server { } // Update record - let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Identity) .update_document(document_id) - .set( - Property::Value, - Archiver::new(identity) - .serialize() - .caused_by(trc::location!())?, - ); - self.store() - .write(batch) - .await - .caused_by(trc::location!())?; - changes.log_update(Collection::Identity, document_id); + .custom( + ObjectIndexBuilder::new() + .with_current(identity) + .with_changes(new_identity), + ) + .caused_by(trc::location!())? + .commit_point(); response.updated.append(id, None); } @@ -171,17 +159,13 @@ impl IdentitySet for Server { let document_id = id.document_id(); if identity_ids.contains(document_id) { // Update record - let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Identity) .delete_document(document_id) - .clear(Property::Value); - self.store() - .write(batch) - .await - .caused_by(trc::location!())?; - changes.log_delete(Collection::Identity, document_id); + .clear(Property::Value) + .log_delete(None) + .commit_point(); response.destroyed.push(id); } else { response.not_destroyed.append(id, SetError::not_found()); @@ -189,8 +173,11 @@ impl IdentitySet for Server { } // Write changes - if !changes.is_empty() { - response.new_state = Some(self.commit_changes(account_id, changes).await?.into()); + if !batch.is_empty() { + let change_id = batch.change_id(); + self.commit_batch(batch).await.caused_by(trc::location!())?; + + response.new_state = State::Exact(change_id).into(); } Ok(response) diff --git a/crates/jmap/src/mailbox/get.rs b/crates/jmap/src/mailbox/get.rs index 97c7397a..d7f8a1a5 100644 --- a/crates/jmap/src/mailbox/get.rs +++ b/crates/jmap/src/mailbox/get.rs @@ -15,7 +15,6 @@ use jmap_proto::{ value::{Object, Value}, }, }; -use store::write::{AlignedBytes, Archive}; use trc::AddContext; use crate::changes::state::StateManager; @@ -98,12 +97,7 @@ impl MailboxGet for Server { let archived_mailbox_ = if fetch_properties { match self - .get_property::>( - account_id, - Collection::Mailbox, - document_id, - &Property::Value, - ) + .get_archive(account_id, Collection::Mailbox, document_id) .await? { Some(values) => values, diff --git a/crates/jmap/src/mailbox/set.rs b/crates/jmap/src/mailbox/set.rs index 534353f7..e1e88624 100644 --- a/crates/jmap/src/mailbox/set.rs +++ b/crates/jmap/src/mailbox/set.rs @@ -20,8 +20,7 @@ use jmap_proto::{ collection::Collection, id::Id, property::Property, - state::StateChange, - type_state::DataType, + state::State, value::{MaybePatchValue, Object, SetValue, Value}, }, }; @@ -29,7 +28,7 @@ use store::{ SerializeInfallible, query::Filter, roaring::RoaringBitmap, - write::{AlignedBytes, Archive, BatchBuilder, assert::AssertValue, log::ChangeLogBuilder}, + write::{Archive, BatchBuilder, assert::AssertValue}, }; use trc::AddContext; use utils::config::utils::ParseValue; @@ -84,13 +83,13 @@ impl MailboxSet for Server { mailbox_ids: self.mailbox_get_or_create(account_id).await?, will_destroy: request.unwrap_destroy(), }; + let mut change_id = None; // Process creates - let mut changes = ChangeLogBuilder::new(); + let mut batch = BatchBuilder::new(); 'create: for (id, object) in request.unwrap_create() { match self.mailbox_set_item(object, None, &ctx).await? { Ok(builder) => { - let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Mailbox); @@ -102,37 +101,20 @@ impl MailboxSet for Server { .assert_value(Property::Value, AssertValue::Some); } - batch - .create_document() - .custom(builder) + let document_id = self + .store() + .assign_document_ids(account_id, Collection::Mailbox, 1) + .await .caused_by(trc::location!())?; - match self - .core - .storage - .data - .write(batch.build()) - .await - .and_then(|ids| ids.last_document_id()) - { - Ok(document_id) => { - changes.log_insert(Collection::Mailbox, document_id); - ctx.mailbox_ids.insert(document_id); - ctx.response.created(id, document_id); - } - Err(err) if err.is_assertion_failure() => { - ctx.response.not_created.append( - id, - SetError::forbidden().with_description( - "Another process deleted the parent mailbox, please try again.", - ), - ); - continue 'create; - } - Err(err) => { - return Err(err.caused_by(trc::location!())); - } - } + batch + .create_document(document_id) + .custom(builder) + .caused_by(trc::location!())? + .commit_point(); + + ctx.mailbox_ids.insert(document_id); + ctx.response.created(id, document_id); } Err(err) => { ctx.response.not_created.append(id, err); @@ -141,7 +123,14 @@ impl MailboxSet for Server { } } + if !batch.is_empty() { + change_id = Some(batch.change_id()); + self.commit_batch(batch).await.caused_by(trc::location!())?; + } + // Process updates + let mut will_update = Vec::with_capacity(request.update.as_ref().map_or(0, |u| u.len())); + let mut batch = BatchBuilder::new(); 'update: for (id, object) in request.unwrap_update() { // Make sure id won't be destroyed if ctx.will_destroy.contains(&id) { @@ -154,12 +143,7 @@ impl MailboxSet for Server { // Obtain mailbox let document_id = id.document_id(); if let Some(mailbox) = self - .get_property::>( - account_id, - Collection::Mailbox, - document_id, - Property::Value, - ) + .get_archive(account_id, Collection::Mailbox, document_id) .await? { // Validate ACL @@ -193,7 +177,6 @@ impl MailboxSet for Server { .await? { Ok(builder) => { - let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Mailbox); @@ -208,25 +191,9 @@ impl MailboxSet for Server { batch .update_document(document_id) .custom(builder) - .caused_by(trc::location!())?; - - if !batch.is_empty() { - match self.core.storage.data.write(batch.build()).await { - Ok(_) => { - changes.log_update(Collection::Mailbox, document_id); - } - Err(err) if err.is_assertion_failure() => { - ctx.response.not_updated.append(id, SetError::forbidden().with_description( - "Another process modified this mailbox, please try again.", - )); - continue 'update; - } - Err(err) => { - return Err(err.caused_by(trc::location!())); - } - } - } - ctx.response.updated.append(id, None); + .caused_by(trc::location!())? + .commit_point(); + will_update.push(id); } Err(err) => { ctx.response.not_updated.append(id, err); @@ -238,21 +205,46 @@ impl MailboxSet for Server { } } + if !batch.is_empty() { + let change_id_ = batch.change_id(); + match self.commit_batch(batch).await { + Ok(_) => { + change_id = Some(change_id_); + for id in will_update { + ctx.response.updated.append(id, None); + } + } + Err(err) if err.is_assertion_failure() => { + for id in will_update { + ctx.response.not_updated.append( + id, + SetError::forbidden().with_description( + "Another process modified this mailbox, please try again.", + ), + ); + } + } + Err(err) => { + return Err(err.caused_by(trc::location!())); + } + } + } + // Process deletions - let mut did_remove_emails = false; for id in ctx.will_destroy { match self .mailbox_destroy( account_id, id.document_id(), - &mut changes, ctx.access_token, on_destroy_remove_emails, ) .await? { - Ok(removed_emails) => { - did_remove_emails |= removed_emails; + Ok(change_id_) => { + if change_id_.is_some() { + change_id = change_id_; + } ctx.response.destroyed.push(id); } Err(err) => { @@ -262,18 +254,8 @@ impl MailboxSet for Server { } // Write changes - if !changes.is_empty() { - let state_change = - StateChange::new(account_id).with_change(DataType::Mailbox, changes.change_id); - ctx.response.state_change = if did_remove_emails { - state_change - .with_change(DataType::Email, changes.change_id) - .with_change(DataType::Thread, changes.change_id) - } else { - state_change - } - .into(); - ctx.response.new_state = Some(self.commit_changes(account_id, changes).await?.into()); + if let Some(change_id) = change_id { + ctx.response.new_state = State::Exact(change_id).into(); } Ok(ctx.response) @@ -408,12 +390,7 @@ impl MailboxSet for Server { let parent_document_id = mailbox_parent_id - 1; if let Some(mailbox_) = self - .get_property::>( - ctx.account_id, - Collection::Mailbox, - parent_document_id, - Property::Value, - ) + .get_archive(ctx.account_id, Collection::Mailbox, parent_document_id) .await? { let mailbox = mailbox_ diff --git a/crates/jmap/src/push/get.rs b/crates/jmap/src/push/get.rs index b86555be..eb4a8b5e 100644 --- a/crates/jmap/src/push/get.rs +++ b/crates/jmap/src/push/get.rs @@ -85,12 +85,7 @@ impl PushSubscriptionFetch for Server { continue; } let push_ = if let Some(push) = self - .get_property::>( - account_id, - Collection::PushSubscription, - document_id, - Property::Value, - ) + .get_archive(account_id, Collection::PushSubscription, document_id) .await? { push diff --git a/crates/jmap/src/push/set.rs b/crates/jmap/src/push/set.rs index dd85b343..095b4761 100644 --- a/crates/jmap/src/push/set.rs +++ b/crates/jmap/src/push/set.rs @@ -15,6 +15,7 @@ use jmap_proto::{ collection::Collection, date::UTCDate, property::Property, + state::State, type_state::DataType, value::{MaybePatchValue, Object, Value}, }, @@ -24,7 +25,7 @@ use std::future::Future; use store::{ Serialize, rand::{Rng, rng}, - write::{AlignedBytes, Archive, Archiver, BatchBuilder, now}, + write::{Archiver, BatchBuilder, now}, }; use trc::AddContext; use utils::map::bitmap::Bitmap; @@ -49,7 +50,7 @@ impl PushSubscriptionSet for Server { access_token: &AccessToken, ) -> trc::Result { let account_id = access_token.primary_id(); - let mut push_ids = self + let push_ids = self .get_document_ids(account_id, Collection::PushSubscription) .await? .unwrap_or_default(); @@ -57,6 +58,7 @@ impl PushSubscriptionSet for Server { let will_destroy = request.unwrap_destroy(); // Process creates + let mut batch = BatchBuilder::new(); 'create: for (id, object) in request.unwrap_create() { let mut push = PushSubscription::default(); @@ -101,23 +103,22 @@ impl PushSubscriptionSet for Server { .collect::(); // Insert record - let mut batch = BatchBuilder::new(); + let document_id = self + .store() + .assign_document_ids(account_id, Collection::PushSubscription, 1) + .await + .caused_by(trc::location!())?; batch .with_account_id(account_id) .with_collection(Collection::PushSubscription) - .create_document() + .create_document(document_id) .set( Property::Value, Archiver::new(push) .serialize() .caused_by(trc::location!())?, - ); - let document_id = self - .store() - .write_expect_id(batch) - .await - .caused_by(trc::location!())?; - push_ids.insert(document_id); + ) + .commit_point(); response.created.insert( id, Object::with_capacity(1) @@ -138,12 +139,7 @@ impl PushSubscriptionSet for Server { // Obtain push subscription let document_id = id.document_id(); let mut push = if let Some(push) = self - .get_property::>( - account_id, - Collection::PushSubscription, - document_id, - Property::Value, - ) + .get_archive(account_id, Collection::PushSubscription, document_id) .await? { push.deserialize::() @@ -164,7 +160,6 @@ impl PushSubscriptionSet for Server { } // Update record - let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::PushSubscription) @@ -174,11 +169,8 @@ impl PushSubscriptionSet for Server { Archiver::new(push) .serialize() .caused_by(trc::location!())?, - ); - self.store() - .write(batch) - .await - .caused_by(trc::location!())?; + ) + .commit_point(); response.updated.append(id, None); } @@ -187,22 +179,25 @@ impl PushSubscriptionSet for Server { let document_id = id.document_id(); if push_ids.contains(document_id) { // Update record - let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::PushSubscription) .delete_document(document_id) - .clear(Property::Value); - self.store() - .write(batch) - .await - .caused_by(trc::location!())?; + .clear(Property::Value) + .commit_point(); response.destroyed.push(id); } else { response.not_destroyed.append(id, SetError::not_found()); } } + // Write changes + if !batch.is_empty() { + let change_id = batch.change_id(); + self.commit_batch(batch).await.caused_by(trc::location!())?; + response.new_state = State::Exact(change_id).into(); + } + // Update push subscriptions if response.has_changes() { self.update_push_subscriptions(account_id).await; diff --git a/crates/jmap/src/sieve/get.rs b/crates/jmap/src/sieve/get.rs index f597d9df..7639732b 100644 --- a/crates/jmap/src/sieve/get.rs +++ b/crates/jmap/src/sieve/get.rs @@ -15,10 +15,7 @@ use jmap_proto::{ value::{Object, Value}, }, }; -use store::{ - BlobClass, - write::{AlignedBytes, Archive}, -}; +use store::BlobClass; use trc::AddContext; use crate::changes::state::StateManager; @@ -76,12 +73,7 @@ impl SieveScriptGet for Server { continue; } let sieve_ = if let Some(sieve) = self - .get_property::>( - account_id, - Collection::SieveScript, - document_id, - Property::Value, - ) + .get_archive(account_id, Collection::SieveScript, document_id) .await? { sieve diff --git a/crates/jmap/src/sieve/query.rs b/crates/jmap/src/sieve/query.rs index 87eb126e..a9b24ff4 100644 --- a/crates/jmap/src/sieve/query.rs +++ b/crates/jmap/src/sieve/query.rs @@ -12,10 +12,7 @@ use jmap_proto::{ types::{collection::Collection, property::Property}, }; use std::future::Future; -use store::{ - SerializeInfallible, - query::{self}, -}; +use store::query::{self}; use crate::JmapMethods; @@ -37,10 +34,9 @@ impl SieveScriptQuery for Server { for cond in std::mem::take(&mut request.filter) { match cond { Filter::Name(name) => filters.push(query::Filter::contains(Property::Name, &name)), - Filter::IsActive(is_active) => filters.push(query::Filter::eq( - Property::IsActive, - (is_active as u32).serialize(), - )), + Filter::IsActive(is_active) => { + filters.push(query::Filter::eq(Property::IsActive, vec![is_active as u8])) + } Filter::And | Filter::Or | Filter::Not | Filter::Close => { filters.push(cond.into()); } diff --git a/crates/jmap/src/sieve/set.rs b/crates/jmap/src/sieve/set.rs index 03ade972..787eb4fb 100644 --- a/crates/jmap/src/sieve/set.rs +++ b/crates/jmap/src/sieve/set.rs @@ -24,6 +24,7 @@ use jmap_proto::{ collection::Collection, id::Id, property::Property, + state::State, value::{MaybePatchValue, Object, SetValue, Value}, }, }; @@ -33,7 +34,7 @@ use store::{ BlobClass, Serialize, query::Filter, rand::{Rng, rng}, - write::{AlignedBytes, Archive, BatchBuilder, LegacyBincode, log::ChangeLogBuilder}, + write::{Archive, BatchBuilder, LegacyBincode}, }; use trc::AddContext; @@ -82,7 +83,7 @@ impl SieveScriptSet for Server { session: &HttpSessionData, ) -> trc::Result { let account_id = request.account_id.document_id(); - let mut sieve_ids = self + let sieve_ids = self .get_document_ids(account_id, Collection::SieveScript) .await? .unwrap_or_default(); @@ -96,7 +97,7 @@ impl SieveScriptSet for Server { let will_destroy = request.unwrap_destroy(); // Process creates - let mut changes = ChangeLogBuilder::new(); + let mut batch = BatchBuilder::new(); for (id, object) in request.unwrap_create() { if sieve_ids.len() as usize <= self.core.jmap.sieve_max_scripts { match self @@ -111,21 +112,18 @@ impl SieveScriptSet for Server { let blob_hash = sieve.blob_hash.clone(); // Write record - let mut batch = BatchBuilder::new(); + let document_id = self + .store() + .assign_document_ids(account_id, Collection::SieveScript, 1) + .await + .caused_by(trc::location!())?; batch .with_account_id(account_id) .with_collection(Collection::SieveScript) - .create_document() + .create_document(document_id) .custom(builder.with_tenant_id(&ctx.resource_token)) - .caused_by(trc::location!())?; - - let document_id = self - .store() - .write_expect_id(batch) - .await - .caused_by(trc::location!())?; - sieve_ids.insert(document_id); - changes.log_insert(Collection::SieveScript, document_id); + .caused_by(trc::location!())? + .commit_point(); // Add result with updated blobId ctx.response.created.insert( @@ -179,12 +177,7 @@ impl SieveScriptSet for Server { // Obtain sieve script let document_id = id.document_id(); if let Some(sieve_) = self - .get_property::>( - account_id, - Collection::SieveScript, - document_id, - Property::Value, - ) + .get_archive(account_id, Collection::SieveScript, document_id) .await? { let sieve = sieve_ @@ -202,7 +195,6 @@ impl SieveScriptSet for Server { { Ok((mut builder, blob)) => { // Prepare write batch - let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::SieveScript) @@ -234,23 +226,8 @@ impl SieveScriptSet for Server { // Write record batch .custom(builder.with_tenant_id(&ctx.resource_token)) - .caused_by(trc::location!())?; - - if !batch.is_empty() { - changes.log_update(Collection::SieveScript, document_id); - match self.core.storage.data.write(batch.build()).await { - Ok(_) => (), - Err(err) if err.is_assertion_failure() => { - ctx.response.not_updated.append(id, SetError::forbidden().with_description( - "Another process modified this sieve, please try again.", - )); - continue 'update; - } - Err(err) => { - return Err(err.caused_by(trc::location!())); - } - } - } + .caused_by(trc::location!())? + .commit_point(); // Add result with updated blobId ctx.response.updated.append( @@ -274,24 +251,36 @@ impl SieveScriptSet for Server { for id in will_destroy { let document_id = id.document_id(); if sieve_ids.contains(document_id) { - if self - .sieve_script_delete(&ctx.resource_token, document_id, true) + match self + .sieve_script_delete(&ctx.resource_token, document_id, true, &mut batch) .await? { - changes.log_delete(Collection::SieveScript, document_id); - ctx.response.destroyed.push(id); - } else { - ctx.response.not_destroyed.append( - id, - SetError::new(SetErrorType::ScriptIsActive) - .with_description("Deactivate Sieve script before deletion."), - ); + Some(true) => { + ctx.response.destroyed.push(id); + } + Some(false) => { + ctx.response.not_destroyed.append( + id, + SetError::new(SetErrorType::ScriptIsActive) + .with_description("Deactivate Sieve script before deletion."), + ); + } + None => { + ctx.response.not_destroyed.append(id, SetError::not_found()); + } } } else { ctx.response.not_destroyed.append(id, SetError::not_found()); } } + // Write changes + if !batch.is_empty() { + let change_id = batch.change_id(); + self.commit_batch(batch).await.caused_by(trc::location!())?; + ctx.response.new_state = State::Exact(change_id).into(); + } + // Activate / deactivate scripts if ctx.response.not_created.is_empty() && ctx.response.not_updated.is_empty() @@ -302,7 +291,9 @@ impl SieveScriptSet for Server { .on_success_deactivate_script .unwrap_or(false)) { - let changed_ids = if let Some(id) = request.arguments.on_success_activate_script { + let (change_id, changed_ids) = if let Some(id) = + request.arguments.on_success_activate_script + { self.sieve_activate_script( account_id, match id { @@ -319,19 +310,18 @@ impl SieveScriptSet for Server { self.sieve_activate_script(account_id, None).await? }; - for (document_id, is_active) in changed_ids { - if let Some(obj) = ctx.response.get_object_by_id(Id::from(document_id)) { - obj.append(Property::IsActive, Value::Bool(is_active)); + if !changed_ids.is_empty() { + for (document_id, is_active) in changed_ids { + if let Some(obj) = ctx.response.get_object_by_id(Id::from(document_id)) { + obj.append(Property::IsActive, Value::Bool(is_active)); + } + } + if change_id > 0 { + ctx.response.new_state = State::Exact(change_id).into(); } - changes.log_update(Collection::SieveScript, document_id); } } - // Write changes - if !changes.is_empty() { - ctx.response.new_state = Some(self.commit_changes(account_id, changes).await?.into()); - } - Ok(ctx.response) } diff --git a/crates/jmap/src/submission/get.rs b/crates/jmap/src/submission/get.rs index a8995828..c43d08ab 100644 --- a/crates/jmap/src/submission/get.rs +++ b/crates/jmap/src/submission/get.rs @@ -21,10 +21,7 @@ use jmap_proto::{ use smtp::queue::{ArchivedStatus, Message, spool::SmtpSpool}; use smtp_proto::ArchivedResponse; use std::future::Future; -use store::{ - rkyv::option::ArchivedOption, - write::{AlignedBytes, Archive}, -}; +use store::rkyv::option::ArchivedOption; use trc::AddContext; use utils::map::vec_map::VecMap; @@ -87,12 +84,7 @@ impl EmailSubmissionGet for Server { continue; } let submission_ = if let Some(submission) = self - .get_property::>( - account_id, - Collection::EmailSubmission, - document_id, - Property::Value, - ) + .get_archive(account_id, Collection::EmailSubmission, document_id) .await? { submission diff --git a/crates/jmap/src/submission/set.rs b/crates/jmap/src/submission/set.rs index a1975cb7..7b6a727e 100644 --- a/crates/jmap/src/submission/set.rs +++ b/crates/jmap/src/submission/set.rs @@ -30,15 +30,16 @@ use jmap_proto::{ collection::Collection, id::Id, property::Property, + state::State, value::{MaybePatchValue, Object, SetValue, Value}, }, }; use smtp::{ - core::{Session, SessionData, State}, + core::{Session, SessionData}, queue::spool::SmtpSpool, }; use smtp_proto::{MailFrom, RcptTo, request::parser::Rfc5321Parser}; -use store::write::{AlignedBytes, Archive, BatchBuilder, log::ChangeLogBuilder, now}; +use store::write::{BatchBuilder, now}; use trc::AddContext; use utils::{BlobHash, map::vec_map::VecMap, sanitize_email}; @@ -74,8 +75,8 @@ impl EmailSubmissionSet for Server { let will_destroy = request.unwrap_destroy(); // Process creates - let mut changes = ChangeLogBuilder::new(); let mut success_email_ids = HashMap::new(); + let mut batch = BatchBuilder::new(); for (id, object) in request.unwrap_create() { match self .send_message(account_id, &response, instance, object) @@ -89,19 +90,18 @@ impl EmailSubmissionSet for Server { ); // Insert record - let mut batch = BatchBuilder::new(); + let document_id = self + .store() + .assign_document_ids(account_id, Collection::EmailSubmission, 1) + .await + .caused_by(trc::location!())?; batch .with_account_id(account_id) .with_collection(Collection::EmailSubmission) - .create_document() + .create_document(document_id) .custom(ObjectIndexBuilder::<(), _>::new().with_changes(submission)) - .caused_by(trc::location!())?; - let document_id = self - .store() - .write_expect_id(batch) - .await - .caused_by(trc::location!())?; - changes.log_insert(Collection::EmailSubmission, document_id); + .caused_by(trc::location!())? + .commit_point(); response.created(id, document_id); } Err(err) => { @@ -121,12 +121,7 @@ impl EmailSubmissionSet for Server { // Obtain submission let document_id = id.document_id(); let submission = if let Some(submission) = self - .get_property::>( - account_id, - Collection::EmailSubmission, - document_id, - Property::Value, - ) + .get_archive(account_id, Collection::EmailSubmission, document_id) .await? { submission @@ -177,7 +172,6 @@ impl EmailSubmissionSet for Server { // Update record let mut new_submission = submission.inner.clone(); new_submission.undo_status = UndoStatus::Canceled; - let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::EmailSubmission) @@ -187,12 +181,8 @@ impl EmailSubmissionSet for Server { .with_current(submission) .with_changes(new_submission), ) - .caused_by(trc::location!())?; - self.store() - .write(batch) - .await - .caused_by(trc::location!())?; - changes.log_update(Collection::EmailSubmission, document_id); + .caused_by(trc::location!())? + .commit_point(); response.updated.append(id, None); } else { response.not_updated.append( @@ -225,16 +215,10 @@ impl EmailSubmissionSet for Server { for id in will_destroy { let document_id = id.document_id(); if let Some(submission) = self - .get_property::>( - account_id, - Collection::EmailSubmission, - document_id, - Property::Value, - ) + .get_archive(account_id, Collection::EmailSubmission, document_id) .await? { // Update record - let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::EmailSubmission) @@ -246,12 +230,8 @@ impl EmailSubmissionSet for Server { .caused_by(trc::location!())?, ), ) - .caused_by(trc::location!())?; - self.store() - .write(batch) - .await - .caused_by(trc::location!())?; - changes.log_delete(Collection::EmailSubmission, document_id); + .caused_by(trc::location!())? + .commit_point(); response.destroyed.push(id); } else { response.not_destroyed.append(id, SetError::not_found()); @@ -259,8 +239,10 @@ impl EmailSubmissionSet for Server { } // Write changes - if !changes.is_empty() { - response.new_state = Some(self.commit_changes(account_id, changes).await?.into()); + if !batch.is_empty() { + let change_id = batch.change_id(); + self.commit_batch(batch).await.caused_by(trc::location!())?; + response.new_state = State::Exact(change_id).into(); } // On success @@ -458,12 +440,7 @@ impl EmailSubmissionSet for Server { // Fetch identity's mailFrom let identity_mail_from = if let Some(identity) = self - .get_property::>( - account_id, - Collection::Identity, - submission.identity_id, - Property::Value, - ) + .get_archive(account_id, Collection::Identity, submission.identity_id) .await? { identity @@ -499,7 +476,7 @@ impl EmailSubmissionSet for Server { // Obtain message metadata let metadata_ = if let Some(metadata) = self - .get_property::>( + .get_archive_by_property( account_id, Collection::Email, submission.email_id, @@ -596,8 +573,19 @@ impl EmailSubmissionSet for Server { } // Begin local SMTP session - let mut session = - Session::::local(self.clone(), instance.clone(), SessionData::default()); + let mut session = Session::::local( + self.clone(), + instance.clone(), + SessionData::local( + Box::pin(self.get_access_token(account_id)) + .await + .caused_by(trc::location!())?, + None, + vec![], + vec![], + 0, + ), + ); // MAIL FROM let _ = Box::pin(session.handle_mail_from(mail_from)).await; @@ -626,7 +614,7 @@ impl EmailSubmissionSet for Server { if has_success { session.data.message = message; let response = Box::pin(session.queue_message()).await; - if let State::Accepted(queue_id) = session.state { + if let smtp::core::State::Accepted(queue_id) = session.state { submission.queue_id = Some(queue_id); } else { return Ok(Err(SetError::new(SetErrorType::ForbiddenToSend) diff --git a/crates/jmap/src/thread/get.rs b/crates/jmap/src/thread/get.rs index 40c49d6d..a2a82f45 100644 --- a/crates/jmap/src/thread/get.rs +++ b/crates/jmap/src/thread/get.rs @@ -5,12 +5,17 @@ */ use common::Server; +use email::thread::cache::ThreadCache; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, types::{collection::Collection, id::Id, property::Property, value::Object}, }; use std::future::Future; -use store::query::{Comparator, ResultSet, sort::Pagination}; +use store::{ + ahash::AHashMap, + query::{Comparator, ResultSet, sort::Pagination}, + roaring::RoaringBitmap, +}; use trc::AddContext; use crate::changes::state::StateManager; @@ -28,13 +33,25 @@ impl ThreadGet for Server { mut request: GetRequest, ) -> trc::Result { let account_id = request.account_id.document_id(); + let mut thread_map: AHashMap = AHashMap::with_capacity(32); + for (document_id, thread_id) in &self + .get_cached_thread_ids(account_id) + .await + .caused_by(trc::location!())? + .threads + { + thread_map + .entry(*thread_id) + .or_default() + .insert(*document_id); + } + let ids = if let Some(ids) = request.unwrap_ids(self.core.jmap.get_max_objects)? { ids } else { - self.get_document_ids(account_id, Collection::Thread) - .await? - .unwrap_or_default() - .into_iter() + thread_map + .keys() + .copied() .take(self.core.jmap.get_max_objects) .map(Into::into) .collect() @@ -51,21 +68,19 @@ impl ThreadGet for Server { for id in ids { let thread_id = id.document_id(); - if let Some(document_ids) = self - .get_tag(account_id, Collection::Email, Property::ThreadId, thread_id) - .await? - { + if let Some(document_ids) = thread_map.remove(&thread_id) { let mut thread = Object::with_capacity(2).with_property(Property::Id, id); if add_email_ids { + let doc_count = document_ids.len() as usize; thread.append( Property::EmailIds, self.core .storage .data .sort( - ResultSet::new(account_id, Collection::Email, document_ids.clone()), + ResultSet::new(account_id, Collection::Email, document_ids), vec![Comparator::ascending(Property::ReceivedAt)], - Pagination::new(document_ids.len() as usize, 0, None, 0), + Pagination::new(doc_count, 0, None, 0), ) .await .caused_by(trc::location!())? diff --git a/crates/jmap/src/vacation/get.rs b/crates/jmap/src/vacation/get.rs index c7880ff0..a60dbc0d 100644 --- a/crates/jmap/src/vacation/get.rs +++ b/crates/jmap/src/vacation/get.rs @@ -19,10 +19,7 @@ use jmap_proto::{ }, }; use std::future::Future; -use store::{ - query::Filter, - write::{AlignedBytes, Archive}, -}; +use store::query::Filter; use trc::AddContext; use crate::{JmapMethods, changes::state::StateManager}; @@ -84,12 +81,7 @@ impl VacationResponseGet for Server { if do_get { if let Some(document_id) = self.get_vacation_sieve_script_id(account_id).await? { if let Some(sieve_) = self - .get_property::>( - account_id, - Collection::SieveScript, - document_id, - Property::Value, - ) + .get_archive(account_id, Collection::SieveScript, document_id) .await? { let sieve = sieve_ diff --git a/crates/jmap/src/vacation/set.rs b/crates/jmap/src/vacation/set.rs index 58df21b4..91064522 100644 --- a/crates/jmap/src/vacation/set.rs +++ b/crates/jmap/src/vacation/set.rs @@ -27,10 +27,7 @@ use mail_parser::decoders::html::html_to_text; use std::future::Future; use store::{ Serialize, - write::{ - AlignedBytes, Archive, BatchBuilder, LegacyBincode, - log::{Changes, LogInsert}, - }, + write::{BatchBuilder, LegacyBincode}, }; use trc::AddContext; @@ -119,9 +116,7 @@ impl VacationResponseSet for Server { // Prepare write batch let mut batch = BatchBuilder::new(); - let change_id = self.assign_change_id(account_id)?; batch - .with_change_id(change_id) .with_account_id(account_id) .with_collection(Collection::SieveScript); @@ -133,12 +128,7 @@ impl VacationResponseSet for Server { let (mut sieve, prev_sieve) = if let Some(document_id) = document_id { let prev_sieve = self - .get_property::>( - account_id, - Collection::SieveScript, - document_id, - Property::Value, - ) + .get_archive(account_id, Collection::SieveScript, document_id) .await? .ok_or_else(|| { trc::StoreEvent::NotFound @@ -261,14 +251,18 @@ impl VacationResponseSet for Server { .with_tenant_id(&resource_token); // Update id - if let Some(document_id) = document_id { - batch - .update_document(document_id) - .clear(Property::EmailIds) - .log(Changes::update([document_id])); + let document_id = if let Some(document_id) = document_id { + batch.update_document(document_id); + document_id } else { - batch.create_document().log(LogInsert()); - } + let document_id = self + .store() + .assign_document_ids(account_id, Collection::SieveScript, 1) + .await + .caused_by(trc::location!())?; + batch.create_document(document_id); + document_id + }; // Create sieve script only if there are changes if build_script { @@ -285,25 +279,19 @@ impl VacationResponseSet for Server { // Write changes batch.custom(obj).caused_by(trc::location!())?; - let document_id = if !batch.is_empty() { - let ids = self - .store() - .write(batch) - .await - .caused_by(trc::location!())?; - response.new_state = Some(change_id.into()); - match document_id { - Some(document_id) => document_id, - None => ids.last_document_id()?, - } - } else { - document_id.unwrap_or(u32::MAX) - }; + if !batch.is_empty() { + response.new_state = Some(batch.change_id().into()); + self.commit_batch(batch).await.caused_by(trc::location!())?; + } // Deactivate other sieve scripts if !was_active && is_active { - self.sieve_activate_script(account_id, document_id.into()) + let (change_id, _) = self + .sieve_activate_script(account_id, document_id.into()) .await?; + if change_id > 0 { + response.new_state = Some(change_id.into()); + } } // Add result @@ -320,9 +308,8 @@ impl VacationResponseSet for Server { if id.is_singleton() { if let Some(document_id) = self.get_vacation_sieve_script_id(account_id).await? { - self.sieve_script_delete(&resource_token, document_id, false) + self.sieve_script_delete(&resource_token, document_id, false, &mut batch) .await?; - batch.log(Changes::delete([document_id])); response.destroyed.push(id); continue; } @@ -333,11 +320,8 @@ impl VacationResponseSet for Server { // Write changes if !batch.is_empty() { - self.store() - .write(batch) - .await - .caused_by(trc::location!())?; - response.new_state = Some(change_id.into()); + response.new_state = Some(batch.change_id().into()); + self.commit_batch(batch).await.caused_by(trc::location!())?; } } diff --git a/crates/main/src/main.rs b/crates/main/src/main.rs index 601fdab6..8aea73df 100644 --- a/crates/main/src/main.rs +++ b/crates/main/src/main.rs @@ -7,7 +7,6 @@ use std::time::Duration; use common::{config::server::ServerProtocol, core::BuildServer, manager::boot::BootManager}; -use directory::backend::internal::MigrateDirectory; use http::HttpSessionManager; use imap::core::ImapSessionManager; use managesieve::core::ManageSieveSessionManager; @@ -44,12 +43,6 @@ async fn main() -> std::io::Result<()> { // Log licensing information #[cfg(feature = "enterprise")] server.log_license_details(); - - // Migrate directory - if let Err(err) = server.store().migrate_directory().await { - trc::error!(err.details("Directory migration failed")); - std::process::exit(1); - } } // Spawn servers diff --git a/crates/managesieve/src/op/deletescript.rs b/crates/managesieve/src/op/deletescript.rs index b13ae6f6..86954232 100644 --- a/crates/managesieve/src/op/deletescript.rs +++ b/crates/managesieve/src/op/deletescript.rs @@ -10,8 +10,7 @@ use common::listener::SessionStream; use directory::Permission; use email::sieve::delete::SieveScriptDelete; use imap_proto::receiver::Request; -use jmap_proto::types::collection::Collection; -use store::write::log::ChangeLogBuilder; +use store::write::BatchBuilder; use trc::AddContext; use crate::core::{Command, ResponseCode, Session, StatusResponse}; @@ -37,34 +36,44 @@ impl Session { let access_token = self.state.access_token(); let account_id = access_token.primary_id(); let document_id = self.get_script_id(account_id, &name).await?; - if self + let mut batch = BatchBuilder::new(); + + match self .server - .sieve_script_delete(&access_token.as_resource_token(), document_id, true) + .sieve_script_delete( + &access_token.as_resource_token(), + document_id, + true, + &mut batch, + ) .await .caused_by(trc::location!())? { - // Write changes - let mut changelog = ChangeLogBuilder::new(); - changelog.log_delete(Collection::SieveScript, document_id); - self.server - .commit_changes(account_id, changelog) - .await - .caused_by(trc::location!())?; + Some(true) => { + if !batch.is_empty() { + self.server + .commit_batch(batch) + .await + .caused_by(trc::location!())?; + } - trc::event!( - ManageSieve(trc::ManageSieveEvent::DeleteScript), - SpanId = self.session_id, - Id = name, - DocumentId = document_id, - Elapsed = op_start.elapsed() - ); + trc::event!( + ManageSieve(trc::ManageSieveEvent::DeleteScript), + SpanId = self.session_id, + Id = name, + DocumentId = document_id, + Elapsed = op_start.elapsed() + ); - Ok(StatusResponse::ok("Deleted.").into_bytes()) - } else { - Err(trc::ManageSieveEvent::Error + Ok(StatusResponse::ok("Deleted.").into_bytes()) + } + Some(false) => Err(trc::ManageSieveEvent::Error .into_err() .details("You may not delete an active script") - .code(ResponseCode::Active)) + .code(ResponseCode::Active)), + None => Err(trc::ManageSieveEvent::Error + .into_err() + .details("Script not found")), } } } diff --git a/crates/managesieve/src/op/getscript.rs b/crates/managesieve/src/op/getscript.rs index 7ba58784..22984f4b 100644 --- a/crates/managesieve/src/op/getscript.rs +++ b/crates/managesieve/src/op/getscript.rs @@ -10,8 +10,7 @@ use common::listener::SessionStream; use directory::Permission; use email::sieve::SieveScript; use imap_proto::receiver::Request; -use jmap_proto::types::{blob::BlobSection, collection::Collection, property::Property}; -use store::write::{AlignedBytes, Archive}; +use jmap_proto::types::{blob::BlobSection, collection::Collection}; use trc::AddContext; use utils::BlobHash; @@ -37,12 +36,7 @@ impl Session { let document_id = self.get_script_id(account_id, &name).await?; let sieve_ = self .server - .get_property::>( - account_id, - Collection::SieveScript, - document_id, - Property::Value, - ) + .get_archive(account_id, Collection::SieveScript, document_id) .await .caused_by(trc::location!())? .ok_or_else(|| { diff --git a/crates/managesieve/src/op/listscripts.rs b/crates/managesieve/src/op/listscripts.rs index 581aa399..93b16b1a 100644 --- a/crates/managesieve/src/op/listscripts.rs +++ b/crates/managesieve/src/op/listscripts.rs @@ -9,8 +9,7 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; use email::sieve::SieveScript; -use jmap_proto::types::{collection::Collection, property::Property}; -use store::write::{AlignedBytes, Archive}; +use jmap_proto::types::collection::Collection; use trc::AddContext; use crate::core::{Session, StatusResponse}; @@ -39,12 +38,7 @@ impl Session { for document_id in document_ids { if let Some(script_) = self .server - .get_property::>( - account_id, - Collection::SieveScript, - document_id, - Property::Value, - ) + .get_archive(account_id, Collection::SieveScript, document_id) .await .caused_by(trc::location!())? { diff --git a/crates/managesieve/src/op/putscript.rs b/crates/managesieve/src/op/putscript.rs index 096b48e2..87c7d80d 100644 --- a/crates/managesieve/src/op/putscript.rs +++ b/crates/managesieve/src/op/putscript.rs @@ -15,7 +15,7 @@ use sieve::compiler::ErrorType; use store::{ Serialize, query::Filter, - write::{AlignedBytes, Archive, BatchBuilder, LegacyBincode, log::LogInsert}, + write::{BatchBuilder, LegacyBincode}, }; use trc::AddContext; @@ -105,12 +105,7 @@ impl Session { // Obtain script values let script_ = self .server - .get_property::>( - account_id, - Collection::SieveScript, - document_id, - Property::Value, - ) + .get_archive(account_id, Collection::SieveScript, document_id) .await .caused_by(trc::location!())? .ok_or_else(|| { @@ -152,8 +147,7 @@ impl Session { .caused_by(trc::location!())?; self.server - .store() - .write(batch) + .commit_batch(batch) .await .caused_by(trc::location!())?; @@ -175,11 +169,16 @@ impl Session { // Write record let mut batch = BatchBuilder::new(); + let document_id = self + .server + .store() + .assign_document_ids(account_id, Collection::SieveScript, 1) + .await + .caused_by(trc::location!())?; batch .with_account_id(account_id) .with_collection(Collection::SieveScript) - .create_document() - .log(LogInsert()) + .create_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes( @@ -191,10 +190,8 @@ impl Session { ) .caused_by(trc::location!())?; - let assigned_ids = self - .server - .store() - .write(batch) + self.server + .commit_batch(batch) .await .caused_by(trc::location!())?; @@ -202,7 +199,7 @@ impl Session { ManageSieve(trc::ManageSieveEvent::CreateScript), SpanId = self.session_id, Id = name, - DocumentId = assigned_ids.last_document_id().ok(), + DocumentId = document_id, Elapsed = op_start.elapsed() ); } diff --git a/crates/managesieve/src/op/renamescript.rs b/crates/managesieve/src/op/renamescript.rs index f1008e68..4b3277e6 100644 --- a/crates/managesieve/src/op/renamescript.rs +++ b/crates/managesieve/src/op/renamescript.rs @@ -10,8 +10,8 @@ use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; use email::sieve::SieveScript; use imap_proto::receiver::Request; -use jmap_proto::types::{collection::Collection, property::Property}; -use store::write::{AlignedBytes, Archive, BatchBuilder, log::ChangeLogBuilder}; +use jmap_proto::types::collection::Collection; +use store::write::BatchBuilder; use trc::AddContext; use crate::core::{Command, ResponseCode, Session, StatusResponse}; @@ -60,12 +60,7 @@ impl Session { // Obtain script values let script = self .server - .get_property::>( - account_id, - Collection::SieveScript, - document_id, - Property::Value, - ) + .get_archive(account_id, Collection::SieveScript, document_id) .await .caused_by(trc::location!())? .ok_or_else(|| { @@ -91,14 +86,7 @@ impl Session { .caused_by(trc::location!())?; if !batch.is_empty() { self.server - .store() - .write(batch) - .await - .caused_by(trc::location!())?; - let mut changelog = ChangeLogBuilder::new(); - changelog.log_update(Collection::SieveScript, document_id); - self.server - .commit_changes(account_id, changelog) + .commit_batch(batch) .await .caused_by(trc::location!())?; } diff --git a/crates/managesieve/src/op/setactive.rs b/crates/managesieve/src/op/setactive.rs index 9e7f0358..aedc58e3 100644 --- a/crates/managesieve/src/op/setactive.rs +++ b/crates/managesieve/src/op/setactive.rs @@ -10,8 +10,6 @@ use common::listener::SessionStream; use directory::Permission; use email::sieve::activate::SieveScriptActivate; use imap_proto::receiver::Request; -use jmap_proto::types::collection::Collection; -use store::write::log::ChangeLogBuilder; use trc::AddContext; use crate::core::{Command, Session, StatusResponse}; @@ -35,8 +33,7 @@ impl Session { // De/activate script let account_id = self.state.access_token().primary_id(); - let changes = self - .server + self.server .sieve_activate_script( account_id, if !name.is_empty() { @@ -48,18 +45,6 @@ impl Session { .await .caused_by(trc::location!())?; - // Write changes - if !changes.is_empty() { - let mut changelog = ChangeLogBuilder::new(); - for (document_id, _) in changes { - changelog.log_update(Collection::SieveScript, document_id); - } - self.server - .commit_changes(account_id, changelog) - .await - .caused_by(trc::location!())?; - } - trc::event!( ManageSieve(trc::ManageSieveEvent::SetActive), SpanId = self.session_id, diff --git a/crates/pop3/src/mailbox.rs b/crates/pop3/src/mailbox.rs index 447adeb4..de11c504 100644 --- a/crates/pop3/src/mailbox.rs +++ b/crates/pop3/src/mailbox.rs @@ -13,9 +13,8 @@ use email::{ }; use jmap_proto::types::{collection::Collection, property::Property}; use store::{ - IndexKey, IterateParams, SerializeInfallible, U32_LEN, - ahash::AHashMap, - write::{AlignedBytes, Archive, key::DeserializeBigEndian}, + IndexKey, IterateParams, SerializeInfallible, U32_LEN, ahash::AHashMap, + write::key::DeserializeBigEndian, }; use trc::AddContext; @@ -66,12 +65,7 @@ impl Session { .caused_by(trc::location!())?; let uid_validity = u32::from( self.server - .get_property::>( - account_id, - Collection::Mailbox, - INBOX_ID, - &Property::Value, - ) + .get_archive(account_id, Collection::Mailbox, INBOX_ID) .await .caused_by(trc::location!())? .ok_or_else(|| { @@ -130,7 +124,6 @@ impl Session { account_id, Collection::Email, &message_ids, - Property::Value, |message_id, uid_mailbox| { // Make sure the message is still in Inbox if let Some(item) = uid_mailbox diff --git a/crates/pop3/src/op/delete.rs b/crates/pop3/src/op/delete.rs index 73ebf36e..8a9297a8 100644 --- a/crates/pop3/src/op/delete.rs +++ b/crates/pop3/src/op/delete.rs @@ -9,8 +9,7 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; use email::message::delete::EmailDeletion; -use jmap_proto::types::{state::StateChange, type_state::DataType}; -use store::roaring::RoaringBitmap; +use store::{roaring::RoaringBitmap, write::BatchBuilder}; use trc::AddContext; use crate::{Session, State, protocol::response::Response}; @@ -87,27 +86,18 @@ impl Session { if !deleted.is_empty() { let num_deleted = deleted.len(); - let (changes, not_deleted) = self + let mut batch = BatchBuilder::new(); + let not_deleted = self .server - .emails_tombstone(mailbox.account_id, deleted) + .emails_tombstone(mailbox.account_id, &mut batch, deleted) .await .caused_by(trc::location!())?; - if !changes.is_empty() { - if let Ok(change_id) = self - .server - .commit_changes(mailbox.account_id, changes) + if !batch.is_empty() { + self.server + .commit_batch(batch) .await - { - self.server - .broadcast_state_change( - StateChange::new(mailbox.account_id) - .with_change(DataType::Email, change_id) - .with_change(DataType::Mailbox, change_id) - .with_change(DataType::Thread, change_id), - ) - .await; - } + .caused_by(trc::location!())?; } if not_deleted.is_empty() { self.write_ok(format!( diff --git a/crates/pop3/src/op/fetch.rs b/crates/pop3/src/op/fetch.rs index fd26bf8b..5b3382b0 100644 --- a/crates/pop3/src/op/fetch.rs +++ b/crates/pop3/src/op/fetch.rs @@ -10,7 +10,6 @@ use common::listener::SessionStream; use directory::Permission; use email::message::metadata::MessageMetadata; use jmap_proto::types::{collection::Collection, property::Property}; -use store::write::{AlignedBytes, Archive}; use trc::AddContext; use crate::{Session, protocol::response::Response}; @@ -27,11 +26,11 @@ impl Session { if let Some(message) = mailbox.messages.get(msg.saturating_sub(1) as usize) { if let Some(metadata_) = self .server - .get_property::>( + .get_archive_by_property( mailbox.account_id, Collection::Email, message.id, - &Property::BodyStructure, + Property::BodyStructure, ) .await .caused_by(trc::location!())? diff --git a/crates/services/src/index/mod.rs b/crates/services/src/index/mod.rs index 7679bfa4..37f27f12 100644 --- a/crates/services/src/index/mod.rs +++ b/crates/services/src/index/mod.rs @@ -20,7 +20,7 @@ use store::{ fts::index::FtsDocument, roaring::RoaringBitmap, write::{ - AlignedBytes, Archive, BatchBuilder, BlobOp, MaybeDynamicId, TaskQueueClass, ValueClass, + BatchBuilder, BlobOp, TaskQueueClass, ValueClass, key::{DeserializeBigEndian, KeySerializer}, now, }, @@ -81,7 +81,7 @@ pub trait Indexer: Sync + Send { impl Indexer for Server { async fn email_task_queued(&self, locked_seq_ids: &mut AHashMap) { - let from_key = ValueKey::> { + let from_key = ValueKey:: { account_id: 0, collection: 0, document_id: 0, @@ -90,7 +90,7 @@ impl Indexer for Server { hash: BlobHash::default(), }), }; - let to_key = ValueKey::> { + let to_key = ValueKey:: { account_id: u32::MAX, collection: u8::MAX, document_id: u32::MAX, @@ -166,7 +166,7 @@ impl Indexer for Server { match event.action { EmailTaskAction::Index => { match self - .get_property::>( + .get_archive_by_property( event.account_id, Collection::Email, event.document_id, @@ -275,7 +275,7 @@ impl Indexer for Server { .with_collection(Collection::Email) .update_document(event.document_id) .clear(event.value_class()) - .build_batch(), + .build_all(), ) .await { @@ -415,7 +415,7 @@ impl Indexer for Server { .await .caused_by(trc::location!())?; - let mut seq = self.generate_snowflake_id().caused_by(trc::location!())?; + let mut seq = self.generate_snowflake_id(); for (account_id, hashes) in hashes { let mut batch = BatchBuilder::new(); @@ -430,8 +430,8 @@ impl Indexer for Server { ); seq += 1; - if batch.ops.len() >= 2000 { - self.core.storage.data.write(batch.build()).await?; + if batch.len() >= 2000 { + self.core.storage.data.write(batch.build_all()).await?; batch = BatchBuilder::new(); batch .with_account_id(account_id) @@ -440,7 +440,7 @@ impl Indexer for Server { } if !batch.is_empty() { - self.core.storage.data.write(batch.build()).await?; + self.core.storage.data.write(batch.build_all()).await?; } } @@ -477,7 +477,7 @@ impl EmailTask { } } - fn value_class(&self) -> ValueClass { + fn value_class(&self) -> ValueClass { ValueClass::TaskQueue(match self.action { EmailTaskAction::Index => TaskQueueClass::IndexEmail { hash: self.hash.clone(), diff --git a/crates/smtp/src/core/mod.rs b/crates/smtp/src/core/mod.rs index 57814f0c..92d24a44 100644 --- a/crates/smtp/src/core/mod.rs +++ b/crates/smtp/src/core/mod.rs @@ -264,6 +264,7 @@ impl Session { impl SessionData { pub fn local( + authenticated_as: Arc, mail_from: Option, rcpt_to: Vec, message: Vec, @@ -284,7 +285,7 @@ impl SessionData { rcpt_errors: 0, rcpt_oks: 0, message, - authenticated_as: Some(Arc::new(AccessToken::from_id(0))), + authenticated_as: Some(authenticated_as), auth_errors: 0, priority: 0, delivery_by: 0, @@ -302,7 +303,7 @@ impl SessionData { impl Default for SessionData { fn default() -> Self { - Self::local(None, vec![], vec![], 0) + Self::local(Arc::new(AccessToken::from_id(0)), None, vec![], vec![], 0) } } diff --git a/crates/smtp/src/inbound/data.rs b/crates/smtp/src/inbound/data.rs index 9ce60a78..2c73762b 100644 --- a/crates/smtp/src/inbound/data.rs +++ b/crates/smtp/src/inbound/data.rs @@ -363,13 +363,7 @@ impl Session { } // Add Received header - let message_id = self - .server - .inner - .data - .queue_id_gen - .generate() - .unwrap_or_else(now); + let message_id = self.server.inner.data.queue_id_gen.generate(); let mut headers = Vec::with_capacity(64); if self .server diff --git a/crates/smtp/src/outbound/delivery.rs b/crates/smtp/src/outbound/delivery.rs index 3c814761..d5fd449b 100644 --- a/crates/smtp/src/outbound/delivery.rs +++ b/crates/smtp/src/outbound/delivery.rs @@ -50,7 +50,7 @@ impl QueuedMessage { let status = if server.try_lock_event(queue_id).await { if let Some(mut message) = server.read_message(queue_id).await { // Generate span id - message.span_id = server.inner.data.span_id_gen.generate().unwrap_or_else(now); + message.span_id = server.inner.data.span_id_gen.generate(); let span_id = message.span_id; trc::event!( @@ -104,7 +104,7 @@ impl QueuedMessage { }, ))); - if let Err(err) = server.store().write(batch.build()).await { + if let Err(err) = server.store().write(batch.build_all()).await { trc::error!( err.details("Failed to delete queue event.") .caused_by(trc::location!()) diff --git a/crates/smtp/src/queue/spool.rs b/crates/smtp/src/queue/spool.rs index 67b27d7d..7b894b60 100644 --- a/crates/smtp/src/queue/spool.rs +++ b/crates/smtp/src/queue/spool.rs @@ -61,7 +61,7 @@ impl SmtpSpool for Server { .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()); Message { - queue_id: self.inner.data.queue_id_gen.generate().unwrap_or(created), + queue_id: self.inner.data.queue_id_gen.generate(), span_id, created, return_path: return_path.into(), @@ -220,7 +220,7 @@ impl Message { }, 0u32.serialize(), ); - if let Err(err) = server.store().write(batch.build()).await { + if let Err(err) = server.store().write(batch.build_all()).await { trc::error!( err.details("Failed to write to store.") .span_id(session_id) @@ -326,7 +326,7 @@ impl Message { }, ); - if let Err(err) = server.store().write(batch.build()).await { + if let Err(err) = server.store().write(batch.build_all()).await { trc::error!( err.details("Failed to write to store.") .span_id(session_id) @@ -458,7 +458,7 @@ impl Message { }, ); - if let Err(err) = server.store().write(batch.build()).await { + if let Err(err) = server.store().write(batch.build_all()).await { trc::error!( err.details("Failed to save changes.") .span_id(span_id) @@ -501,7 +501,7 @@ impl Message { ))) .clear(ValueClass::Queue(QueueClass::Message(self.queue_id))); - if let Err(err) = server.store().write(batch.build()).await { + if let Err(err) = server.store().write(batch.build_all()).await { trc::error!( err.details("Failed to write to update queue.") .span_id(self.span_id) diff --git a/crates/smtp/src/reporting/analysis.rs b/crates/smtp/src/reporting/analysis.rs index a7cccdad..780121c5 100644 --- a/crates/smtp/src/reporting/analysis.rs +++ b/crates/smtp/src/reporting/analysis.rs @@ -274,7 +274,7 @@ impl AnalyzeReport for Server { // Store report if let Some(expires_in) = &core.core.smtp.report.analysis.store { let expires = now() + expires_in.as_secs(); - let id = core.inner.data.queue_id_gen.generate().unwrap_or(expires); + let id = core.inner.data.queue_id_gen.generate(); let mut batch = BatchBuilder::new(); match report { @@ -318,8 +318,7 @@ impl AnalyzeReport for Server { ); } } - let batch = batch.build(); - if let Err(err) = core.core.storage.data.write(batch).await { + if let Err(err) = core.core.storage.data.write(batch.build_all()).await { trc::error!( err.span_id(session_id) .caused_by(trc::location!()) diff --git a/crates/smtp/src/reporting/dmarc.rs b/crates/smtp/src/reporting/dmarc.rs index 79ab24e2..7807b86a 100644 --- a/crates/smtp/src/reporting/dmarc.rs +++ b/crates/smtp/src/reporting/dmarc.rs @@ -22,7 +22,7 @@ use mail_auth::{ }; use store::{ Deserialize, IterateParams, Serialize, ValueKey, - write::{BatchBuilder, LegacyBincode, QueueClass, ReportEvent, ValueClass, now}, + write::{BatchBuilder, LegacyBincode, QueueClass, ReportEvent, ValueClass}, }; use trc::{AddContext, OutgoingReportEvent}; use utils::config::Rate; @@ -320,7 +320,7 @@ pub trait DmarcReporting: Sync + Send { impl DmarcReporting for Server { async fn send_dmarc_aggregate_report(&self, event: ReportEvent) { - let span_id = self.inner.data.span_id_gen.generate().unwrap_or_else(now); + let span_id = self.inner.data.span_id_gen.generate(); trc::event!( OutgoingReport(OutgoingReportEvent::DmarcAggregateReport), @@ -606,7 +606,7 @@ impl DmarcReporting for Server { let mut batch = BatchBuilder::new(); batch.clear(ValueClass::Queue(QueueClass::DmarcReportHeader(event))); - if let Err(err) = self.core.storage.data.write(batch.build()).await { + if let Err(err) = self.core.storage.data.write(batch.build_all()).await { trc::error!( err.caused_by(trc::location!()) .details("Failed to delete DMARC report") @@ -664,7 +664,7 @@ impl DmarcReporting for Server { } // Write entry - report_event.seq_id = self.inner.data.queue_id_gen.generate().unwrap_or_else(now); + report_event.seq_id = self.inner.data.queue_id_gen.generate(); builder.set( ValueClass::Queue(QueueClass::DmarcReportEvent(report_event)), match LegacyBincode::new(event.report_record).serialize() { @@ -679,7 +679,7 @@ impl DmarcReporting for Server { }, ); - if let Err(err) = self.core.storage.data.write(builder.build()).await { + if let Err(err) = self.core.storage.data.write(builder.build_all()).await { trc::error!( err.caused_by(trc::location!()) .details("Failed to write DMARC report") diff --git a/crates/smtp/src/reporting/scheduler.rs b/crates/smtp/src/reporting/scheduler.rs index dd474b36..c37b37d1 100644 --- a/crates/smtp/src/reporting/scheduler.rs +++ b/crates/smtp/src/reporting/scheduler.rs @@ -170,7 +170,7 @@ async fn next_report_event(store: &Store) -> Vec { for event in old_locks { batch.clear(ValueClass::Queue(event)); } - if let Err(err) = store.write(batch.build()).await { + if let Err(err) = store.write(batch.build_all()).await { trc::error!( err.caused_by(trc::location!()) .details("Failed to remove old report events") diff --git a/crates/smtp/src/reporting/tls.rs b/crates/smtp/src/reporting/tls.rs index cbe2b041..bedc8962 100644 --- a/crates/smtp/src/reporting/tls.rs +++ b/crates/smtp/src/reporting/tls.rs @@ -28,7 +28,7 @@ use reqwest::header::CONTENT_TYPE; use std::fmt::Write; use store::{ Deserialize, IterateParams, Serialize, ValueKey, - write::{BatchBuilder, LegacyBincode, QueueClass, ReportEvent, ValueClass, now}, + write::{BatchBuilder, LegacyBincode, QueueClass, ReportEvent, ValueClass}, }; use trc::{AddContext, OutgoingReportEvent}; @@ -75,7 +75,7 @@ impl TlsReporting for Server { .map(|e| (e.domain.as_str(), e.seq_id, e.due)) .unwrap(); - let span_id = self.inner.data.span_id_gen.generate().unwrap_or_else(now); + let span_id = self.inner.data.span_id_gen.generate(); trc::event!( OutgoingReport(OutgoingReportEvent::TlsAggregate), @@ -505,7 +505,7 @@ impl TlsReporting for Server { } // Write entry - report_event.seq_id = self.inner.data.queue_id_gen.generate().unwrap_or_else(now); + report_event.seq_id = self.inner.data.queue_id_gen.generate(); builder.set( ValueClass::Queue(QueueClass::TlsReportEvent(report_event)), match LegacyBincode::new(event.failure).serialize() { @@ -520,7 +520,7 @@ impl TlsReporting for Server { }, ); - if let Err(err) = self.core.storage.data.write(builder.build()).await { + if let Err(err) = self.core.storage.data.write(builder.build_all()).await { trc::error!( err.caused_by(trc::location!()) .details("Failed to write TLS report") @@ -568,7 +568,7 @@ impl TlsReporting for Server { batch.clear(ValueClass::Queue(QueueClass::TlsReportHeader(event))); } - if let Err(err) = self.core.storage.data.write(batch.build()).await { + if let Err(err) = self.core.storage.data.write(batch.build_all()).await { trc::error!( err.caused_by(trc::location!()) .details("Failed to delete TLS reports") diff --git a/crates/store/src/backend/composite/read_replica.rs b/crates/store/src/backend/composite/read_replica.rs index 356c0eaa..b2a0aed5 100644 --- a/crates/store/src/backend/composite/read_replica.rs +++ b/crates/store/src/backend/composite/read_replica.rs @@ -194,7 +194,7 @@ impl SQLReadReplica { pub async fn get_bitmap( &self, - key: BitmapKey>, + key: BitmapKey, ) -> trc::Result> { self.run_op(move |store| { let key = key.clone(); @@ -242,7 +242,7 @@ impl SQLReadReplica { pub async fn get_counter( &self, - key: impl Into>> + Sync + Send, + key: impl Into> + Sync + Send, ) -> trc::Result { let key = key.into(); self.run_op(move |store| { @@ -261,7 +261,7 @@ impl SQLReadReplica { .await } - pub async fn write(&self, batch: Batch) -> trc::Result { + pub async fn write(&self, batch: Batch<'_>) -> trc::Result { match &self.primary { #[cfg(feature = "postgres")] Store::PostgreSQL(store) => store.write(batch).await, diff --git a/crates/store/src/backend/foundationdb/read.rs b/crates/store/src/backend/foundationdb/read.rs index 622efd8b..f2fdc95d 100644 --- a/crates/store/src/backend/foundationdb/read.rs +++ b/crates/store/src/backend/foundationdb/read.rs @@ -47,7 +47,7 @@ impl FdbStore { pub(crate) async fn get_bitmap( &self, - mut key: BitmapKey>, + mut key: BitmapKey, ) -> trc::Result> { let mut bm = RoaringBitmap::new(); let begin = key.serialize(WITH_SUBSPACE); @@ -150,7 +150,7 @@ impl FdbStore { pub(crate) async fn get_counter( &self, - key: impl Into>> + Sync + Send, + key: impl Into> + Sync + Send, ) -> trc::Result { let key = key.into().serialize(WITH_SUBSPACE); if let Some(bytes) = self diff --git a/crates/store/src/backend/foundationdb/write.rs b/crates/store/src/backend/foundationdb/write.rs index 38d66fe1..901641cd 100644 --- a/crates/store/src/backend/foundationdb/write.rs +++ b/crates/store/src/backend/foundationdb/write.rs @@ -11,20 +11,18 @@ use std::{ use foundationdb::{ FdbError, KeySelector, RangeOption, Transaction, - options::{self, MutationType, StreamingMode}, + options::{self, MutationType}, }; use futures::TryStreamExt; use rand::Rng; -use roaring::RoaringBitmap; use crate::{ - BitmapKey, IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, - U32_LEN, WITH_SUBSPACE, + IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, + WITH_SUBSPACE, backend::deserialize_i64_le, write::{ - AssignedIds, Batch, BitmapClass, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, Operation, - RandomAvailableId, ValueOp, - key::{DeserializeBigEndian, KeySerializer}, + AssignedIds, Batch, BitmapClass, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, Operation, ValueOp, + key::KeySerializer, }, }; @@ -34,7 +32,7 @@ use super::{ }; impl FdbStore { - pub(crate) async fn write(&self, batch: Batch) -> trc::Result { + pub(crate) async fn write(&self, batch: Batch<'_>) -> trc::Result { let start = Instant::now(); let mut retry_count = 0; @@ -42,12 +40,11 @@ impl FdbStore { let mut account_id = u32::MAX; let mut collection = u8::MAX; let mut document_id = u32::MAX; - let mut change_id = u64::MAX; let mut result = AssignedIds::default(); let trx = self.db.create_trx().map_err(into_error)?; - for op in &batch.ops { + for op in batch.ops { match op { Operation::AccountId { account_id: account_id_, @@ -64,24 +61,13 @@ impl FdbStore { } => { document_id = *document_id_; } - Operation::ChangeId { - change_id: change_id_, - } => { - change_id = *change_id_; - } Operation::Value { class, op } => { - let mut key = class.serialize( - account_id, - collection, - document_id, - WITH_SUBSPACE, - (&result).into(), - ); + let mut key = + class.serialize(account_id, collection, document_id, WITH_SUBSPACE); let do_chunk = !class.is_counter(collection); match op { ValueOp::Set(value) => { - let value = value.resolve(&result)?; if !value.is_empty() && do_chunk { for (pos, chunk) in value.chunks(MAX_VALUE_SIZE).enumerate() { match pos.cmp(&1) { @@ -154,59 +140,12 @@ impl FdbStore { } } Operation::Bitmap { class, set } => { - // Find the next available document id - let assign_id = *set - && matches!(class, BitmapClass::DocumentIds) - && document_id == u32::MAX; - if assign_id { - let begin = BitmapKey { - account_id, - collection, - class: BitmapClass::DocumentIds, - document_id: 0, - } - .serialize(WITH_SUBSPACE); - let end = BitmapKey { - account_id, - collection, - class: BitmapClass::DocumentIds, - document_id: u32::MAX, - } - .serialize(WITH_SUBSPACE); - let key_len = begin.len(); - let mut values = trx.get_ranges_keyvalues( - RangeOption { - begin: KeySelector::first_greater_or_equal(begin), - end: KeySelector::first_greater_or_equal(end), - mode: StreamingMode::WantAll, - reverse: false, - ..RangeOption::default() - }, - true, - ); - let mut found_ids = RoaringBitmap::new(); - while let Some(value) = values.try_next().await.map_err(into_error)? { - let key = value.key(); - if key.len() == key_len { - found_ids.insert(key.deserialize_be_u32(key_len - U32_LEN)?); - } else { - break; - } - } - document_id = found_ids.random_available_id(); - result.push_document_id(document_id); - } - - let key = class.serialize( - account_id, - collection, - document_id, - WITH_SUBSPACE, - (&result).into(), - ); + let is_document_id = matches!(class, BitmapClass::DocumentIds); + let key = + class.serialize(account_id, collection, document_id, WITH_SUBSPACE); if *set { - if assign_id { + if is_document_id { trx.add_conflict_range( &key, &class.serialize( @@ -214,7 +153,6 @@ impl FdbStore { collection, document_id + 1, WITH_SUBSPACE, - (&result).into(), ), options::ConflictRangeType::Read, ) @@ -226,26 +164,25 @@ impl FdbStore { trx.clear(&key); } } - Operation::Log { set } => { + Operation::Log { + collection, + change_id, + set, + } => { let key = LogKey { account_id, - collection, - change_id, + collection: *collection, + change_id: *change_id, } .serialize(WITH_SUBSPACE); - trx.set(&key, set.resolve(&result)?.as_ref()); + trx.set(&key, set); } Operation::AssertValue { class, assert_value, } => { - let key = class.serialize( - account_id, - collection, - document_id, - WITH_SUBSPACE, - (&result).into(), - ); + let key = + class.serialize(account_id, collection, document_id, WITH_SUBSPACE); let matches = match read_chunked_value(&key, &trx, false).await { Ok(ChunkedValue::Single(bytes)) => assert_value.matches(bytes.as_ref()), diff --git a/crates/store/src/backend/mysql/read.rs b/crates/store/src/backend/mysql/read.rs index 72f03d49..3a470b72 100644 --- a/crates/store/src/backend/mysql/read.rs +++ b/crates/store/src/backend/mysql/read.rs @@ -43,7 +43,7 @@ impl MysqlStore { pub(crate) async fn get_bitmap( &self, - mut key: BitmapKey>, + mut key: BitmapKey, ) -> trc::Result> { let begin = key.serialize(0); key.document_id = u32::MAX; @@ -140,7 +140,7 @@ impl MysqlStore { pub(crate) async fn get_counter( &self, - key: impl Into>> + Sync + Send, + key: impl Into> + Sync + Send, ) -> trc::Result { let key = key.into(); let table = char::from(key.subspace()); diff --git a/crates/store/src/backend/mysql/write.rs b/crates/store/src/backend/mysql/write.rs index f206e1dd..2fd10b1d 100644 --- a/crates/store/src/backend/mysql/write.rs +++ b/crates/store/src/backend/mysql/write.rs @@ -7,17 +7,13 @@ use std::time::{Duration, Instant}; use ahash::AHashMap; -use futures::TryStreamExt; use mysql_async::{Conn, Error, IsolationLevel, TxOpts, params, prelude::Queryable}; use rand::Rng; -use roaring::RoaringBitmap; use crate::{ - BitmapKey, IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, - U32_LEN, + IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, write::{ - AssignedIds, Batch, BitmapClass, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, Operation, - RandomAvailableId, ValueOp, key::DeserializeBigEndian, + AssignedIds, Batch, BitmapClass, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, Operation, ValueOp, }, }; @@ -31,7 +27,7 @@ enum CommitError { } impl MysqlStore { - pub(crate) async fn write(&self, batch: Batch) -> trc::Result { + pub(crate) async fn write(&self, batch: Batch<'_>) -> trc::Result { let start = Instant::now(); let mut retry_count = 0; let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; @@ -70,11 +66,14 @@ impl MysqlStore { } } - async fn write_trx(&self, conn: &mut Conn, batch: &Batch) -> Result { + async fn write_trx( + &self, + conn: &mut Conn, + batch: &Batch<'_>, + ) -> Result { let mut account_id = u32::MAX; let mut collection = u8::MAX; let mut document_id = u32::MAX; - let mut change_id = u64::MAX; let mut asserted_values = AHashMap::new(); let mut tx_opts = TxOpts::default(); tx_opts @@ -83,7 +82,7 @@ impl MysqlStore { let mut trx = conn.start_transaction(tx_opts).await?; let mut result = AssignedIds::default(); - for op in &batch.ops { + for op in batch.ops { match op { Operation::AccountId { account_id: account_id_, @@ -100,14 +99,8 @@ impl MysqlStore { } => { document_id = *document_id_; } - Operation::ChangeId { - change_id: change_id_, - } => { - change_id = *change_id_; - } Operation::Value { class, op } => { - let key = - class.serialize(account_id, collection, document_id, 0, (&result).into()); + let key = class.serialize(account_id, collection, document_id, 0); let table = char::from(class.subspace(collection)); match op { @@ -132,13 +125,7 @@ impl MysqlStore { .await? }; - match trx - .exec_drop( - &s, - params! {"k" => key, "v" => value.resolve(&result)?.as_ref()}, - ) - .await - { + match trx.exec_drop(&s, params! {"k" => key, "v" => value}).await { Ok(_) => { if exists.is_some() && trx.affected_rows() == 0 { trx.rollback().await?; @@ -221,42 +208,8 @@ impl MysqlStore { trx.exec_drop(&s, (key,)).await?; } Operation::Bitmap { class, set } => { - // Find the next available document id let is_document_id = matches!(class, BitmapClass::DocumentIds); - if *set && is_document_id && document_id == u32::MAX { - let begin = BitmapKey { - account_id, - collection, - class: BitmapClass::DocumentIds, - document_id: 0, - } - .serialize(0); - let end = BitmapKey { - account_id, - collection, - class: BitmapClass::DocumentIds, - document_id: u32::MAX, - } - .serialize(0); - let key_len = begin.len(); - - let s = trx.prep("SELECT k FROM b WHERE k >= ? AND k <= ?").await?; - let mut rows = trx.exec_stream::, _, _>(&s, (begin, end)).await?; - let mut found_ids = RoaringBitmap::new(); - - while let Some(key) = rows.try_next().await? { - if key.len() == key_len { - found_ids.insert( - key.as_slice().deserialize_be_u32(key.len() - U32_LEN)?, - ); - } - } - - document_id = found_ids.random_available_id(); - result.push_document_id(document_id); - } - let key = - class.serialize(account_id, collection, document_id, 0, (&result).into()); + let key = class.serialize(account_id, collection, document_id, 0); let table = char::from(class.subspace()); let s = if *set { @@ -284,11 +237,15 @@ impl MysqlStore { ); } } - Operation::Log { set } => { + Operation::Log { + collection, + change_id, + set, + } => { let key = LogKey { account_id, - collection, - change_id, + collection: *collection, + change_id: *change_id, } .serialize(0); @@ -296,15 +253,13 @@ impl MysqlStore { .prep("INSERT INTO l (k, v) VALUES (?, ?) ON DUPLICATE KEY UPDATE v = VALUES(v)") .await?; - trx.exec_drop(&s, (key, set.resolve(&result)?.as_ref())) - .await?; + trx.exec_drop(&s, (key, &set)).await?; } Operation::AssertValue { class, assert_value, } => { - let key = - class.serialize(account_id, collection, document_id, 0, (&result).into()); + let key = class.serialize(account_id, collection, document_id, 0); let table = char::from(class.subspace(collection)); let s = trx diff --git a/crates/store/src/backend/postgres/read.rs b/crates/store/src/backend/postgres/read.rs index d6cb5a29..af33beed 100644 --- a/crates/store/src/backend/postgres/read.rs +++ b/crates/store/src/backend/postgres/read.rs @@ -42,7 +42,7 @@ impl PostgresStore { pub(crate) async fn get_bitmap( &self, - mut key: BitmapKey>, + mut key: BitmapKey, ) -> trc::Result> { let begin = key.serialize(0); key.document_id = u32::MAX; @@ -132,7 +132,7 @@ impl PostgresStore { pub(crate) async fn get_counter( &self, - key: impl Into>> + Sync + Send, + key: impl Into> + Sync + Send, ) -> trc::Result { let key = key.into(); let table = char::from(key.subspace()); diff --git a/crates/store/src/backend/postgres/write.rs b/crates/store/src/backend/postgres/write.rs index 869a6f55..0576cc69 100644 --- a/crates/store/src/backend/postgres/write.rs +++ b/crates/store/src/backend/postgres/write.rs @@ -8,17 +8,13 @@ use std::time::{Duration, Instant}; use ahash::AHashMap; use deadpool_postgres::Object; -use futures::{TryStreamExt, pin_mut}; use rand::Rng; -use roaring::RoaringBitmap; use tokio_postgres::{IsolationLevel, error::SqlState}; use crate::{ - BitmapKey, IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, - U32_LEN, + IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, write::{ - AssignedIds, Batch, BitmapClass, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, Operation, - RandomAvailableId, ValueOp, key::DeserializeBigEndian, + AssignedIds, Batch, BitmapClass, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, Operation, ValueOp, }, }; @@ -32,7 +28,7 @@ enum CommitError { } impl PostgresStore { - pub(crate) async fn write(&self, batch: Batch) -> trc::Result { + pub(crate) async fn write(&self, batch: Batch<'_>) -> trc::Result { let mut conn = self.conn_pool.get().await.map_err(into_error)?; let start = Instant::now(); let mut retry_count = 0; @@ -76,12 +72,11 @@ impl PostgresStore { async fn write_trx( &self, conn: &mut Object, - batch: &Batch, + batch: &Batch<'_>, ) -> Result { let mut account_id = u32::MAX; let mut collection = u8::MAX; let mut document_id = u32::MAX; - let mut change_id = u64::MAX; let mut asserted_values = AHashMap::new(); let trx = conn .build_transaction() @@ -90,7 +85,7 @@ impl PostgresStore { .await?; let mut result = AssignedIds::default(); - for op in &batch.ops { + for op in batch.ops { match op { Operation::AccountId { account_id: account_id_, @@ -107,14 +102,8 @@ impl PostgresStore { } => { document_id = *document_id_; } - Operation::ChangeId { - change_id: change_id_, - } => { - change_id = *change_id_; - } Operation::Value { class, op } => { - let key = - class.serialize(account_id, collection, document_id, 0, (&result).into()); + let key = class.serialize(account_id, collection, document_id, 0); let table = char::from(class.subspace(collection)); match op { @@ -144,11 +133,7 @@ impl PostgresStore { .await? }; - if trx - .execute(&s, &[&key, &value.resolve(&result)?.as_ref()]) - .await? - == 0 - { + if trx.execute(&s, &[&key, &value]).await? == 0 { return Err(trc::StoreEvent::AssertValueFailed.into_err().into()); } } @@ -218,47 +203,8 @@ impl PostgresStore { trx.execute(&s, &[&key]).await?; } Operation::Bitmap { class, set } => { - // Find the next available document id let is_document_id = matches!(class, BitmapClass::DocumentIds); - if *set && is_document_id && document_id == u32::MAX { - let begin = BitmapKey { - account_id, - collection, - class: BitmapClass::DocumentIds, - document_id: 0, - } - .serialize(0); - let end = BitmapKey { - account_id, - collection, - class: BitmapClass::DocumentIds, - document_id: u32::MAX, - } - .serialize(0); - let key_len = begin.len(); - - let s = trx - .prepare_cached("SELECT k FROM b WHERE k >= $1 AND k <= $2") - .await?; - let rows = trx.query_raw(&s, &[&begin, &end]).await?; - - pin_mut!(rows); - - let mut found_ids = RoaringBitmap::new(); - - while let Some(row) = rows.try_next().await? { - let key: &[u8] = row.try_get(0)?; - if key.len() == key_len { - found_ids.insert(key.deserialize_be_u32(key_len - U32_LEN)?); - } - } - - document_id = found_ids.random_available_id(); - result.push_document_id(document_id); - } - - let key = - class.serialize(account_id, collection, document_id, 0, (&result).into()); + let key = class.serialize(account_id, collection, document_id, 0); let table = char::from(class.subspace()); let s = if *set { @@ -285,11 +231,15 @@ impl PostgresStore { } })?; } - Operation::Log { set } => { + Operation::Log { + collection, + change_id, + set, + } => { let key = LogKey { account_id, - collection, - change_id, + collection: *collection, + change_id: *change_id, } .serialize(0); @@ -300,15 +250,13 @@ impl PostgresStore { )) .await?; - trx.execute(&s, &[&key, &set.resolve(&result)?.as_ref()]) - .await?; + trx.execute(&s, &[&key, &set]).await?; } Operation::AssertValue { class, assert_value, } => { - let key = - class.serialize(account_id, collection, document_id, 0, (&result).into()); + let key = class.serialize(account_id, collection, document_id, 0); let table = char::from(class.subspace(collection)); let s = trx diff --git a/crates/store/src/backend/rocksdb/read.rs b/crates/store/src/backend/rocksdb/read.rs index c1d4b496..f31067b1 100644 --- a/crates/store/src/backend/rocksdb/read.rs +++ b/crates/store/src/backend/rocksdb/read.rs @@ -41,7 +41,7 @@ impl RocksDbStore { pub(crate) async fn get_bitmap( &self, - mut key: BitmapKey>, + mut key: BitmapKey, ) -> trc::Result> { let db = self.db.clone(); self.spawn_worker(move || { @@ -104,7 +104,7 @@ impl RocksDbStore { pub(crate) async fn get_counter( &self, - key: impl Into>> + Sync + Send, + key: impl Into> + Sync + Send, ) -> trc::Result { let key = key.into(); let db = self.db.clone(); diff --git a/crates/store/src/backend/rocksdb/write.rs b/crates/store/src/backend/rocksdb/write.rs index 97624d98..407416dd 100644 --- a/crates/store/src/backend/rocksdb/write.rs +++ b/crates/store/src/backend/rocksdb/write.rs @@ -11,25 +11,21 @@ use std::{ }; use rand::Rng; -use roaring::RoaringBitmap; use rocksdb::{ - BoundColumnFamily, Direction, ErrorKind, IteratorMode, OptimisticTransactionDB, + BoundColumnFamily, ErrorKind, IteratorMode, OptimisticTransactionDB, OptimisticTransactionOptions, WriteOptions, }; use super::{CF_INDEXES, CF_LOGS, CfHandle, RocksDbStore, into_error}; use crate::{ - BitmapKey, Deserialize, IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, - SUBSPACE_QUOTA, U32_LEN, + Deserialize, IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, + SUBSPACE_QUOTA, backend::deserialize_i64_le, - write::{ - AssignedIds, Batch, BitmapClass, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, Operation, - RandomAvailableId, ValueOp, key::DeserializeBigEndian, - }, + write::{AssignedIds, Batch, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, Operation, ValueOp}, }; impl RocksDbStore { - pub(crate) async fn write(&self, batch: Batch) -> trc::Result { + pub(crate) async fn write(&self, batch: Batch<'_>) -> trc::Result { let db = self.db.clone(); self.spawn_worker(move || { @@ -128,7 +124,7 @@ struct RocksDBTransaction<'x> { cf_indexes: Arc>, cf_logs: Arc>, txn_opts: OptimisticTransactionOptions, - batch: &'x Batch, + batch: &'x Batch<'x>, } enum CommitError { @@ -141,14 +137,13 @@ impl RocksDBTransaction<'_> { let mut account_id = u32::MAX; let mut collection = u8::MAX; let mut document_id = u32::MAX; - let mut change_id = u64::MAX; let mut result = AssignedIds::default(); let txn = self .db .transaction_opt(&WriteOptions::default(), &self.txn_opts); - for op in &self.batch.ops { + for op in self.batch.ops { match op { Operation::AccountId { account_id: account_id_, @@ -165,19 +160,13 @@ impl RocksDBTransaction<'_> { } => { document_id = *document_id_; } - Operation::ChangeId { - change_id: change_id_, - } => { - change_id = *change_id_; - } Operation::Value { class, op } => { - let key = - class.serialize(account_id, collection, document_id, 0, (&result).into()); + let key = class.serialize(account_id, collection, document_id, 0); let cf = self.db.subspace_handle(class.subspace(collection)); match op { ValueOp::Set(value) => { - txn.put_cf(&cf, &key, value.resolve(&result)?.as_ref())?; + txn.put_cf(&cf, &key, value)?; } ValueOp::AtomicAdd(by) => { txn.merge_cf(&cf, &key, &by.to_le_bytes()[..])?; @@ -220,46 +209,8 @@ impl RocksDBTransaction<'_> { } } Operation::Bitmap { class, set } => { - let is_document_id = matches!(class, BitmapClass::DocumentIds); let cf = self.db.subspace_handle(class.subspace()); - if *set && is_document_id && document_id == u32::MAX { - let begin = BitmapKey { - account_id, - collection, - class: BitmapClass::DocumentIds, - document_id: 0, - } - .serialize(0); - let end = BitmapKey { - account_id, - collection, - class: BitmapClass::DocumentIds, - document_id: u32::MAX, - } - .serialize(0); - let key_len = begin.len(); - let mut found_ids = RoaringBitmap::new(); - - for row in - txn.iterator_cf(&cf, IteratorMode::From(&begin, Direction::Forward)) - { - let (key, _) = row?; - let key = key.as_ref(); - if key.len() == key_len - && key >= begin.as_slice() - && key <= end.as_slice() - { - found_ids.insert(key.deserialize_be_u32(key.len() - U32_LEN)?); - } else { - break; - } - } - - document_id = found_ids.random_available_id(); - result.push_document_id(document_id); - } - let key = - class.serialize(account_id, collection, document_id, 0, (&result).into()); + let key = class.serialize(account_id, collection, document_id, 0); if *set { txn.put_cf(&cf, &key, [])?; @@ -267,22 +218,25 @@ impl RocksDBTransaction<'_> { txn.delete_cf(&cf, &key)?; } } - Operation::Log { set } => { + Operation::Log { + collection, + change_id, + set, + } => { let key = LogKey { account_id, - collection, - change_id, + collection: *collection, + change_id: *change_id, } .serialize(0); - txn.put_cf(&self.cf_logs, &key, set.resolve(&result)?.as_ref())?; + txn.put_cf(&self.cf_logs, &key, set)?; } Operation::AssertValue { class, assert_value, } => { - let key = - class.serialize(account_id, collection, document_id, 0, (&result).into()); + let key = class.serialize(account_id, collection, document_id, 0); let cf = self.db.subspace_handle(class.subspace(collection)); let matches = txn diff --git a/crates/store/src/backend/sqlite/read.rs b/crates/store/src/backend/sqlite/read.rs index 5dc736a1..43e1b576 100644 --- a/crates/store/src/backend/sqlite/read.rs +++ b/crates/store/src/backend/sqlite/read.rs @@ -41,7 +41,7 @@ impl SqliteStore { pub(crate) async fn get_bitmap( &self, - mut key: BitmapKey>, + mut key: BitmapKey, ) -> trc::Result> { let begin = key.serialize(0); key.document_id = u32::MAX; @@ -147,7 +147,7 @@ impl SqliteStore { pub(crate) async fn get_counter( &self, - key: impl Into>> + Sync + Send, + key: impl Into> + Sync + Send, ) -> trc::Result { let key = key.into(); let table = char::from(key.subspace()); diff --git a/crates/store/src/backend/sqlite/write.rs b/crates/store/src/backend/sqlite/write.rs index 8c40a151..40d07400 100644 --- a/crates/store/src/backend/sqlite/write.rs +++ b/crates/store/src/backend/sqlite/write.rs @@ -4,34 +4,28 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use roaring::RoaringBitmap; use rusqlite::{OptionalExtension, TransactionBehavior, params}; use crate::{ - BitmapKey, IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, - U32_LEN, - write::{ - AssignedIds, Batch, BitmapClass, Operation, RandomAvailableId, ValueOp, - key::DeserializeBigEndian, - }, + IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, + write::{AssignedIds, Batch, BitmapClass, Operation, ValueOp}, }; use super::{SqliteStore, into_error}; impl SqliteStore { - pub(crate) async fn write(&self, batch: Batch) -> trc::Result { + pub(crate) async fn write(&self, batch: Batch<'_>) -> trc::Result { let mut conn = self.conn_pool.get().map_err(into_error)?; self.spawn_worker(move || { let mut account_id = u32::MAX; let mut collection = u8::MAX; let mut document_id = u32::MAX; - let mut change_id = u64::MAX; let trx = conn .transaction_with_behavior(TransactionBehavior::Immediate) .map_err(into_error)?; let mut result = AssignedIds::default(); - for op in &batch.ops { + for op in batch.ops { match op { Operation::AccountId { account_id: account_id_, @@ -48,19 +42,8 @@ impl SqliteStore { } => { document_id = *document_id_; } - Operation::ChangeId { - change_id: change_id_, - } => { - change_id = *change_id_; - } Operation::Value { class, op } => { - let key = class.serialize( - account_id, - collection, - document_id, - 0, - (&result).into(), - ); + let key = class.serialize(account_id, collection, document_id, 0); let table = char::from(class.subspace(collection)); match op { @@ -70,7 +53,7 @@ impl SqliteStore { table )) .map_err(into_error)? - .execute([&key, value.resolve(&result)?.as_ref()]) + .execute([&key, value]) .map_err(into_error)?; } ValueOp::AtomicAdd(by) => { @@ -140,51 +123,8 @@ impl SqliteStore { } } Operation::Bitmap { class, set } => { - // Find the next available document id let is_document_id = matches!(class, BitmapClass::DocumentIds); - if *set && is_document_id && document_id == u32::MAX { - let begin = BitmapKey { - account_id, - collection, - class: BitmapClass::DocumentIds, - document_id: 0, - } - .serialize(0); - let end = BitmapKey { - account_id, - collection, - class: BitmapClass::DocumentIds, - document_id: u32::MAX, - } - .serialize(0); - let key_len = begin.len(); - - let mut query = trx - .prepare_cached("SELECT k FROM b WHERE k >= ? AND k <= ?") - .map_err(into_error)?; - let mut rows = query.query([&begin, &end]).map_err(into_error)?; - let mut found_ids = RoaringBitmap::new(); - while let Some(row) = rows.next().map_err(into_error)? { - let key = row - .get_ref(0) - .map_err(into_error)? - .as_bytes() - .map_err(into_error)?; - if key.len() == key_len { - found_ids.insert(key.deserialize_be_u32(key.len() - U32_LEN)?); - } - } - - document_id = found_ids.random_available_id(); - result.push_document_id(document_id); - } - let key = class.serialize( - account_id, - collection, - document_id, - 0, - (&result).into(), - ); + let key = class.serialize(account_id, collection, document_id, 0); let table = char::from(class.subspace()); if *set { @@ -209,30 +149,28 @@ impl SqliteStore { .map_err(into_error)?; }; } - Operation::Log { set } => { + Operation::Log { + collection, + change_id, + set, + } => { let key = LogKey { account_id, - collection, - change_id, + collection: *collection, + change_id: *change_id, } .serialize(0); trx.prepare_cached("INSERT OR REPLACE INTO l (k, v) VALUES (?, ?)") .map_err(into_error)? - .execute([&key, set.resolve(&result).map_err(into_error)?.as_ref()]) + .execute([&key, set]) .map_err(into_error)?; } Operation::AssertValue { class, assert_value, } => { - let key = class.serialize( - account_id, - collection, - document_id, - 0, - (&result).into(), - ); + let key = class.serialize(account_id, collection, document_id, 0); let table = char::from(class.subspace(collection)); let matches = trx diff --git a/crates/store/src/dispatch/lookup.rs b/crates/store/src/dispatch/lookup.rs index 81d7cacd..d2ac6a33 100644 --- a/crates/store/src/dispatch/lookup.rs +++ b/crates/store/src/dispatch/lookup.rs @@ -21,7 +21,7 @@ use crate::{ use crate::{ SerializeInfallible, backend::http::lookup::HttpStoreGet, - write::{InMemoryClass, MaybeDynamicId, assert::AssertValue}, + write::{InMemoryClass, assert::AssertValue}, }; pub struct KeyValue { @@ -35,17 +35,16 @@ impl InMemoryStore { match self { InMemoryStore::Store(store) => { let mut batch = BatchBuilder::new(); - batch.ops.push(Operation::Value { + batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Key(kv.key)), op: ValueOp::Set( KeySerializer::new(kv.value.len() + U64_LEN) .write(kv.expires.map_or(u64::MAX, |expires| now() + expires)) .write(kv.value.as_slice()) - .finalize() - .into(), + .finalize(), ), }); - store.write(batch.build()).await.map(|_| ()) + store.write(batch.build_all()).await.map(|_| ()) } #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store.key_set(&kv.key, &kv.value, kv.expires).await, @@ -64,35 +63,34 @@ impl InMemoryStore { let mut batch = BatchBuilder::new(); if let Some(expires) = kv.expires { - batch.ops.push(Operation::Value { + batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Key(kv.key.clone())), op: ValueOp::Set( KeySerializer::new(U64_LEN * 2) .write(0u64) .write(now() + expires) - .finalize() - .into(), + .finalize(), ), }); } if return_value { - batch.ops.push(Operation::Value { + batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Counter(kv.key)), op: ValueOp::AddAndGet(kv.value), }); store - .write(batch.build()) + .write(batch.build_all()) .await .and_then(|r| r.last_counter_id()) } else { - batch.ops.push(Operation::Value { + batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Counter(kv.key)), op: ValueOp::AtomicAdd(kv.value), }); - store.write(batch.build()).await.map(|_| 0) + store.write(batch.build_all()).await.map(|_| 0) } } #[cfg(feature = "redis")] @@ -110,11 +108,11 @@ impl InMemoryStore { match self { InMemoryStore::Store(store) => { let mut batch = BatchBuilder::new(); - batch.ops.push(Operation::Value { + batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Key(key.into().into_bytes())), op: ValueOp::Clear, }); - store.write(batch.build()).await.map(|_| ()) + store.write(batch.build_all()).await.map(|_| ()) } #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store.key_delete(key.into().as_bytes()).await, @@ -131,11 +129,11 @@ impl InMemoryStore { match self { InMemoryStore::Store(store) => { let mut batch = BatchBuilder::new(); - batch.ops.push(Operation::Value { + batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Counter(key.into().into_bytes())), op: ValueOp::Clear, }); - store.write(batch.build()).await.map(|_| ()) + store.write(batch.build_all()).await.map(|_| ()) } #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store.key_delete(key.into().as_bytes()).await, @@ -300,12 +298,12 @@ impl InMemoryStore { { // TODO remove in 1.0 let mut batch = BatchBuilder::new(); - batch.ops.push(Operation::Value { + batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Key(key.clone())), op: ValueOp::Clear, }); store - .write(batch.build()) + .write(batch.build_all()) .await .caused_by(trc::location!())?; None @@ -322,7 +320,7 @@ impl InMemoryStore { return Ok(false); } - let key: ValueClass = ValueClass::InMemory(InMemoryClass::Key(key)); + let key: ValueClass = ValueClass::InMemory(InMemoryClass::Key(key)); let mut batch = BatchBuilder::new(); batch.assert_value( key.clone(), @@ -332,7 +330,7 @@ impl InMemoryStore { }, ); batch.set(key.clone(), (now + duration).serialize()); - match store.write(batch.build()).await { + match store.write(batch.build_all()).await { Ok(_) => Ok(true), Err(err) if err.is_assertion_failure() => Ok(false), Err(err) => Err(err @@ -394,21 +392,21 @@ impl InMemoryStore { if !expired_keys.is_empty() { let mut batch = BatchBuilder::new(); for key in expired_keys { - batch.ops.push(Operation::Value { + batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Key(key)), op: ValueOp::Clear, }); - if batch.ops.len() >= 1000 { + if batch.len() >= 1000 { store - .write(batch.build()) + .write(batch.build_all()) .await .caused_by(trc::location!())?; batch = BatchBuilder::new(); } } - if !batch.ops.is_empty() { + if !batch.is_empty() { store - .write(batch.build()) + .write(batch.build_all()) .await .caused_by(trc::location!())?; } @@ -417,25 +415,25 @@ impl InMemoryStore { if !expired_counters.is_empty() { let mut batch = BatchBuilder::new(); for key in expired_counters { - batch.ops.push(Operation::Value { + batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Counter(key.clone())), op: ValueOp::Clear, }); - batch.ops.push(Operation::Value { + batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Key(key)), op: ValueOp::Clear, }); - if batch.ops.len() >= 1000 { + if batch.len() >= 1000 { store - .write(batch.build()) + .write(batch.build_all()) .await .caused_by(trc::location!())?; batch = BatchBuilder::new(); } } - if !batch.ops.is_empty() { + if !batch.is_empty() { store - .write(batch.build()) + .write(batch.build_all()) .await .caused_by(trc::location!())?; } diff --git a/crates/store/src/dispatch/store.rs b/crates/store/src/dispatch/store.rs index f4c7d202..6b5f925e 100644 --- a/crates/store/src/dispatch/store.rs +++ b/crates/store/src/dispatch/store.rs @@ -61,7 +61,7 @@ impl Store { pub async fn get_bitmap( &self, - key: BitmapKey>, + key: BitmapKey, ) -> trc::Result> { match self { #[cfg(feature = "sqlite")] @@ -83,7 +83,7 @@ impl Store { pub async fn get_bitmaps_intersection( &self, - keys: Vec>>, + keys: Vec>, ) -> trc::Result> { let mut result: Option = None; for key in keys { @@ -136,7 +136,7 @@ impl Store { pub async fn get_counter( &self, - key: impl Into>> + Sync + Send, + key: impl Into> + Sync + Send, ) -> trc::Result { match self { #[cfg(feature = "sqlite")] @@ -183,8 +183,7 @@ impl Store { result.caused_by(trc::location!()) } - pub async fn write(&self, batch: impl Into) -> trc::Result { - let batch = batch.into(); + pub async fn write(&self, batch: Batch<'_>) -> trc::Result { #[cfg(feature = "test_mode")] if std::env::var("PARANOID_WRITE").is_ok_and(|v| v == "1") { let mut account_id = u32::MAX; @@ -192,9 +191,8 @@ impl Store { let mut document_id = u32::MAX; let mut bitmaps = Vec::new(); - let mut result = AssignedIds::default(); - for op in &batch.ops { + for op in batch.ops { match op { Operation::AccountId { account_id: account_id_, @@ -212,18 +210,7 @@ impl Store { document_id = *document_id_; } Operation::Bitmap { class, set } => { - if *set && matches!(class, BitmapClass::DocumentIds) { - let id = result.document_ids.len() as u32; - result.document_ids.push(id); - } - - let key = class.serialize( - account_id, - collection, - document_id, - 0, - (&result).into(), - ); + let key = class.serialize(account_id, collection, document_id, 0); bitmaps.push((key, class.clone(), document_id, *set)); } @@ -303,11 +290,24 @@ impl Store { result } - #[inline] - pub async fn write_expect_id(&self, batch: impl Into) -> trc::Result { - self.write(batch) - .await - .and_then(|ids| ids.last_document_id()) + pub async fn assign_document_ids( + &self, + account_id: u32, + collection: impl Into, + num_ids: u64, + ) -> trc::Result { + // Increment UID next + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .with_collection(collection) + .add_and_get(ValueClass::DocumentId, num_ids as i64); + self.write(batch.build_all()).await.and_then(|v| { + v.last_counter_id().map(|id| { + debug_assert!(id >= num_ids as i64, "{} < {}", id, num_ids); + id as u32 + }) + }) } pub async fn purge_store(&self) -> trc::Result<()> { @@ -437,19 +437,19 @@ impl Store { let mut batch = BatchBuilder::new(); for key in delete_keys { - if batch.ops.len() >= 1000 { - self.write(std::mem::take(&mut batch).build()) + if batch.len() >= 1000 { + self.write(std::mem::take(&mut batch).build_all()) .await .caused_by(trc::location!())?; } - batch.ops.push(Operation::Value { + batch.any_op(Operation::Value { class: ValueClass::Any(AnyClass { subspace, key }), op: ValueOp::Clear, }); } if !batch.is_empty() { - self.write(batch.build()) + self.write(batch.build_all()) .await .caused_by(trc::location!())?; } @@ -604,6 +604,7 @@ impl Store { SUBSPACE_BLOB_RESERVE, SUBSPACE_BLOB_LINK, SUBSPACE_LOGS, + SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_IN_MEMORY_VALUE, SUBSPACE_COUNTER, SUBSPACE_PROPERTY, @@ -680,7 +681,7 @@ impl Store { batch.with_account_id(account_id); } - batch.ops.push(Operation::Value { + batch.any_op(Operation::Value { class: ValueClass::Blob(BlobOp::Reserve { hash: BlobHash::try_from_hash_slice( key.get(U32_LEN..U32_LEN + BLOB_HASH_LEN).unwrap(), @@ -698,7 +699,7 @@ impl Store { ) .await .unwrap(); - self.write(batch.build()).await.unwrap(); + self.write(batch.build_all()).await.unwrap(); } #[cfg(feature = "test_mode")] @@ -727,38 +728,38 @@ impl Store { if !expired_keys.is_empty() { let mut batch = BatchBuilder::new(); for key in expired_keys { - batch.ops.push(Operation::Value { + batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Key(key)), op: ValueOp::Clear, }); - if batch.ops.len() >= 1000 { - self.write(batch.build()).await.unwrap(); + if batch.len() >= 1000 { + self.write(batch.build_all()).await.unwrap(); batch = BatchBuilder::new(); } } - if !batch.ops.is_empty() { - self.write(batch.build()).await.unwrap(); + if !batch.is_empty() { + self.write(batch.build_all()).await.unwrap(); } } if !expired_counters.is_empty() { let mut batch = BatchBuilder::new(); for key in expired_counters { - batch.ops.push(Operation::Value { + batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Counter(key.clone())), op: ValueOp::Clear, }); - batch.ops.push(Operation::Value { + batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Key(key)), op: ValueOp::Clear, }); - if batch.ops.len() >= 1000 { - self.write(batch.build()).await.unwrap(); + if batch.len() >= 1000 { + self.write(batch.build_all()).await.unwrap(); batch = BatchBuilder::new(); } } - if !batch.ops.is_empty() { - self.write(batch.build()).await.unwrap(); + if !batch.is_empty() { + self.write(batch.build_all()).await.unwrap(); } } } @@ -873,6 +874,10 @@ impl Store { value ); } + SUBSPACE_COUNTER if key.len() == std::mem::size_of::() + 1 => { + // Message ID counters + return Ok(true); + } SUBSPACE_INDEXES => { println!( concat!( @@ -907,7 +912,7 @@ impl Store { .unwrap(); } - // Delete logs + // Delete logs and counters self.delete_range( AnyKey { subspace: SUBSPACE_LOGS, @@ -929,14 +934,21 @@ impl Store { .await .unwrap(); + self.delete_range( + AnyKey { + subspace: SUBSPACE_COUNTER, + key: &[0u8], + }, + AnyKey { + subspace: SUBSPACE_COUNTER, + key: (u32::MAX / 2).to_be_bytes().as_slice(), + }, + ) + .await + .unwrap(); + if failed { panic!("Store is not empty."); } } } - -impl From for Batch { - fn from(builder: BatchBuilder) -> Self { - builder.build() - } -} diff --git a/crates/store/src/fts/index.rs b/crates/store/src/fts/index.rs index 0b9c9190..dd9c2b40 100644 --- a/crates/store/src/fts/index.rs +++ b/crates/store/src/fts/index.rs @@ -22,7 +22,7 @@ use crate::{ backend::MAX_TOKEN_LENGTH, dispatch::DocumentSet, write::{ - BatchBuilder, BitmapHash, MaybeDynamicId, Operation, ValueClass, ValueOp, hash::TokenType, + BatchBuilder, BitmapHash, Operation, ValueClass, ValueOp, hash::TokenType, key::DeserializeBigEndian, }, }; @@ -202,7 +202,7 @@ impl Store { for (hash, postings) in tokens.into_iter() { keys.push(Operation::Value { class: ValueClass::FtsIndex(hash), - op: ValueOp::Set(postings.serialize().into()), + op: ValueOp::Set(postings.serialize()), }); } @@ -214,19 +214,19 @@ impl Store { .update_document(document.document_id); for key in keys.into_iter() { - if batch.ops.len() >= 1000 { - self.write(batch.build()).await?; + if batch.len() >= 1000 { + self.write(batch.build_all()).await?; batch = BatchBuilder::new(); batch .with_account_id(document.account_id) .with_collection(document.collection) .update_document(document.document_id); } - batch.ops.push(key); + batch.any_op(key); } if !batch.is_empty() { - self.write(batch.build()).await?; + self.write(batch.build_all()).await?; } Ok(()) @@ -239,7 +239,7 @@ impl Store { document_ids: &impl DocumentSet, ) -> trc::Result<()> { // Find keys to delete - let mut delete_keys: AHashMap>> = AHashMap::new(); + let mut delete_keys: AHashMap> = AHashMap::new(); self.iterate( IterateParams::new( ValueKey { @@ -308,15 +308,15 @@ impl Store { batch.update_document(document_id); for key in keys { - if batch.ops.len() >= 1000 { - self.write(batch.build()).await?; + if batch.len() >= 1000 { + self.write(batch.build_all()).await?; batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(collection) .update_document(document_id); } - batch.ops.push(Operation::Value { + batch.any_op(Operation::Value { class: key, op: ValueOp::Clear, }); @@ -324,7 +324,7 @@ impl Store { } if !batch.is_empty() { - self.write(batch.build()).await?; + self.write(batch.build_all()).await?; } Ok(()) diff --git a/crates/store/src/fts/query.rs b/crates/store/src/fts/query.rs index 47f6c4a8..ac993815 100644 --- a/crates/store/src/fts/query.rs +++ b/crates/store/src/fts/query.rs @@ -18,9 +18,7 @@ use crate::{ BitmapKey, IterateParams, Store, U32_LEN, ValueKey, backend::MAX_TOKEN_LENGTH, fts::FtsFilter, - write::{ - BitmapHash, DynamicDocumentId, ValueClass, hash::TokenType, key::DeserializeBigEndian, - }, + write::{BitmapHash, ValueClass, hash::TokenType, key::DeserializeBigEndian}, }; use super::postings::SerializedPostings; @@ -324,7 +322,7 @@ impl Store { } // Fetch from store - let key_len = ValueClass::FtsIndex::(*token).serialized_size(); + let key_len = ValueClass::FtsIndex(*token).serialized_size(); self.iterate( IterateParams::new( ValueKey { diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 09932aca..6401f94a 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -15,6 +15,7 @@ pub mod write; pub use ahash; pub use blake3; +pub use gxhash; pub use parking_lot; pub use rand; pub use rkyv; @@ -99,7 +100,7 @@ pub trait Key: Sync + Send + Clone { } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct BitmapKey>> { +pub struct BitmapKey> { pub account_id: u32, pub collection: u8, pub class: T, @@ -123,7 +124,7 @@ pub struct IndexKeyPrefix { } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct ValueKey>> { +pub struct ValueKey> { pub account_id: u32, pub collection: u8, pub document_id: u32, diff --git a/crates/store/src/query/acl.rs b/crates/store/src/query/acl.rs index 52101bd9..e4525af9 100644 --- a/crates/store/src/query/acl.rs +++ b/crates/store/src/query/acl.rs @@ -9,7 +9,7 @@ use trc::AddContext; use crate::{ Deserialize, IterateParams, Store, U32_LEN, ValueKey, - write::{BatchBuilder, Operation, ValueClass, ValueOp, key::DeserializeBigEndian}, + write::{BatchBuilder, ValueClass, key::DeserializeBigEndian}, }; pub enum AclQuery { @@ -102,10 +102,7 @@ impl Store { if account_id == key.deserialize_be_u32(U32_LEN)? { let owner_account_id = key.deserialize_be_u32(0)?; revoked_accounts.insert(owner_account_id); - delete_keys.push(( - ValueClass::Acl(owner_account_id), - AclItem::deserialize(key)?, - )); + delete_keys.push((owner_account_id, AclItem::deserialize(key)?)); } Ok(true) @@ -118,9 +115,9 @@ impl Store { let mut batch = BatchBuilder::new(); batch.with_account_id(account_id); let mut last_collection = u8::MAX; - for (class, acl_item) in delete_keys.into_iter() { - if batch.ops.len() >= 1000 { - self.write(batch.build()) + for (revoke_account_id, acl_item) in delete_keys.into_iter() { + if batch.len() >= 1000 { + self.write(batch.build_all()) .await .caused_by(trc::location!())?; batch = BatchBuilder::new(); @@ -131,14 +128,12 @@ impl Store { batch.with_collection(acl_item.to_collection); last_collection = acl_item.to_collection; } - batch.update_document(acl_item.to_document_id); - batch.ops.push(Operation::Value { - class, - op: ValueOp::Clear, - }) + batch + .update_document(acl_item.to_document_id) + .acl_revoke(revoke_account_id); } if !batch.is_empty() { - self.write(batch.build()) + self.write(batch.build_all()) .await .caused_by(trc::location!())?; } diff --git a/crates/store/src/query/mod.rs b/crates/store/src/query/mod.rs index cbebefbe..0c574d76 100644 --- a/crates/store/src/query/mod.rs +++ b/crates/store/src/query/mod.rs @@ -38,7 +38,7 @@ pub enum Filter { text: String, tokenize: bool, }, - InBitmap(BitmapClass), + InBitmap(BitmapClass), DocumentSet(RoaringBitmap), And, Or, @@ -152,7 +152,7 @@ impl Filter { } } - pub fn is_in_bitmap(field: impl Into, value: impl Into>) -> Self { + pub fn is_in_bitmap(field: impl Into, value: impl Into) -> Self { Self::InBitmap(BitmapClass::Tag { field: field.into(), value: value.into(), @@ -191,7 +191,7 @@ impl Comparator { } } -impl BitmapKey> { +impl BitmapKey { pub fn document_ids(account_id: u32, collection: impl Into) -> Self { BitmapKey { account_id, @@ -222,7 +222,7 @@ impl BitmapKey> { account_id: u32, collection: impl Into, field: impl Into, - value: impl Into>, + value: impl Into, ) -> Self { BitmapKey { account_id, diff --git a/crates/store/src/write/batch.rs b/crates/store/src/write/batch.rs index aa151151..0220a577 100644 --- a/crates/store/src/write/batch.rs +++ b/crates/store/src/write/batch.rs @@ -4,63 +4,97 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{ - Batch, BatchBuilder, BitmapClass, IntoOperations, MaybeDynamicId, MaybeDynamicValue, Operation, - TagValue, ValueClass, ValueOp, assert::ToAssertValue, +use std::sync::{ + LazyLock, + atomic::{AtomicU64, Ordering}, }; +use utils::{ + map::bitmap::{Bitmap, ShortId}, + snowflake::SnowflakeIdGenerator, +}; + +use crate::U32_LEN; + +use super::{ + Batch, BatchBuilder, BitmapClass, IntoOperations, Operation, TagValue, ValueClass, ValueOp, + assert::ToAssertValue, +}; + +static CHANGE_SEQ: AtomicU64 = AtomicU64::new(0); +static NODE_MUM: LazyLock = LazyLock::new(|| CHANGE_SEQ.swap(0, Ordering::Relaxed) as u16); + impl BatchBuilder { pub fn new() -> Self { Self { - ops: Vec::with_capacity(16), + ops: Vec::with_capacity(32), + current_change_id: None, + current_account_id: None, + current_collection: None, + current_document_id: None, + changed_collections: Default::default(), + batch_size: 0, + batch_ops: 0, + has_assertions: false, + commit_points: Vec::new(), + changelog: Default::default(), } } - pub fn with_change_id(&mut self, change_id: u64) -> &mut Self { - self.ops.push(Operation::ChangeId { change_id }); - self + pub fn init_id_generator(node_number: u16) { + CHANGE_SEQ.store(node_number as u64, Ordering::Relaxed); + } + + fn generate_change_id(&mut self) -> u64 { + let change_id = SnowflakeIdGenerator::from_params( + CHANGE_SEQ.fetch_add(1, Ordering::Relaxed), + *NODE_MUM, + ); + self.current_change_id = Some(change_id); + change_id } pub fn with_account_id(&mut self, account_id: u32) -> &mut Self { - self.ops.push(Operation::AccountId { account_id }); + if self + .current_account_id + .is_none_or(|current_account_id| current_account_id != account_id) + { + if self.current_account_id.is_some() && self.current_change_id.is_some() { + self.serialize_changes(); + } + self.current_account_id = account_id.into(); + self.ops.push(Operation::AccountId { account_id }); + } self } pub fn with_collection(&mut self, collection: impl Into) -> &mut Self { - self.ops.push(Operation::Collection { - collection: collection.into(), - }); + let collection = collection.into(); + let collection_ = Some(collection); + if collection_ != self.current_collection { + self.current_collection = collection_; + self.ops.push(Operation::Collection { collection }); + } self } - pub fn create_document(&mut self) -> &mut Self { - self.ops.push(Operation::DocumentId { - document_id: u32::MAX, - }); - - // Add document id - self.ops.push(Operation::Bitmap { - class: BitmapClass::DocumentIds, - set: true, - }); - - self - } - - pub fn create_document_with_id(&mut self, document_id: u32) -> &mut Self { + pub fn create_document(&mut self, document_id: u32) -> &mut Self { self.ops.push(Operation::DocumentId { document_id }); - - // Add document id self.ops.push(Operation::Bitmap { class: BitmapClass::DocumentIds, set: true, }); - + self.current_document_id = Some(document_id); + self.batch_size += U32_LEN * 3; + self.batch_ops += 1; + self.has_assertions = false; self } pub fn update_document(&mut self, document_id: u32) -> &mut Self { self.ops.push(Operation::DocumentId { document_id }); + self.current_document_id = Some(document_id); + self.has_assertions = false; self } @@ -70,179 +104,255 @@ impl BatchBuilder { class: BitmapClass::DocumentIds, set: false, }); + self.current_document_id = Some(document_id); + self.batch_size += U32_LEN * 3; + self.batch_ops += 1; + self.has_assertions = false; self } pub fn assert_value( &mut self, - class: impl Into>, + class: impl Into, value: impl ToAssertValue, ) -> &mut Self { self.ops.push(Operation::AssertValue { class: class.into(), assert_value: value.to_assert_value(), }); - self - } - - pub fn set_and_index(&mut self, field: impl Into, value: impl Into>) -> &mut Self { - let field = field.into(); - let value = value.into(); - - self.ops.push(Operation::Index { - field, - key: value.clone(), - set: true, - }); - self.ops.push(Operation::Value { - class: ValueClass::Property(field), - op: ValueOp::Set(value.into()), - }); - - self - } - - pub fn unset_and_unindex( - &mut self, - field: impl Into, - value: impl Into>, - ) -> &mut Self { - let field = field.into(); - let value = value.into(); - - self.ops.push(Operation::Index { - field, - key: value, - set: false, - }); - self.ops.push(Operation::Value { - class: ValueClass::Property(field), - op: ValueOp::Clear, - }); - + self.batch_ops += 1; + self.has_assertions = true; self } pub fn index(&mut self, field: impl Into, value: impl Into>) -> &mut Self { let field = field.into(); let value = value.into(); + let value_len = value.len(); self.ops.push(Operation::Index { field, - key: value.clone(), + key: value, set: true, }); + self.batch_size += (U32_LEN * 3) + value_len; + self.batch_ops += 1; self } pub fn unindex(&mut self, field: impl Into, value: impl Into>) -> &mut Self { let field = field.into(); let value = value.into(); + let value_len = value.len(); self.ops.push(Operation::Index { field, - key: value.clone(), + key: value, set: false, }); + self.batch_size += (U32_LEN * 3) + value_len; + self.batch_ops += 1; self } - pub fn tag( - &mut self, - field: impl Into, - value: impl Into>, - ) -> &mut Self { + pub fn tag(&mut self, field: impl Into, value: impl Into) -> &mut Self { + let value = value.into(); + let value_len = value.serialized_size(); self.ops.push(Operation::Bitmap { class: BitmapClass::Tag { field: field.into(), - value: value.into(), + value, }, set: true, }); + self.batch_size += (U32_LEN * 3) + value_len; + self.batch_ops += 1; self } - pub fn untag( - &mut self, - field: impl Into, - value: impl Into>, - ) -> &mut Self { + pub fn untag(&mut self, field: impl Into, value: impl Into) -> &mut Self { + let value = value.into(); + let value_len = value.serialized_size(); self.ops.push(Operation::Bitmap { class: BitmapClass::Tag { field: field.into(), - value: value.into(), + value, }, set: false, }); + self.batch_size += (U32_LEN * 3) + value_len; + self.batch_ops += 1; self } - pub fn tag_many(&mut self, field: impl Into, values: T) -> &mut Self - where - T: Iterator, - V: Into>, - { - let field = field.into(); - for value in values { - self.tag(field, value); - } - self - } - - pub fn untag_many(&mut self, field: impl Into, values: T) -> &mut Self - where - T: Iterator, - V: Into>, - { - let field = field.into(); - for value in values { - self.untag(field, value); - } - self - } - - pub fn add(&mut self, class: impl Into>, value: i64) -> &mut Self { + pub fn add(&mut self, class: impl Into, value: i64) -> &mut Self { + let class = class.into(); + self.batch_size += class.serialized_size() + std::mem::size_of::(); self.ops.push(Operation::Value { - class: class.into(), + class, op: ValueOp::AtomicAdd(value), }); + self.batch_ops += 1; self } - pub fn add_and_get( - &mut self, - class: impl Into>, - value: i64, - ) -> &mut Self { + pub fn add_and_get(&mut self, class: impl Into, value: i64) -> &mut Self { + let class = class.into(); + self.batch_size += class.serialized_size() + (std::mem::size_of::() * 2); self.ops.push(Operation::Value { - class: class.into(), + class, op: ValueOp::AddAndGet(value), }); + self.batch_ops += 1; self } - pub fn set( - &mut self, - class: impl Into>, - value: impl Into, - ) -> &mut Self { + pub fn set(&mut self, class: impl Into, value: impl Into>) -> &mut Self { + let class = class.into(); + let value = value.into(); + self.batch_size += class.serialized_size() + value.len(); self.ops.push(Operation::Value { - class: class.into(), - op: ValueOp::Set(value.into()), + class, + op: ValueOp::Set(value), }); + self.batch_ops += 1; self } - pub fn clear(&mut self, class: impl Into>) -> &mut Self { + pub fn clear(&mut self, class: impl Into) -> &mut Self { + let class = class.into(); + self.batch_size += class.serialized_size(); self.ops.push(Operation::Value { - class: class.into(), + class, op: ValueOp::Clear, }); + self.batch_ops += 1; self } - pub fn log(&mut self, value: impl Into) -> &mut Self { - self.ops.push(Operation::Log { set: value.into() }); + pub fn acl_grant(&mut self, grant_account_id: u32, op: Vec) -> &mut Self { + self.batch_size += (U32_LEN * 3) + op.len(); + self.ops.push(Operation::Value { + class: ValueClass::Acl(grant_account_id), + op: ValueOp::Set(op), + }); + self.batch_ops += 1; + self + } + + pub fn acl_revoke(&mut self, grant_account_id: u32) -> &mut Self { + self.batch_size += U32_LEN * 3; + self.ops.push(Operation::Value { + class: ValueClass::Acl(grant_account_id), + op: ValueOp::Clear, + }); + self.batch_ops += 1; + self + } + + pub fn log_insert(&mut self, prefix: Option) -> &mut Self { + if let (Some(account_id), Some(collection)) = + (self.current_account_id, self.current_collection) + { + self.changed_collections + .get_mut_or_insert(account_id) + .insert(ShortId(collection)); + if let Some(document_id) = self.current_document_id { + self.changelog.log_insert(collection, prefix, document_id); + } + } + if self.current_change_id.is_none() { + self.generate_change_id(); + self.batch_ops += 1; + } + self + } + + pub fn log_update(&mut self, prefix: Option) -> &mut Self { + if let (Some(account_id), Some(collection)) = + (self.current_account_id, self.current_collection) + { + self.changed_collections + .get_mut_or_insert(account_id) + .insert(ShortId(collection)); + if let Some(document_id) = self.current_document_id { + self.changelog.log_update(collection, prefix, document_id); + } + } + if self.current_change_id.is_none() { + self.generate_change_id(); + self.batch_ops += 1; + } + self + } + + pub fn log_delete(&mut self, prefix: Option) -> &mut Self { + if let (Some(account_id), Some(collection)) = + (self.current_account_id, self.current_collection) + { + self.changed_collections + .get_mut_or_insert(account_id) + .insert(ShortId(collection)); + if let Some(document_id) = self.current_document_id { + self.changelog.log_delete(collection, prefix, document_id); + } + } + if self.current_change_id.is_none() { + self.generate_change_id(); + self.batch_ops += 1; + } + self + } + + pub fn log_child_update(&mut self, collection: impl Into, parent_id: u32) -> &mut Self { + let collection = collection.into(); + + if let Some(account_id) = self.current_account_id { + self.changed_collections + .get_mut_or_insert(account_id) + .insert(ShortId(collection)); + } + self.changelog.log_child_update(collection, None, parent_id); + if self.current_change_id.is_none() { + self.generate_change_id(); + self.batch_ops += 1; + } + self + } + + fn serialize_changes(&mut self) { + if let Some(change_id) = self.current_change_id.take() { + if !self.changelog.is_empty() { + for (collection, set) in std::mem::take(&mut self.changelog).serialize() { + self.ops.push(Operation::Log { + change_id, + collection, + set, + }); + } + } + } + } + + pub fn commit_point(&mut self) -> &mut Self { + if self.batch_size > 5_000_000 || self.batch_ops > 1000 { + self.serialize_changes(); + self.commit_points.push(self.ops.len()); + self.batch_ops = 0; + self.batch_size = 0; + if let Some(account_id) = self.current_account_id { + self.ops.push(Operation::AccountId { account_id }); + } + if let Some(collection) = self.current_collection { + self.ops.push(Operation::Collection { collection }); + } + } + self + } + + pub fn any_op(&mut self, op: Operation) -> &mut Self { + self.ops.push(op); + self.batch_ops += 1; self } @@ -251,38 +361,68 @@ impl BatchBuilder { Ok(self) } - pub fn build(self) -> Batch { - Batch { ops: self.ops } + pub fn last_account_id(&self) -> Option { + self.current_account_id } - pub fn build_batch(&mut self) -> Batch { + pub fn change_id(&mut self) -> u64 { + self.current_change_id + .unwrap_or_else(|| self.generate_change_id()) + } + + pub fn last_change_id(&self) -> Option { + self.current_change_id + } + + pub fn build(&mut self) -> impl Iterator> { + self.serialize_changes(); + self.build_batches() + } + + fn build_batches(&self) -> impl Iterator> { + let mut offset_start = 0; + self.commit_points + .iter() + .copied() + .chain([self.ops.len()]) + .map(move |point| { + let batch = Batch { + ops: &self.ops[offset_start..point], + }; + offset_start = point; + batch + }) + } + + pub fn build_all(&mut self) -> Batch<'_> { + self.serialize_changes(); Batch { - ops: std::mem::take(&mut self.ops), + ops: self.ops.as_slice(), } } - pub fn last_account_id(&self) -> Option { - self.ops.iter().rev().find_map(|op| match op { - Operation::AccountId { account_id } => Some(*account_id), - _ => None, - }) + pub fn changed_collections(&mut self) -> impl Iterator)> { + self.changed_collections.iter() + } + + pub fn has_logs(&self) -> bool { + !self.changed_collections.is_empty() + } + + pub fn ops(&self) -> &[Operation] { + self.ops.as_slice() + } + + pub fn len(&self) -> usize { + self.batch_size } pub fn is_empty(&self) -> bool { - self.ops.is_empty() - || !self.ops.iter().any(|op| { - !matches!( - op, - Operation::AccountId { .. } - | Operation::Collection { .. } - | Operation::DocumentId { .. } - | Operation::AssertValue { .. } - ) - }) + self.batch_ops == 0 } } -impl Batch { +impl Batch<'_> { pub fn is_atomic(&self) -> bool { !self.ops.iter().any(|op| { matches!( diff --git a/crates/store/src/write/blob.rs b/crates/store/src/write/blob.rs index df128365..ae2bccc6 100644 --- a/crates/store/src/write/blob.rs +++ b/crates/store/src/write/blob.rs @@ -215,9 +215,9 @@ impl Store { let mut batch = BatchBuilder::new(); let mut last_account_id = u32::MAX; for (account_id, op) in delete_keys.into_iter() { - if batch.ops.len() >= 1000 { + if batch.len() >= 1000 { last_account_id = u32::MAX; - self.write(batch.build()) + self.write(batch.build_all()) .await .caused_by(trc::location!())?; batch = BatchBuilder::new(); @@ -226,13 +226,13 @@ impl Store { batch.with_account_id(account_id); last_account_id = account_id; } - batch.ops.push(Operation::Value { + batch.any_op(Operation::Value { class: ValueClass::Blob(op), op: ValueOp::Clear, - }) + }); } if !batch.is_empty() { - self.write(batch.build()) + self.write(batch.build_all()) .await .caused_by(trc::location!())?; } @@ -290,8 +290,8 @@ impl Store { batch.with_account_id(account_id); let mut last_collection = u8::MAX; for (collection, document_id, op) in delete_keys.into_iter() { - if batch.ops.len() >= 1000 { - self.write(batch.build()) + if batch.len() >= 1000 { + self.write(batch.build_all()) .await .caused_by(trc::location!())?; batch = BatchBuilder::new(); @@ -303,13 +303,13 @@ impl Store { last_collection = collection; } batch.update_document(document_id); - batch.ops.push(Operation::Value { + batch.any_op(Operation::Value { class: ValueClass::Blob(op), op: ValueOp::Clear, }); } if !batch.is_empty() { - self.write(batch.build()) + self.write(batch.build_all()) .await .caused_by(trc::location!())?; } diff --git a/crates/store/src/write/hash.rs b/crates/store/src/write/hash.rs index a15003b6..903c902a 100644 --- a/crates/store/src/write/hash.rs +++ b/crates/store/src/write/hash.rs @@ -8,7 +8,7 @@ use crate::backend::MAX_TOKEN_LENGTH; use super::{BitmapClass, BitmapHash}; -impl BitmapClass { +impl BitmapClass { pub fn word(token: impl AsRef<[u8]>, field: impl Into) -> Self { BitmapClass::Text { field: field.into(), diff --git a/crates/store/src/write/key.rs b/crates/store/src/write/key.rs index cfa17064..4899c24f 100644 --- a/crates/store/src/write/key.rs +++ b/crates/store/src/write/key.rs @@ -19,8 +19,8 @@ use crate::{ }; use super::{ - AnyKey, AssignedIds, BitmapClass, BlobOp, DirectoryClass, InMemoryClass, QueueClass, - ReportClass, ReportEvent, ResolveId, TagValue, TaskQueueClass, TelemetryClass, ValueClass, + AnyKey, BitmapClass, BlobOp, DirectoryClass, InMemoryClass, QueueClass, ReportClass, + ReportEvent, TagValue, TaskQueueClass, TelemetryClass, ValueClass, }; pub struct KeySerializer { @@ -136,13 +136,13 @@ impl DeserializeBigEndian for &[u8] { } } -impl>> ValueKey { +impl> ValueKey { pub fn property( account_id: u32, collection: impl Into, document_id: u32, field: impl Into, - ) -> ValueKey> { + ) -> ValueKey { ValueKey { account_id, collection: collection.into(), @@ -210,30 +210,25 @@ impl Key for LogKey { } } -impl> + Sync + Send + Clone> Key for ValueKey { +impl + Sync + Send + Clone> Key for ValueKey { fn subspace(&self) -> u8 { self.class.as_ref().subspace(self.collection) } fn serialize(&self, flags: u32) -> Vec { - self.class.as_ref().serialize( - self.account_id, - self.collection, - self.document_id, - flags, - None, - ) + self.class + .as_ref() + .serialize(self.account_id, self.collection, self.document_id, flags) } } -impl ValueClass { +impl ValueClass { pub fn serialize( &self, account_id: u32, collection: u8, document_id: u32, flags: u32, - assigned_ids: Option<&AssignedIds>, ) -> Vec { let serializer = if (flags & WITH_SUBSPACE) != 0 { KeySerializer::new(self.serialized_size() + 2).write(self.subspace(collection)) @@ -314,24 +309,19 @@ impl ValueClass { ValueClass::Directory(directory) => match directory { DirectoryClass::NameToId(name) => serializer.write(0u8).write(name.as_slice()), DirectoryClass::EmailToId(email) => serializer.write(1u8).write(email.as_slice()), - DirectoryClass::Principal(uid) => serializer - .write(2u8) - .write_leb128(uid.resolve_id(assigned_ids)), + DirectoryClass::Principal(uid) => serializer.write(2u8).write_leb128(*uid), DirectoryClass::UsedQuota(uid) => serializer.write(4u8).write_leb128(*uid), DirectoryClass::MemberOf { principal_id, member_of, - } => serializer - .write(5u8) - .write(principal_id.resolve_id(assigned_ids)) - .write(member_of.resolve_id(assigned_ids)), + } => serializer.write(5u8).write(*principal_id).write(*member_of), DirectoryClass::Members { principal_id, has_member, } => serializer .write(6u8) - .write(principal_id.resolve_id(assigned_ids)) - .write(has_member.resolve_id(assigned_ids)), + .write(*principal_id) + .write(*has_member), }, ValueClass::Queue(queue) => match queue { QueueClass::Message(queue_id) => serializer.write(*queue_id), @@ -392,6 +382,7 @@ impl ValueClass { .write_leb128(*metric_id) .write_leb128(*node_id), }, + ValueClass::DocumentId => serializer.write(account_id).write(collection), ValueClass::Any(any) => serializer.write(any.key.as_slice()), } .finalize() @@ -422,23 +413,19 @@ impl + Sync + Send + Clone> Key for IndexKey { } } -impl> + Sync + Send + Clone> Key for BitmapKey { +impl + Sync + Send + Clone> Key for BitmapKey { fn subspace(&self) -> u8 { self.class.as_ref().subspace() } fn serialize(&self, flags: u32) -> Vec { - self.class.as_ref().serialize( - self.account_id, - self.collection, - self.document_id, - flags, - None, - ) + self.class + .as_ref() + .serialize(self.account_id, self.collection, self.document_id, flags) } } -impl BitmapClass { +impl BitmapClass { pub fn subspace(&self) -> u8 { match self { BitmapClass::DocumentIds => SUBSPACE_BITMAP_ID, @@ -453,7 +440,6 @@ impl BitmapClass { collection: u8, document_id: u32, flags: u32, - assigned_ids: Option<&AssignedIds>, ) -> Vec { const BM_MARKER: u8 = 1 << 7; @@ -474,7 +460,7 @@ impl BitmapClass { .write(account_id) .write(collection) .write(*field) - .write_leb128(id.resolve_id(assigned_ids)), + .write_leb128(*id), TagValue::Text(text) => if (flags & WITH_SUBSPACE) != 0 { KeySerializer::new(U32_LEN + 4 + text.len()).write(SUBSPACE_BITMAP_TAG) } else { @@ -530,7 +516,7 @@ impl + Sync + Send + Clone> Key for AnyKey { } } -impl ValueClass { +impl ValueClass { pub fn serialized_size(&self) -> usize { match self { ValueClass::Property(_) => U32_LEN * 2 + 3, @@ -573,6 +559,7 @@ impl ValueClass { TelemetryClass::Index { value, .. } => U64_LEN + value.len() + 1, TelemetryClass::Metric { .. } => U64_LEN * 2 + 1, }, + ValueClass::DocumentId => U32_LEN + 1, ValueClass::Any(v) => v.key.len(), } } @@ -619,6 +606,7 @@ impl ValueClass { TelemetryClass::Index { .. } => SUBSPACE_TELEMETRY_INDEX, TelemetryClass::Metric { .. } => SUBSPACE_TELEMETRY_METRIC, }, + ValueClass::DocumentId => SUBSPACE_COUNTER, ValueClass::Any(any) => any.subspace, } } @@ -627,15 +615,16 @@ impl ValueClass { match self { ValueClass::Directory(DirectoryClass::UsedQuota(_)) | ValueClass::InMemory(InMemoryClass::Counter(_)) - | ValueClass::Queue(QueueClass::QuotaCount(_) | QueueClass::QuotaSize(_)) => true, + | ValueClass::Queue(QueueClass::QuotaCount(_) | QueueClass::QuotaSize(_)) + | ValueClass::DocumentId => true, ValueClass::Property(84) if collection == 1 => true, // TODO: Find a more elegant way to do this _ => false, } } } -impl From> for ValueKey> { - fn from(class: ValueClass) -> Self { +impl From for ValueKey { + fn from(class: ValueClass) -> Self { ValueKey { account_id: 0, collection: 0, @@ -645,8 +634,8 @@ impl From> for ValueKey> { } } -impl From> for ValueKey> { - fn from(value: DirectoryClass) -> Self { +impl From for ValueKey { + fn from(value: DirectoryClass) -> Self { ValueKey { account_id: 0, collection: 0, @@ -656,13 +645,13 @@ impl From> for ValueKey> { } } -impl From> for ValueClass { - fn from(value: DirectoryClass) -> Self { +impl From for ValueClass { + fn from(value: DirectoryClass) -> Self { ValueClass::Directory(value) } } -impl From for ValueClass { +impl From for ValueClass { fn from(value: BlobOp) -> Self { ValueClass::Blob(value) } diff --git a/crates/store/src/write/log.rs b/crates/store/src/write/log.rs index 48ffbf8c..0d6abc36 100644 --- a/crates/store/src/write/log.rs +++ b/crates/store/src/write/log.rs @@ -9,11 +9,8 @@ use utils::{codec::leb128::Leb128Vec, map::vec_map::VecMap}; use crate::SerializeInfallible; -use super::{IntoOperations, MaybeDynamicValue, Operation, SerializeWithId}; - #[derive(Default, Debug)] -pub struct ChangeLogBuilder { - pub change_id: u64, +pub(crate) struct ChangeLogBuilder { pub changes: VecMap, } @@ -26,98 +23,43 @@ pub struct Changes { } impl ChangeLogBuilder { - pub fn new() -> ChangeLogBuilder { - ChangeLogBuilder { - change_id: u64::MAX, - changes: VecMap::default(), - } + pub fn serialize(self) -> impl Iterator)> { + self.changes + .into_iter() + .map(|(collection, changes)| (collection, changes.serialize())) } - pub fn with_change_id(change_id: u64) -> ChangeLogBuilder { - ChangeLogBuilder { - change_id, - changes: VecMap::default(), - } - } - - pub fn log_insert(&mut self, collection: impl Into, jmap_id: impl Into) { + pub fn log_insert(&mut self, collection: impl Into, prefix: Option, document_id: u32) { self.changes .get_mut_or_insert(collection.into()) .inserts - .insert(jmap_id.into()); + .insert(build_id(prefix, document_id)); } - pub fn log_update(&mut self, collection: impl Into, jmap_id: impl Into) { + pub fn log_update(&mut self, collection: impl Into, prefix: Option, document_id: u32) { self.changes .get_mut_or_insert(collection.into()) .updates - .insert(jmap_id.into()); + .insert(build_id(prefix, document_id)); } - pub fn log_child_update(&mut self, collection: impl Into, jmap_id: impl Into) { + pub fn log_delete(&mut self, collection: impl Into, prefix: Option, document_id: u32) { + let changes = self.changes.get_mut_or_insert(collection.into()); + let id = build_id(prefix, document_id); + changes.updates.remove(&id); + changes.deletes.insert(id); + } + + pub fn log_child_update( + &mut self, + collection: impl Into, + prefix: Option, + document_id: u32, + ) { self.changes .get_mut_or_insert(collection.into()) .child_updates - .insert(jmap_id.into()); - } - - pub fn log_delete(&mut self, collection: impl Into, jmap_id: impl Into) { - self.changes - .get_mut_or_insert(collection.into()) - .deletes - .insert(jmap_id.into()); - } - - pub fn log_move( - &mut self, - collection: impl Into, - old_jmap_id: impl Into, - new_jmap_id: impl Into, - ) { - let change = self.changes.get_mut_or_insert(collection.into()); - change.deletes.insert(old_jmap_id.into()); - change.inserts.insert(new_jmap_id.into()); - } - - pub fn with_log_insert(mut self, collection: impl Into, jmap_id: impl Into) -> Self { - self.log_insert(collection, jmap_id); - self - } - - pub fn with_log_move( - mut self, - collection: impl Into, - old_jmap_id: impl Into, - new_jmap_id: impl Into, - ) -> Self { - self.log_move(collection, old_jmap_id, new_jmap_id); - self - } - - pub fn with_log_update(mut self, collection: impl Into, jmap_id: impl Into) -> Self { - self.log_update(collection, jmap_id); - self - } - - pub fn with_log_delete(mut self, collection: impl Into, jmap_id: impl Into) -> Self { - self.log_delete(collection, jmap_id); - self - } - - pub fn merge(&mut self, changes: ChangeLogBuilder) { - for (collection, other) in changes.changes { - let this = self.changes.get_mut_or_insert(collection); - for id in other.deletes { - if !this.inserts.remove(&id) { - this.deletes.insert(id); - } - this.updates.remove(&id); - this.child_updates.remove(&id); - } - this.inserts.extend(other.inserts); - this.updates.extend(other.updates); - this.child_updates.extend(other.child_updates); - } + .insert(build_id(prefix, document_id)); } pub fn is_empty(&self) -> bool { @@ -125,16 +67,12 @@ impl ChangeLogBuilder { } } -impl IntoOperations for ChangeLogBuilder { - fn build(self, batch: &mut super::BatchBuilder) -> trc::Result<()> { - batch.with_change_id(self.change_id); - for (collection, changes) in self.changes { - batch.ops.push(Operation::Collection { collection }); - batch.ops.push(Operation::Log { - set: changes.serialize().into(), - }); - } - Ok(()) +#[inline(always)] +fn build_id(prefix: Option, document_id: u32) -> u64 { + if let Some(prefix) = prefix { + ((prefix as u64) << 32) | document_id as u64 + } else { + document_id as u64 } } @@ -213,24 +151,3 @@ impl SerializeInfallible for Changes { buf } } - -impl From for MaybeDynamicValue { - fn from(changes: Changes) -> Self { - MaybeDynamicValue::Static(changes.serialize()) - } -} - -pub struct LogInsert(); - -impl SerializeWithId for LogInsert { - fn serialize_with_id(&self, ids: &super::AssignedIds) -> trc::Result> { - ids.last_document_id() - .map(|id| Changes::insert([id]).serialize()) - } -} - -impl From for MaybeDynamicValue { - fn from(value: LogInsert) -> Self { - MaybeDynamicValue::Dynamic(Box::new(value)) - } -} diff --git a/crates/store/src/write/mod.rs b/crates/store/src/write/mod.rs index 73ea75ba..fa70d9b1 100644 --- a/crates/store/src/write/mod.rs +++ b/crates/store/src/write/mod.rs @@ -5,20 +5,22 @@ */ use std::{ - borrow::Cow, collections::HashSet, - fmt::{self, Formatter}, - hash::Hash, time::{Duration, SystemTime}, }; +use log::ChangeLogBuilder; use nlp::tokenizers::word::WordTokenizer; -use rand::Rng; use rkyv::util::AlignedVec; -use roaring::RoaringBitmap; -use utils::BlobHash; +use utils::{ + BlobHash, + map::{ + bitmap::{Bitmap, ShortId}, + vec_map::VecMap, + }, +}; -use crate::{BlobClass, SerializeInfallible, backend::MAX_TOKEN_LENGTH}; +use crate::{BlobClass, backend::MAX_TOKEN_LENGTH}; use self::assert::AssertValue; @@ -62,32 +64,13 @@ pub struct LegacyBincode { pub inner: T, } -pub trait SerializeWithId: Send + Sync { - fn serialize_with_id(&self, ids: &AssignedIds) -> trc::Result>; -} - -pub trait ResolveId { - fn resolve_id(&self, ids: Option<&AssignedIds>) -> u32; -} - -pub enum MaybeDynamicValue { - Static(Vec), - Dynamic(Box), -} - -#[derive(Debug, PartialEq, Clone, Copy, Eq, Hash)] -pub enum MaybeDynamicId { - Static(u32), - Dynamic(usize), -} - #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub struct DynamicDocumentId(pub usize); #[derive(Debug, Default)] pub struct AssignedIds { - pub document_ids: Vec, pub counter_ids: Vec, + pub change_id: Option, } #[cfg(not(feature = "test_mode"))] @@ -101,13 +84,23 @@ pub(crate) const MAX_COMMIT_ATTEMPTS: u32 = 1000; pub(crate) const MAX_COMMIT_TIME: Duration = Duration::from_secs(3600); #[derive(Debug)] -pub struct Batch { - pub ops: Vec, +pub struct Batch<'x> { + pub(crate) ops: &'x [Operation], } #[derive(Debug)] pub struct BatchBuilder { - pub ops: Vec, + current_change_id: Option, + current_account_id: Option, + current_collection: Option, + current_document_id: Option, + changed_collections: VecMap>, + changelog: ChangeLogBuilder, + has_assertions: bool, + batch_size: usize, + batch_ops: usize, + commit_points: Vec, + ops: Vec, } #[derive(Debug, PartialEq, Eq, Hash)] @@ -121,15 +114,12 @@ pub enum Operation { DocumentId { document_id: u32, }, - ChangeId { - change_id: u64, - }, AssertValue { - class: ValueClass, + class: ValueClass, assert_value: AssertValue, }, Value { - class: ValueClass, + class: ValueClass, op: ValueOp, }, Index { @@ -138,18 +128,20 @@ pub enum Operation { set: bool, }, Bitmap { - class: BitmapClass, + class: BitmapClass, set: bool, }, Log { - set: MaybeDynamicValue, + change_id: u64, + collection: u8, + set: Vec, }, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum BitmapClass { +pub enum BitmapClass { DocumentIds, - Tag { field: u8, value: TagValue }, + Tag { field: u8, value: TagValue }, Text { field: u8, token: BitmapHash }, } @@ -160,25 +152,26 @@ pub struct BitmapHash { } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum TagValue { - Id(T), +pub enum TagValue { + Id(u32), Text(Vec), } #[derive(Debug, PartialEq, Clone, Eq, Hash)] -pub enum ValueClass { +pub enum ValueClass { Property(u8), Acl(u32), InMemory(InMemoryClass), FtsIndex(BitmapHash), TaskQueue(TaskQueueClass), - Directory(DirectoryClass), + Directory(DirectoryClass), Blob(BlobOp), Config(Vec), Queue(QueueClass), Report(ReportClass), Telemetry(TelemetryClass), Any(AnyClass), + DocumentId, } #[derive(Debug, PartialEq, Clone, Eq, Hash)] @@ -207,12 +200,12 @@ pub enum InMemoryClass { } #[derive(Debug, PartialEq, Clone, Eq, Hash)] -pub enum DirectoryClass { +pub enum DirectoryClass { NameToId(Vec), EmailToId(Vec), - MemberOf { principal_id: T, member_of: T }, - Members { principal_id: T, has_member: T }, - Principal(T), + MemberOf { principal_id: u32, member_of: u32 }, + Members { principal_id: u32, has_member: u32 }, + Principal(u32), UsedQuota(u32), } @@ -267,7 +260,7 @@ pub struct ReportEvent { #[derive(Debug, PartialEq, Eq, Hash, Default)] pub enum ValueOp { - Set(MaybeDynamicValue), + Set(Vec), AtomicAdd(i64), AddAndGet(i64), #[default] @@ -288,43 +281,31 @@ pub struct AnyKey> { pub key: T, } -impl From for TagValue { - fn from(value: u32) -> Self { - TagValue::Id(MaybeDynamicId::Static(value)) - } -} - -impl From for TagValue { +impl From for TagValue { fn from(value: u32) -> Self { TagValue::Id(value) } } -impl From> for TagValue { +impl From> for TagValue { fn from(value: Vec) -> Self { TagValue::Text(value) } } -impl From for TagValue { +impl From for TagValue { fn from(value: String) -> Self { TagValue::Text(value.into_bytes()) } } -impl From for TagValue { +impl From for TagValue { fn from(value: u8) -> Self { TagValue::Id(value as u32) } } -impl From for TagValue { - fn from(value: u8) -> Self { - TagValue::Id(MaybeDynamicId::Static(value as u32)) - } -} - -impl From<()> for TagValue { +impl From<()> for TagValue { fn from(_: ()) -> Self { TagValue::Text(vec![]) } @@ -353,17 +334,6 @@ pub trait IntoOperations { fn build(self, batch: &mut BatchBuilder) -> trc::Result<()>; } -impl Operation { - pub fn acl(grant_account_id: u32, set: Option>) -> Self { - Operation::Value { - class: ValueClass::Acl(grant_account_id), - op: set - .map(|op| ValueOp::Set(op.into())) - .unwrap_or(ValueOp::Clear), - } - } -} - #[inline(always)] pub fn now() -> u64 { SystemTime::now() @@ -371,22 +341,22 @@ pub fn now() -> u64 { .map_or(0, |d| d.as_secs()) } -impl AsRef> for ValueClass { - fn as_ref(&self) -> &ValueClass { +impl AsRef for ValueClass { + fn as_ref(&self) -> &ValueClass { self } } -impl AsRef> for BitmapClass { - fn as_ref(&self) -> &BitmapClass { +impl AsRef for BitmapClass { + fn as_ref(&self) -> &BitmapClass { self } } -impl BitmapClass { +impl BitmapClass { pub fn tag_id(property: impl Into, id: u32) -> Self where - TagValue: From, + TagValue: From, { BitmapClass::Tag { field: property.into(), @@ -419,31 +389,15 @@ impl BlobClass { } impl AssignedIds { - pub fn push_document_id(&mut self, id: u32) { - self.document_ids.push(id); - } - pub fn push_counter_id(&mut self, id: i64) { self.counter_ids.push(id); } - pub fn get_document_id(&self, idx: usize) -> trc::Result { - self.document_ids.get(idx).copied().ok_or_else(|| { + pub fn change_id(&self) -> trc::Result { + self.change_id.ok_or_else(|| { trc::StoreEvent::UnexpectedError .caused_by(trc::location!()) - .ctx(trc::Key::Reason, "No document ids were created") - }) - } - - pub fn first_document_id(&self) -> trc::Result { - self.get_document_id(0) - } - - pub fn last_document_id(&self) -> trc::Result { - self.document_ids.last().copied().ok_or_else(|| { - trc::StoreEvent::UnexpectedError - .caused_by(trc::location!()) - .ctx(trc::Key::Reason, "No document ids were created") + .ctx(trc::Key::Reason, "No change id was assigned") }) } @@ -456,131 +410,6 @@ impl AssignedIds { } } -impl From for MaybeDynamicValue { - fn from(value: String) -> Self { - MaybeDynamicValue::Static(value.into_bytes()) - } -} - -impl From<&[u8]> for MaybeDynamicValue { - fn from(value: &[u8]) -> Self { - MaybeDynamicValue::Static(value.to_vec()) - } -} - -impl From> for MaybeDynamicValue { - fn from(value: Vec) -> Self { - MaybeDynamicValue::Static(value) - } -} - -impl MaybeDynamicValue { - pub fn resolve(&self, ids: &AssignedIds) -> trc::Result> { - match self { - MaybeDynamicValue::Static(value) => Ok(Cow::Borrowed(value.as_slice())), - MaybeDynamicValue::Dynamic(value) => value.serialize_with_id(ids).map(Cow::Owned), - } - } -} - -impl MaybeDynamicId { - pub fn resolve(&self, ids: &AssignedIds) -> trc::Result { - match self { - MaybeDynamicId::Static(id) => Ok(*id), - MaybeDynamicId::Dynamic(idx) => ids.get_document_id(*idx), - } - } -} - -impl ResolveId for u32 { - fn resolve_id(&self, _: Option<&AssignedIds>) -> u32 { - *self - } -} - -impl ResolveId for MaybeDynamicId { - fn resolve_id(&self, ids: Option<&AssignedIds>) -> u32 { - match self { - MaybeDynamicId::Static(id) => *id, - MaybeDynamicId::Dynamic(idx) => ids - .and_then(|ids| ids.document_ids.get(*idx)) - .copied() - .unwrap_or(u32::MAX), - } - } -} - -impl std::fmt::Debug for MaybeDynamicValue { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - match self { - MaybeDynamicValue::Static(value) => write!(f, "{:?}", value), - MaybeDynamicValue::Dynamic(_) => write!(f, "Dynamic"), - } - } -} - -impl PartialEq for MaybeDynamicValue { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - (MaybeDynamicValue::Static(a), MaybeDynamicValue::Static(b)) => a == b, - (MaybeDynamicValue::Dynamic(_), MaybeDynamicValue::Dynamic(_)) => true, - _ => false, - } - } -} - -impl Eq for MaybeDynamicValue {} - -impl Hash for MaybeDynamicValue { - fn hash(&self, state: &mut H) { - match self { - MaybeDynamicValue::Static(value) => value.hash(state), - MaybeDynamicValue::Dynamic(_) => 0.hash(state), - } - } -} - -impl From for MaybeDynamicValue { - fn from(value: MaybeDynamicId) -> Self { - match value { - MaybeDynamicId::Static(id) => MaybeDynamicValue::Static(id.serialize()), - MaybeDynamicId::Dynamic(idx) => { - MaybeDynamicValue::Dynamic(Box::new(DynamicDocumentId(idx))) - } - } - } -} - -impl SerializeWithId for DynamicDocumentId { - fn serialize_with_id(&self, ids: &AssignedIds) -> trc::Result> { - ids.get_document_id(self.0).map(|id| id.serialize()) - } -} - -pub(crate) trait RandomAvailableId { - fn random_available_id(&self) -> u32; -} - -impl RandomAvailableId for RoaringBitmap { - fn random_available_id(&self) -> u32 { - let mut last_id = 0; - let mut available_ids = Vec::with_capacity(100); - for id in self.iter() { - for i in last_id..id { - available_ids.push(i); - } - last_id = id + 1; - } - - while available_ids.len() < 100 { - available_ids.push(last_id); - last_id += 1; - } - - available_ids[rand::rng().random_range(0..available_ids.len())] - } -} - impl QueueClass { pub fn due(&self) -> Option { match self { @@ -596,3 +425,12 @@ impl> AsRef<[u8]> for Archive { self.inner.as_ref() } } + +impl TagValue { + pub fn serialized_size(&self) -> usize { + match self { + TagValue::Id(_) => std::mem::size_of::(), + TagValue::Text(items) => items.len(), + } + } +} diff --git a/crates/utils/src/map/bitmap.rs b/crates/utils/src/map/bitmap.rs index 368d2126..29a038c5 100644 --- a/crates/utils/src/map/bitmap.rs +++ b/crates/utils/src/map/bitmap.rs @@ -106,6 +106,20 @@ impl Bitmap { false } + #[inline(always)] + pub fn contains_all(&self, items: impl Iterator) -> bool { + if !self.is_empty() { + for item in items { + if self.bitmap & (1 << item.into()) == 0 { + return false; + } + } + true + } else { + false + } + } + #[inline(always)] pub fn is_empty(&self) -> bool { self.bitmap == 0 @@ -245,3 +259,29 @@ impl Default for Bitmap { } } } + +#[repr(transparent)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct ShortId(pub u8); + +impl BitmapItem for ShortId { + fn max() -> u64 { + u8::MAX as u64 + } + + fn is_valid(&self) -> bool { + true + } +} + +impl From for ShortId { + fn from(value: u64) -> Self { + ShortId(value as u8) + } +} + +impl From for u64 { + fn from(value: ShortId) -> Self { + value.0 as u64 + } +} diff --git a/crates/utils/src/snowflake.rs b/crates/utils/src/snowflake.rs index 79edc586..4debe77c 100644 --- a/crates/utils/src/snowflake.rs +++ b/crates/utils/src/snowflake.rs @@ -22,11 +22,14 @@ const NODE_ID_LEN: u64 = 9; const SEQUENCE_MASK: u64 = (1 << SEQUENCE_LEN) - 1; const NODE_ID_MASK: u64 = (1 << NODE_ID_LEN) - 1; +const DEFAULT_EPOCH: u64 = 1632280000; // 52 years after UNIX_EPOCH +const DEFAULT_EPOCH_MS: u128 = (DEFAULT_EPOCH as u128) * 1000; // 52 years after UNIX_EPOCH in milliseconds + /* ID characteristics: -- 43 bits for milliseconds since January 1st, 2022: 2^43 / (1000 * 60 * 60 * 24 * 365) = 278.92 years +- 43 bits for milliseconds since January 1st, 2022: 2^43 / (1000 * 60 * 60 * 24 * 365) = 278.92 years (from year 2022 until 2300) - 9 bits for a node id: 2^9 = 512 nodes - 12 bits for a sequence number: 2^12 = 4096 ids per millisecond @@ -38,7 +41,7 @@ impl SnowflakeIdGenerator { } pub fn from_duration(period: Duration) -> Option { - (SystemTime::UNIX_EPOCH + Duration::from_secs(1632280000)) + (SystemTime::UNIX_EPOCH + Duration::from_secs(DEFAULT_EPOCH)) .elapsed() .ok() .and_then(|elapsed| elapsed.checked_sub(period)) @@ -55,7 +58,7 @@ impl SnowflakeIdGenerator { pub fn with_node_id(node_id: u64) -> Self { Self { - epoch: SystemTime::UNIX_EPOCH + Duration::from_secs(1632280000), // 52 years after UNIX_EPOCH + epoch: SystemTime::UNIX_EPOCH + Duration::from_secs(DEFAULT_EPOCH), // 52 years after UNIX_EPOCH node_id, sequence: 0.into(), } @@ -70,15 +73,35 @@ impl SnowflakeIdGenerator { .map(|elapsed| (elapsed.as_millis() as u64) << (SEQUENCE_LEN + NODE_ID_LEN)) } + pub fn is_valid(&self) -> bool { + self.epoch.elapsed().is_ok() + } + #[inline(always)] - pub fn generate(&self) -> Option { - let elapsed = self.epoch.elapsed().ok()?.as_millis() as u64; + pub fn generate(&self) -> u64 { + let elapsed = self + .epoch + .elapsed() + .map(|e| e.as_millis()) + .unwrap_or_default() as u64; let sequence = self.sequence.fetch_add(1, Ordering::Relaxed); - ((elapsed << (SEQUENCE_LEN + NODE_ID_LEN)) + (elapsed << (SEQUENCE_LEN + NODE_ID_LEN)) | ((self.node_id & NODE_ID_MASK) << SEQUENCE_LEN) - | (sequence & SEQUENCE_MASK)) - .into() + | (sequence & SEQUENCE_MASK) + } + + #[inline(always)] + pub fn from_params(sequence: u64, node_id: u16) -> u64 { + let elapsed = SystemTime::UNIX_EPOCH + .elapsed() + .map(|e| e.as_millis()) + .unwrap_or_default() + .saturating_sub(DEFAULT_EPOCH_MS) as u64; + + (elapsed << (SEQUENCE_LEN + NODE_ID_LEN)) + | (((node_id as u64) & NODE_ID_MASK) << SEQUENCE_LEN) + | (sequence & SEQUENCE_MASK) } } diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 31739f5f..939aafb1 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -5,8 +5,8 @@ edition = "2024" resolver = "2" [features] -#default = ["sqlite", "postgres", "mysql", "rocks", "elastic", "s3", "redis", "azure", "foundationdb"] -default = ["sqlite", "postgres", "mysql", "rocks", "s3", "redis"] +default = ["sqlite", "postgres", "mysql", "rocks", "elastic", "s3", "redis", "azure", "foundationdb"] +#default = ["sqlite", "postgres", "mysql", "rocks", "s3", "redis"] #default = ["rocks", "redis", "s3"] sqlite = ["store/sqlite"] foundationdb = ["store/foundation", "common/foundation"] diff --git a/tests/src/directory/internal.rs b/tests/src/directory/internal.rs index e6fee5b7..8f3eaff9 100644 --- a/tests/src/directory/internal.rs +++ b/tests/src/directory/internal.rs @@ -629,17 +629,19 @@ async fn internal_directory() { let mut document_id = u32::MAX; for account_id in [john_id, jane_id] { document_id = store + .assign_document_ids(u32::MAX, Collection::Principal, 1) + .await + .unwrap(); + store .write( BatchBuilder::new() .with_account_id(account_id) .with_collection(Collection::Email) - .create_document() + .create_document(document_id) .set(ValueClass::Property(0), "hello".as_bytes()) - .build_batch(), + .build_all(), ) .await - .unwrap() - .last_document_id() .unwrap(); assert_eq!( store diff --git a/tests/src/jmap/email_changes.rs b/tests/src/jmap/email_changes.rs index a8e6eb68..aa9c8f71 100644 --- a/tests/src/jmap/email_changes.rs +++ b/tests/src/jmap/email_changes.rs @@ -8,10 +8,7 @@ use jmap_proto::{ parser::{JsonObjectParser, json::Parser}, types::{collection::Collection, id::Id, state::State}, }; -use store::{ - ahash::AHashSet, - write::{BatchBuilder, log::ChangeLogBuilder}, -}; +use store::{ahash::AHashSet, write::BatchBuilder}; use crate::jmap::assert_is_empty; @@ -24,7 +21,7 @@ pub async fn test(params: &mut JMAPTest) { params.client.set_default_account_id(Id::new(1)); let mut states = vec![State::Initial]; - for (change_id, (changes, expected_changelog)) in [ + for (changes, expected_changelog) in [ ( vec![ LogAction::Insert(0), @@ -135,18 +132,30 @@ pub async fn test(params: &mut JMAPTest) { ), ] .into_iter() - .enumerate() { - let mut changelog = ChangeLogBuilder::with_change_id(change_id as u64); + let mut batch = BatchBuilder::new(); + batch.with_account_id(1).with_collection(Collection::Email); for change in changes { match change { - LogAction::Insert(id) => changelog.log_insert(Collection::Email, id), - LogAction::Update(id) => changelog.log_update(Collection::Email, id), - LogAction::Delete(id) => changelog.log_delete(Collection::Email, id), - LogAction::UpdateChild(id) => changelog.log_child_update(Collection::Email, id), + LogAction::Insert(id) => { + batch.update_document(id as u32).log_insert(None); + } + LogAction::Update(id) => { + batch.update_document(id as u32).log_update(None); + } + LogAction::Delete(id) => { + batch.update_document(id as u32).log_delete(None); + } + LogAction::UpdateChild(id) => { + batch.log_child_update(Collection::Email, id as u32); + } LogAction::Move(old_id, new_id) => { - changelog.log_move(Collection::Email, old_id, new_id) + batch + .update_document(old_id as u32) + .log_delete(None) + .update_document(new_id as u32) + .log_insert(None); } } } @@ -155,14 +164,7 @@ pub async fn test(params: &mut JMAPTest) { .core .storage .data - .write( - BatchBuilder::new() - .with_account_id(1) - .with_collection(Collection::Email) - .custom(changelog) - .unwrap() - .build_batch(), - ) + .write(batch.build_all()) .await .unwrap(); diff --git a/tests/src/jmap/email_query.rs b/tests/src/jmap/email_query.rs index 0b445fb3..d5d6d12b 100644 --- a/tests/src/jmap/email_query.rs +++ b/tests/src/jmap/email_query.rs @@ -11,6 +11,8 @@ use crate::{ store::{deflate_test_resource, query::FIELDS}, }; +use ::email::thread::cache::ThreadCache; +use ahash::AHashSet; use jmap_client::{ client::Client, core::query::{Comparator, Filter}, @@ -45,9 +47,15 @@ pub async fn test(params: &mut JMAPTest, insert: bool) { .with_account_id(account_id) .with_collection(Collection::Mailbox); for mailbox_id in 1545..3010 { - batch.create_document_with_id(mailbox_id); + batch.create_document(mailbox_id); } - server.core.storage.data.write(batch.build()).await.unwrap(); + server + .core + .storage + .data + .write(batch.build_all()) + .await + .unwrap(); // Create test messages println!("Inserting JMAP Mail query test messages..."); @@ -63,16 +71,25 @@ pub async fn test(params: &mut JMAPTest, insert: bool) { .delete_document(mailbox_id) .clear(ValueClass::Property(Property::EmailIds.into())); } - server.core.storage.data.write(batch.build()).await.unwrap(); + server + .core + .storage + .data + .write(batch.build_all()) + .await + .unwrap(); assert_eq!( params .server - .get_document_ids(account_id, Collection::Thread) + .get_cached_thread_ids(account_id) .await .unwrap() - .unwrap() - .len() as usize, + .threads + .values() + .copied() + .collect::>() + .len(), MAX_THREADS ); diff --git a/tests/src/jmap/email_query_changes.rs b/tests/src/jmap/email_query_changes.rs index e4ef1496..99ad3703 100644 --- a/tests/src/jmap/email_query_changes.rs +++ b/tests/src/jmap/email_query_changes.rs @@ -4,18 +4,18 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use ::email::message::{ingest::EmailIngest, metadata::MessageData}; +use ::email::message::metadata::MessageData; use common::storage::index::ObjectIndexBuilder; use jmap_client::{ core::query::{Comparator, Filter}, email, mailbox::Role, }; -use jmap_proto::types::{collection::Collection, id::Id, property::Property, state::State}; +use jmap_proto::types::{collection::Collection, id::Id, state::State}; use store::{ ahash::{AHashMap, AHashSet}, - write::{AlignedBytes, Archive, BatchBuilder, log::ChangeLogBuilder}, + write::BatchBuilder, }; use crate::jmap::{ @@ -120,9 +120,11 @@ pub async fn test(params: &mut JMAPTest) { } LogAction::Update(id) => { let id = *id_map.get(id).unwrap(); - let mut changelog = ChangeLogBuilder::new(); - changelog.log_update(Collection::Email, id); - server.commit_changes(1, changelog).await.unwrap(); + let mut batch = BatchBuilder::new(); + batch + .update_document(id.document_id()) + .log_update(id.prefix_id().into()); + server.store().write(batch.build_all()).await.unwrap(); updated_ids.insert(id); } LogAction::Delete(id) => { @@ -134,21 +136,16 @@ pub async fn test(params: &mut JMAPTest) { let id = *id_map.get(from).unwrap(); let new_id = Id::from_parts(thread_id, id.document_id()); - let new_thread_id = server.create_thread_id(1).await.unwrap(); + //let new_thread_id = store::rand::random::(); let old_message_ = server - .get_property::>( - 1, - Collection::Email, - id.document_id(), - Property::Value, - ) + .get_archive(1, Collection::Email, id.document_id()) .await .unwrap() .unwrap(); let old_message = old_message_.to_unarchived::().unwrap(); - let mut new_message = old_message.deserialize().unwrap(); - new_message.thread_id = new_thread_id; + let mut new_message = old_message.deserialize::().unwrap(); + new_message.thread_id = thread_id; server .core @@ -157,8 +154,6 @@ pub async fn test(params: &mut JMAPTest) { .write( BatchBuilder::new() .with_account_id(1) - .with_collection(Collection::Thread) - .create_document() .with_collection(Collection::Email) .update_document(id.document_id()) .custom( @@ -167,13 +162,7 @@ pub async fn test(params: &mut JMAPTest) { .with_changes(new_message), ) .unwrap() - .custom(server.begin_changes(1).unwrap().with_log_move( - Collection::Email, - id, - new_id, - )) - .unwrap() - .build_batch(), + .build_all(), ) .await .unwrap(); @@ -250,9 +239,9 @@ pub async fn test(params: &mut JMAPTest) { let id = Id::from_bytes(id.as_bytes()).unwrap(); assert!( removed_ids.contains(&id), - "{:?} (id: {})", + "{:?} (id: {:?})", changes, - id_map.iter().find(|(_, v)| **v == id).unwrap().0 + id_map.iter().find(|(_, v)| **v == id).map(|(k, _)| k) ); } } @@ -262,9 +251,9 @@ pub async fn test(params: &mut JMAPTest) { let id = Id::from_bytes(item.id().as_bytes()).unwrap(); assert!( type1_ids.contains(&id), - "{:?} (id: {})", + "{:?} (id: {:?})", changes, - id_map.iter().find(|(_, v)| **v == id).unwrap().0 + id_map.iter().find(|(_, v)| **v == id).map(|(k, _)| k) ); } } @@ -286,26 +275,6 @@ pub async fn test(params: &mut JMAPTest) { } destroy_all_mailboxes(params).await; - - // Delete virtual threads - let mut batch = BatchBuilder::new(); - batch.with_account_id(1).with_collection(Collection::Thread); - for thread_id in server - .get_document_ids(1, Collection::Thread) - .await - .unwrap() - .unwrap_or_default() - { - batch.delete_document(thread_id); - } - server - .core - .storage - .data - .write(batch.build_batch()) - .await - .unwrap(); - assert_is_empty(server).await; } diff --git a/tests/src/jmap/email_submission.rs b/tests/src/jmap/email_submission.rs index 5d3a90e9..e1452677 100644 --- a/tests/src/jmap/email_submission.rs +++ b/tests/src/jmap/email_submission.rs @@ -94,7 +94,7 @@ pub async fn test(params: &mut JMAPTest) { // Test automatic identity creation client.set_default_account_id(&account_id); - for (identity_id, email) in [(0u64, "jdoe@example.com"), (1u64, "john.doe@example.com")] { + for (identity_id, email) in [(2u64, "jdoe@example.com"), (1u64, "john.doe@example.com")] { let identity = client .identity_get(&Id::from(identity_id).to_string(), None) .await @@ -482,8 +482,8 @@ pub async fn test(params: &mut JMAPTest) { // Destroy the created mailbox, identity and all submissions for identity_id in [ identity_id, - Id::from(0u64).to_string(), Id::from(1u64).to_string(), + Id::from(2u64).to_string(), ] { client.identity_destroy(&identity_id).await.unwrap(); } diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 5c6bac96..5194cdf8 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -375,7 +375,7 @@ pub async fn jmap_tests() { webhooks::test(&mut params).await; email_query::test(&mut params, delete).await; - /*email_get::test(&mut params).await; + email_get::test(&mut params).await; email_set::test(&mut params).await; email_parse::test(&mut params).await; email_search_snippet::test(&mut params).await; @@ -386,7 +386,7 @@ pub async fn jmap_tests() { thread_merge::test(&mut params).await; mailbox::test(&mut params).await; delivery::test(&mut params).await; - auth_acl::test(&mut params).await;*/ + auth_acl::test(&mut params).await; auth_limits::test(&mut params).await; auth_oauth::test(&mut params).await; event_source::test(&mut params).await; @@ -451,7 +451,7 @@ pub async fn wait_for_index(server: &Server) { .data .iterate( IterateParams::new( - ValueKey::> { + ValueKey:: { account_id: 0, collection: 0, document_id: 0, @@ -460,7 +460,7 @@ pub async fn wait_for_index(server: &Server) { hash: BlobHash::default(), }), }, - ValueKey::> { + ValueKey:: { account_id: u32::MAX, collection: u8::MAX, document_id: u32::MAX, diff --git a/tests/src/jmap/stress_test.rs b/tests/src/jmap/stress_test.rs index e09fdd1d..a989c547 100644 --- a/tests/src/jmap/stress_test.rs +++ b/tests/src/jmap/stress_test.rs @@ -17,10 +17,7 @@ use jmap_client::{ mailbox::{self, Mailbox, Role}, }; use jmap_proto::types::{collection::Collection, id::Id, property::Property}; -use store::{ - rand::{self, Rng}, - write::{AlignedBytes, Archive}, -}; +use store::rand::{self, Rng}; use super::assert_is_empty; @@ -232,12 +229,7 @@ async fn email_tests(server: Server, client: Arc) { for email_id in &email_ids_in_mailbox { if let Some(mailbox_tags) = server - .get_property::>( - TEST_USER_ID, - Collection::Email, - email_id, - &Property::MailboxIds, - ) + .get_archive(TEST_USER_ID, Collection::Email, email_id) .await .unwrap() { diff --git a/tests/src/jmap/webhooks.rs b/tests/src/jmap/webhooks.rs index 1e0dad34..c09d3f20 100644 --- a/tests/src/jmap/webhooks.rs +++ b/tests/src/jmap/webhooks.rs @@ -93,7 +93,7 @@ pub fn spawn_mock_webhook_endpoint() -> Arc { let listener = TcpListener::bind("127.0.0.1:8821") .await .unwrap_or_else(|e| { - panic!("Failed to bind mock Milter server to 127.0.0.1:8821: {e}"); + panic!("Failed to bind mock Webhooks server to 127.0.0.1:8821: {e}"); }); let mut rx_ = rx.clone(); @@ -113,7 +113,7 @@ pub fn spawn_mock_webhook_endpoint() -> Arc { async move { // Verify HMAC signature let key = hmac::Key::new(hmac::HMAC_SHA256, "ovos-moles".as_bytes()); - let body = fetch_body(&mut req, 1024 * 1024, 0).await.unwrap(); + let body = fetch_body(&mut req, usize::MAX, 0).await.unwrap(); let tag = STANDARD.decode(req.headers().get("X-Signature").unwrap().to_str().unwrap()).unwrap(); hmac::verify(&key, &body, &tag).expect("Invalid signature"); diff --git a/tests/src/smtp/session.rs b/tests/src/smtp/session.rs index 00562bf8..635f61cb 100644 --- a/tests/src/smtp/session.rs +++ b/tests/src/smtp/session.rs @@ -259,7 +259,7 @@ impl TestSession for Session { dsn_info: None, }, ], - self.server.inner.data.queue_id_gen.generate().unwrap(), + self.server.inner.data.queue_id_gen.generate(), 0, ) .await; diff --git a/tests/src/store/assign_id.rs b/tests/src/store/assign_id.rs deleted file mode 100644 index 2cf3ef77..00000000 --- a/tests/src/store/assign_id.rs +++ /dev/null @@ -1,108 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use std::collections::HashSet; - -use store::{Store, write::BatchBuilder}; - -pub async fn test(db: Store) { - println!("Running Store ID assignment tests..."); - - test_0(db).await; -} - -async fn test_0(db: Store) { - // Test document id assignment - println!("Creating 1000 documentIds concurrently..."); - let mut handles = Vec::new(); - let mut assigned_ids = HashSet::new(); - - // Create 1000 ids concurrently - for _ in 0..1000 { - handles.push({ - let db = db.clone(); - tokio::spawn(async move { - db.write( - BatchBuilder::new() - .with_account_id(0) - .with_collection(u8::MAX) - .create_document() - .build_batch(), - ) - .await - .unwrap() - .last_document_id() - .unwrap() - }) - }); - } - - for handle in handles { - let assigned_id = handle.await.unwrap(); - assert!( - assigned_ids.insert(assigned_id), - "already assigned or invalid: {assigned_id}" - ); - } - assert_eq!(assigned_ids.len(), 1000); - - // Create 1000 ids concurrently - println!("Deleting 1000 documentIds concurrently..."); - let mut handles = Vec::new(); - for document_id in assigned_ids { - let db = db.clone(); - handles.push({ - tokio::spawn(async move { - db.write( - BatchBuilder::new() - .with_account_id(0) - .with_collection(u8::MAX) - .delete_document(document_id) - .build_batch(), - ) - .await - .unwrap(); - }) - }); - } - for handle in handles { - handle.await.unwrap(); - } - - // Reuse 1000 ids concurrently - println!("Reusing 1000 freed documentIds concurrently..."); - let mut handles = Vec::new(); - let mut assigned_ids = HashSet::new(); - for _ in 0..1000 { - handles.push({ - let db = db.clone(); - tokio::spawn(async move { - db.write( - BatchBuilder::new() - .with_account_id(0) - .with_collection(u8::MAX) - .create_document() - .build_batch(), - ) - .await - .unwrap() - .last_document_id() - .unwrap() - }) - }); - } - - for handle in handles { - let assigned_id = handle.await.unwrap(); - assert!( - assigned_ids.insert(assigned_id), - "freed id already assigned or invalid: {assigned_id}" - ); - } - assert_eq!(assigned_ids.len(), 1000); - - db.destroy().await; -} diff --git a/tests/src/store/blob.rs b/tests/src/store/blob.rs index eb89049e..eddb5659 100644 --- a/tests/src/store/blob.rs +++ b/tests/src/store/blob.rs @@ -51,7 +51,7 @@ pub async fn blob_tests() { }, 1024u32.serialize(), ) - .build_batch(), + .build_all(), ) .await .unwrap(); @@ -67,7 +67,7 @@ pub async fn blob_tests() { .write( BatchBuilder::new() .set(BlobOp::Commit { hash: hash.clone() }, Vec::new()) - .build_batch(), + .build_all(), ) .await .unwrap(); @@ -180,7 +180,7 @@ pub async fn blob_tests() { .update_document(document_id as u32) .set(blob_op, blob_value) .set(BlobOp::Commit { hash: hash.clone() }, vec![]) - .build_batch(), + .build_all(), ) .await .unwrap(); @@ -294,7 +294,7 @@ pub async fn blob_tests() { .clear(BlobOp::Link { hash: BlobHash::generate(b"789".as_slice()), }) - .build_batch(), + .build_all(), ) .await .unwrap(); diff --git a/tests/src/store/import_export.rs b/tests/src/store/import_export.rs index cca42a83..13898efc 100644 --- a/tests/src/store/import_export.rs +++ b/tests/src/store/import_export.rs @@ -11,7 +11,7 @@ use store::{ rand, write::{ AnyKey, BatchBuilder, BitmapClass, BitmapHash, BlobOp, DirectoryClass, InMemoryClass, - MaybeDynamicId, MaybeDynamicValue, Operation, QueueClass, QueueEvent, TagValue, ValueClass, + Operation, QueueClass, QueueEvent, TagValue, ValueClass, }, *, }; @@ -44,7 +44,7 @@ pub async fn test(db: Store) { .unwrap(); batch.set(ValueClass::Blob(BlobOp::Commit { hash }), vec![]); } - db.write(batch.build()).await.unwrap(); + db.write(batch.build_all()).await.unwrap(); // Create account data println!("Creating account data..."); @@ -57,7 +57,7 @@ pub async fn test(db: Store) { batch.with_collection(collection); for document_id in [0, 10, 20, 30, 40] { - batch.create_document_with_id(document_id); + batch.create_document(document_id); if collection == u8::from(Collection::Mailbox) { batch @@ -98,28 +98,30 @@ pub async fn test(db: Store) { ); } - batch.ops.push(Operation::ChangeId { + batch.log_insert(None); + + /*batch.any_op(Operation::ChangeId { change_id: document_id as u64 + account_id as u64 + collection as u64, }); - batch.ops.push(Operation::Log { + batch.any_op(Operation::Log { set: MaybeDynamicValue::Static(vec![ account_id as u8, collection, document_id as u8, ]), - }); + });*/ for field in 0..5 { - batch.ops.push(Operation::Bitmap { + batch.any_op(Operation::Bitmap { class: BitmapClass::Tag { field, - value: TagValue::Id(MaybeDynamicId::Static(rand::random())), + value: TagValue::Id(rand::random()), }, set: true, }); - batch.ops.push(Operation::Bitmap { + batch.any_op(Operation::Bitmap { class: BitmapClass::Tag { field, value: TagValue::Text(random_bytes(field as usize + 2)), @@ -127,7 +129,7 @@ pub async fn test(db: Store) { set: true, }); - batch.ops.push(Operation::Bitmap { + batch.any_op(Operation::Bitmap { class: BitmapClass::Text { field, token: BitmapHash::new(random_bytes(field as usize + 2)), @@ -135,7 +137,7 @@ pub async fn test(db: Store) { set: true, }); - batch.ops.push(Operation::Index { + batch.any_op(Operation::Index { field, key: random_bytes(field as usize + 2), set: true, @@ -144,7 +146,7 @@ pub async fn test(db: Store) { } } - db.write(batch.build()).await.unwrap(); + db.write(batch.build_all()).await.unwrap(); } // Create queue, config and lookup data @@ -175,7 +177,7 @@ pub async fn test(db: Store) { random_bytes(idx + 10), ); } - db.write(batch.build()).await.unwrap(); + db.write(batch.build_all()).await.unwrap(); // Create directory data println!("Creating directory data..."); @@ -186,7 +188,7 @@ pub async fn test(db: Store) { for account_id in [1, 2, 3, 4, 5] { batch - .create_document_with_id(account_id) + .create_document(account_id) .add( ValueClass::Directory(DirectoryClass::UsedQuota(account_id)), rand::random(), @@ -204,27 +206,25 @@ pub async fn test(db: Store) { random_bytes(4), ) .set( - ValueClass::Directory(DirectoryClass::Principal(MaybeDynamicId::Static( - account_id, - ))), + ValueClass::Directory(DirectoryClass::Principal(account_id)), random_bytes(30), ) .set( ValueClass::Directory(DirectoryClass::MemberOf { - principal_id: MaybeDynamicId::Static(account_id), - member_of: MaybeDynamicId::Static(rand::random()), + principal_id: account_id, + member_of: rand::random(), }), random_bytes(15), ) .set( ValueClass::Directory(DirectoryClass::Members { - principal_id: MaybeDynamicId::Static(account_id), - has_member: MaybeDynamicId::Static(rand::random()), + principal_id: account_id, + has_member: rand::random(), }), random_bytes(15), ); } - db.write(batch.build()).await.unwrap(); + db.write(batch.build_all()).await.unwrap(); // Obtain store hash println!("Calculating store hash..."); diff --git a/tests/src/store/mod.rs b/tests/src/store/mod.rs index cc789182..c8391be5 100644 --- a/tests/src/store/mod.rs +++ b/tests/src/store/mod.rs @@ -4,7 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -pub mod assign_id; pub mod blob; pub mod import_export; pub mod lookup; @@ -97,7 +96,6 @@ pub async fn store_tests() { } import_export::test(store.clone()).await; - assign_id::test(store.clone()).await; ops::test(store.clone()).await; query::test(store.clone(), FtsStore::Store(store.clone()), insert).await; diff --git a/tests/src/store/ops.rs b/tests/src/store/ops.rs index b6547ace..595571f0 100644 --- a/tests/src/store/ops.rs +++ b/tests/src/store/ops.rs @@ -6,10 +6,9 @@ use std::collections::HashSet; -use jmap_proto::types::{collection::Collection, property::Property}; use store::{ - BitmapKey, Store, ValueKey, - write::{BatchBuilder, BitmapClass, DirectoryClass, MaybeDynamicId, TagValue, ValueClass}, + Store, ValueKey, + write::{BatchBuilder, DirectoryClass, ValueClass}, }; // FDB max value @@ -33,7 +32,7 @@ pub async fn test(db: Store) { ); if n % 10000 == 0 { - db.write(batch.build_batch()).await.unwrap(); + db.write(batch.build_all()).await.unwrap(); batch = BatchBuilder::new(); batch .with_account_id(0) @@ -41,7 +40,7 @@ pub async fn test(db: Store) { .update_document(0); } } - db.write(batch.build_batch()).await.unwrap(); + db.write(batch.build_all()).await.unwrap(); println!("Created 900.000 keys..."); // Iterate over all keys @@ -85,7 +84,7 @@ pub async fn test(db: Store) { batch.clear(ValueClass::Config(format!("key{n:10}").into_bytes())); if n % 10000 == 0 { - db.write(batch.build_batch()).await.unwrap(); + db.write(batch.build_all()).await.unwrap(); batch = BatchBuilder::new(); batch .with_account_id(0) @@ -93,94 +92,9 @@ pub async fn test(db: Store) { .update_document(0); } } - db.write(batch.build_batch()).await.unwrap(); + db.write(batch.build_all()).await.unwrap(); } - // Testing ID assignment - println!("Running dynamic ID assignment tests..."); - let mut builder = BatchBuilder::new(); - builder - .with_account_id(0) - .with_collection(Collection::Thread) - .create_document() - .with_collection(Collection::Email) - .create_document() - .tag(Property::ThreadId, TagValue::Id(MaybeDynamicId::Dynamic(0))) - .set(Property::ThreadId, MaybeDynamicId::Dynamic(0)); - - let assigned_ids = db.write(builder.build_batch()).await.unwrap(); - assert_eq!(assigned_ids.document_ids.len(), 2); - let thread_id = assigned_ids.first_document_id().unwrap(); - let email_id = assigned_ids.last_document_id().unwrap(); - - let email_ids = db - .get_bitmap(BitmapKey { - account_id: 0, - collection: Collection::Email.into(), - class: BitmapClass::DocumentIds, - document_id: 0, - }) - .await - .unwrap() - .unwrap(); - assert_eq!(email_ids.len(), 1); - assert!(email_ids.contains(email_id)); - - let thread_ids = db - .get_bitmap(BitmapKey { - account_id: 0, - collection: Collection::Thread.into(), - class: BitmapClass::DocumentIds, - document_id: 0, - }) - .await - .unwrap() - .unwrap(); - assert_eq!(thread_ids.len(), 1); - assert!(thread_ids.contains(thread_id)); - - let tagged_ids = db - .get_bitmap(BitmapKey { - account_id: 0, - collection: Collection::Email.into(), - class: BitmapClass::Tag { - field: Property::ThreadId.into(), - value: TagValue::Id(thread_id), - }, - document_id: 0, - }) - .await - .unwrap() - .unwrap(); - assert_eq!(tagged_ids.len(), 1); - assert!(tagged_ids.contains(email_id)); - - let stored_thread_id = db - .get_value::(ValueKey { - account_id: 0, - collection: Collection::Email.into(), - document_id: email_id, - class: ValueClass::Property(Property::ThreadId.into()), - }) - .await - .unwrap() - .unwrap(); - assert_eq!(stored_thread_id, thread_id); - - let mut builder = BatchBuilder::new(); - builder - .with_account_id(0) - .with_collection(Collection::Thread) - .delete_document(thread_id) - .with_collection(Collection::Email) - .delete_document(email_id) - .untag( - Property::ThreadId, - TagValue::Id(MaybeDynamicId::Static(thread_id)), - ) - .clear(Property::ThreadId); - db.write(builder.build_batch()).await.unwrap(); - // Increment a counter 1000 times concurrently let mut handles = Vec::new(); let mut assigned_ids = HashSet::new(); @@ -195,7 +109,7 @@ pub async fn test(db: Store) { .with_collection(0) .update_document(0) .add_and_get(ValueClass::Directory(DirectoryClass::UsedQuota(0)), 1); - db.write(builder.build_batch()) + db.write(builder.build_all()) .await .unwrap() .last_counter_id() @@ -255,7 +169,7 @@ pub async fn test(db: Store) { .set(ValueClass::Property(1), value.as_slice()) .set(ValueClass::Property(0), "check1".as_bytes()) .set(ValueClass::Property(2), "check2".as_bytes()) - .build_batch(), + .build_all(), ) .await .unwrap(); @@ -282,7 +196,7 @@ pub async fn test(db: Store) { .with_collection(0) .update_document(0) .clear(ValueClass::Property(1)) - .build_batch(), + .build_all(), ) .await .unwrap(); @@ -328,7 +242,7 @@ pub async fn test(db: Store) { .clear(ValueClass::Property(0)) .clear(ValueClass::Property(2)) .clear(ValueClass::Directory(DirectoryClass::UsedQuota(0))) - .build_batch(), + .build_all(), ) .await .unwrap(); diff --git a/tests/src/store/query.rs b/tests/src/store/query.rs index 9802954d..6be81301 100644 --- a/tests/src/store/query.rs +++ b/tests/src/store/query.rs @@ -139,7 +139,7 @@ pub async fn test(db: Store, fts_store: FtsStore, do_insert: bool) { builder .with_account_id(0) .with_collection(COLLECTION_ID) - .create_document_with_id(document_id as u32); + .create_document(document_id as u32); for (pos, field) in record.iter().enumerate() { let field_id = pos as u8; match FIELDS_OPTIONS[pos] { @@ -186,10 +186,7 @@ pub async fn test(db: Store, fts_store: FtsStore, do_insert: bool) { } } - documents - .lock() - .unwrap() - .push((builder.build(), fts_builder)); + documents.lock().unwrap().push((builder, fts_builder)); }); } }); @@ -206,11 +203,11 @@ pub async fn test(db: Store, fts_store: FtsStore, do_insert: bool) { let mut fts_chunk = Vec::new(); print!("Inserting... ",); - for (batch, fts_batch) in batches { + for (mut batch, fts_batch) in batches { let chunk_instance = Instant::now(); chunk.push({ let db = db.clone(); - tokio::spawn(async move { db.write(batch).await }) + tokio::spawn(async move { db.write(batch.build_all()).await }) }); fts_chunk.push({ let fts_store = fts_store.clone();