From 2b614aa536c80b7f211787b0f97bf679a5c2d89a Mon Sep 17 00:00:00 2001 From: mdecimus <11444311+mdecimus@users.noreply.github.com> Date: Tue, 25 Nov 2025 12:14:51 +0100 Subject: [PATCH] Database schema optimization - part 13 (fixes #1882 fixes #2415) --- Cargo.lock | 18 +- crates/common/Cargo.toml | 1 + crates/common/src/auth/access_token.rs | 5 + crates/common/src/core.rs | 144 ++- crates/common/src/enterprise/undelete.rs | 113 +- crates/common/src/manager/backup.rs | 1098 +++-------------- crates/common/src/manager/restore.rs | 462 ++----- crates/common/src/sharing/document.rs | 112 -- crates/common/src/sharing/mod.rs | 1 - crates/common/src/storage/index.rs | 52 +- crates/dav/src/file/update.rs | 16 +- .../directory/src/backend/internal/manage.rs | 10 - crates/email/src/message/copy.rs | 20 +- crates/email/src/message/delete.rs | 61 +- crates/email/src/message/index/metadata.rs | 6 +- crates/email/src/message/index/mod.rs | 51 +- crates/email/src/message/ingest.rs | 95 +- crates/email/src/message/metadata.rs | 10 + crates/email/src/sieve/ingest.rs | 20 +- crates/http/src/form/mod.rs | 35 +- .../src/management/enterprise/undelete.rs | 112 +- crates/http/src/management/principal.rs | 97 +- crates/http/src/management/stores.rs | 160 ++- crates/imap/src/op/copy_move.rs | 34 +- crates/jmap/Cargo.toml | 2 +- crates/jmap/src/blob/copy.rs | 6 +- crates/jmap/src/blob/download.rs | 136 +- crates/jmap/src/blob/upload.rs | 4 +- crates/jmap/src/email/set.rs | 26 +- crates/jmap/src/sieve/set.rs | 10 +- crates/jmap/src/vacation/set.rs | 11 +- crates/managesieve/src/op/putscript.rs | 21 +- crates/migration/Cargo.toml | 2 +- crates/services/src/task_manager/index.rs | 170 +-- crates/smtp/src/queue/spool.rs | 28 +- crates/store/Cargo.toml | 2 +- crates/store/src/backend/mysql/main.rs | 2 +- crates/store/src/backend/postgres/main.rs | 2 +- crates/store/src/backend/rocksdb/main.rs | 2 +- crates/store/src/backend/sqlite/main.rs | 2 +- crates/store/src/dispatch/store.rs | 11 +- crates/store/src/lib.rs | 9 +- crates/store/src/write/blob.rs | 224 ++-- crates/store/src/write/key.rs | 58 +- crates/store/src/write/mod.rs | 13 +- crates/types/src/field.rs | 2 + crates/types/src/keyword.rs | 2 +- tests/src/directory/internal.rs | 41 +- tests/src/jmap/auth/permissions.rs | 7 +- tests/src/jmap/mail/delivery.rs | 72 +- tests/src/jmap/mod.rs | 4 +- tests/src/jmap/server/enterprise.rs | 19 +- tests/src/jmap/server/purge.rs | 27 +- tests/src/smtp/inbound/data.rs | 2 +- tests/src/smtp/queue/concurrent.rs | 7 +- tests/src/smtp/queue/virtualq.rs | 7 +- tests/src/store/blob.rs | 99 +- tests/src/store/cleanup.rs | 70 +- tests/src/store/import_export.rs | 78 +- tests/src/store/lookup.rs | 8 +- tests/src/store/mod.rs | 4 +- tests/src/store/ops.rs | 2 +- 62 files changed, 1661 insertions(+), 2264 deletions(-) delete mode 100644 crates/common/src/sharing/document.rs diff --git a/Cargo.lock b/Cargo.lock index f45e55e8..48aa87d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1172,6 +1172,7 @@ dependencies = [ "infer 0.19.0", "jmap_proto", "libc", + "lz4_flex 0.12.0", "mail-auth", "mail-builder", "mail-parser", @@ -3718,7 +3719,7 @@ dependencies = [ "hyper-util", "jmap-tools", "jmap_proto", - "lz4_flex", + "lz4_flex 0.12.0", "mail-auth", "mail-builder", "mail-parser", @@ -4183,6 +4184,15 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "lz4_flex" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e" +dependencies = [ + "twox-hash", +] + [[package]] name = "lzma-rust2" version = "0.13.0" @@ -4378,7 +4388,7 @@ dependencies = [ "directory", "email", "groupware", - "lz4_flex", + "lz4_flex 0.12.0", "mail-auth", "mail-parser", "nlp", @@ -7595,7 +7605,7 @@ dependencies = [ "foundationdb", "futures", "lru-cache", - "lz4_flex", + "lz4_flex 0.12.0", "memchr", "mysql_async", "nlp", @@ -9777,7 +9787,7 @@ dependencies = [ "crossbeam-utils", "flume", "lazy_static", - "lz4_flex", + "lz4_flex 0.11.5", "paste", "rand 0.8.5", "ringbuffer-spsc", diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index c373afcf..da24181e 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -73,6 +73,7 @@ rkyv = { version = "0.8.10", features = ["little_endian"] } indexmap = "2.7.1" tinyvec = "1.9.0" compact_str = { version = "0.9.0", features = ["rkyv", "serde"] } +lz4_flex = { version = "0.12", features = ["frame"], default-features = false } [target.'cfg(unix)'.dependencies] privdrop = "0.5.3" diff --git a/crates/common/src/auth/access_token.rs b/crates/common/src/auth/access_token.rs index 7e332f1d..786edb12 100644 --- a/crates/common/src/auth/access_token.rs +++ b/crates/common/src/auth/access_token.rs @@ -414,6 +414,11 @@ impl AccessToken { self } + pub fn with_tenant_id(mut self, tenant_id: Option) -> Self { + self.tenant = tenant_id.map(|id| TenantInfo { id, quota: 0 }); + self + } + pub fn state(&self) -> u32 { // Hash state let mut s = DefaultHasher::new(); diff --git a/crates/common/src/core.rs b/crates/common/src/core.rs index 7debf547..4f0b4111 100644 --- a/crates/common/src/core.rs +++ b/crates/common/src/core.rs @@ -29,8 +29,8 @@ use store::{ dispatch::DocumentSet, roaring::RoaringBitmap, write::{ - AlignedBytes, AnyClass, Archive, AssignedIds, BatchBuilder, BlobOp, DirectoryClass, - QueueClass, ValueClass, key::DeserializeBigEndian, now, + AlignedBytes, AnyClass, Archive, AssignedIds, BatchBuilder, BlobLink, BlobOp, + DirectoryClass, QueueClass, ValueClass, key::DeserializeBigEndian, now, }, }; use trc::AddContext; @@ -541,6 +541,52 @@ impl Server { }) } + pub async fn all_archives( + &self, + account_id: u32, + collection: Collection, + field: u8, + mut cb: CB, + ) -> trc::Result<()> + where + CB: FnMut(u32, Archive) -> trc::Result<()> + Send + Sync, + { + let collection: u8 = collection.into(); + + self.core + .storage + .data + .iterate( + IterateParams::new( + ValueKey { + account_id, + collection, + document_id: 0, + class: ValueClass::Property(field), + }, + ValueKey { + account_id, + collection, + document_id: u32::MAX, + class: ValueClass::Property(field), + }, + ), + |key, value| { + let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; + let archive = as Deserialize>::deserialize(value)?; + cb(document_id, archive)?; + + Ok(true) + }, + ) + .await + .add_context(|err| { + err.caused_by(trc::location!()) + .account_id(account_id) + .collection(collection) + }) + } + pub async fn document_ids( &self, account_id: u32, @@ -751,6 +797,7 @@ impl Server { SyncCollection::FileNode, SyncCollection::AddressBook, SyncCollection::Calendar, + SyncCollection::CalendarEventNotification, ] { let collection = sync_collection.into(); let from_key = LogKey { @@ -901,24 +948,29 @@ impl Server { } #[allow(clippy::blocks_in_conditions)] - pub async fn put_blob( - &self, - account_id: u32, - data: &[u8], - set_quota: bool, - ) -> trc::Result { + pub async fn put_jmap_blob(&self, account_id: u32, data: &[u8]) -> trc::Result { // First reserve the hash let hash = BlobHash::generate(data); let mut batch = BatchBuilder::new(); let until = now() + self.core.jmap.upload_tmp_ttl; - batch.with_account_id(account_id).set( - BlobOp::Reserve { - hash: hash.clone(), - until, - }, - (if set_quota { data.len() as u32 } else { 0u32 }).serialize(), - ); + batch + .with_account_id(account_id) + .set( + BlobOp::Link { + hash: hash.clone(), + to: BlobLink::Temporary { until }, + }, + vec![], + ) + .set( + BlobOp::Quota { + hash: hash.clone(), + until, + }, + (data.len() as u32).serialize(), + ); + self.core .storage .data @@ -963,6 +1015,68 @@ impl Server { }) } + pub async fn put_temporary_blob( + &self, + account_id: u32, + data: &[u8], + hold_for: u64, + ) -> trc::Result<(BlobHash, BlobOp)> { + // First reserve the hash + let hash = BlobHash::generate(data); + let mut batch = BatchBuilder::new(); + let until = now() + hold_for; + + batch.with_account_id(account_id).set( + BlobOp::Link { + hash: hash.clone(), + to: BlobLink::Temporary { until }, + }, + vec![], + ); + + self.core + .storage + .data + .write(batch.build_all()) + .await + .caused_by(trc::location!())?; + + if !self + .core + .storage + .data + .blob_exists(&hash) + .await + .caused_by(trc::location!())? + { + // Upload blob to store + self.core + .storage + .blob + .put_blob(hash.as_ref(), data) + .await + .caused_by(trc::location!())?; + + // Commit blob + let mut batch = BatchBuilder::new(); + batch.set(BlobOp::Commit { hash: hash.clone() }, Vec::new()); + self.core + .storage + .data + .write(batch.build_all()) + .await + .caused_by(trc::location!())?; + } + + Ok(( + hash.clone(), + BlobOp::Link { + hash, + to: BlobLink::Temporary { until }, + }, + )) + } + pub async fn total_accounts(&self) -> trc::Result { self.store() .count_principals(None, Type::Individual.into(), None) diff --git a/crates/common/src/enterprise/undelete.rs b/crates/common/src/enterprise/undelete.rs index 6cd793f5..28281bf3 100644 --- a/crates/common/src/enterprise/undelete.rs +++ b/crates/common/src/enterprise/undelete.rs @@ -9,74 +9,66 @@ */ use crate::Core; -use serde::{Deserialize, Serialize}; use store::{ - IterateParams, U32_LEN, U64_LEN, ValueKey, - write::{ - BatchBuilder, BlobOp, ValueClass, - key::{DeserializeBigEndian, KeySerializer}, - now, - }, + Deserialize, IterateParams, U32_LEN, U64_LEN, ValueKey, + write::{AlignedBytes, Archive, BlobOp, ValueClass, key::DeserializeBigEndian, now}, }; use trc::AddContext; use types::blob_hash::{BLOB_HASH_LEN, BlobHash}; -#[derive(Debug, Serialize, Deserialize)] -pub struct DeletedBlob { - pub hash: H, - pub size: usize, - #[serde(rename = "deletedAt")] - pub deleted_at: T, - #[serde(rename = "expiresAt")] - pub expires_at: T, - pub collection: C, +pub struct DeletedBlob { + pub hash: BlobHash, + pub expires_at: u64, + pub item: DeletedItem, +} + +#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] +pub struct DeletedItem { + pub typ: DeletedItemType, + pub size: u32, + pub deleted_at: u64, +} + +#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] +pub enum DeletedItemType { + Email { + from: Box, + subject: Box, + received_at: u64, + }, + FileNode { + name: Box, + }, + CalendarEvent { + title: Box, + start_time: u64, + }, + ContactCard { + name: Box, + }, + SieveScript { + name: Box, + }, } impl Core { - pub fn hold_undelete( - &self, - batch: &mut BatchBuilder, - collection: u8, - blob_hash: &BlobHash, - blob_size: usize, - ) { - if let Some(undelete) = self.enterprise.as_ref().and_then(|e| e.undelete.as_ref()) { - let now = now(); - - batch.set( - BlobOp::Reserve { - hash: blob_hash.clone(), - until: now + undelete.retention.as_secs(), - }, - KeySerializer::new(U64_LEN + U64_LEN) - .write(blob_size as u32) - .write(now) - .write(collection) - .finalize(), - ); - } - } - - pub async fn list_deleted( - &self, - account_id: u32, - ) -> trc::Result>> { + pub async fn list_deleted(&self, account_id: u32) -> trc::Result> { let from_key = ValueKey { account_id, collection: 0, document_id: 0, - class: ValueClass::Blob(BlobOp::Reserve { + class: ValueClass::Blob(BlobOp::Undelete { hash: BlobHash::default(), until: 0, }), }; let to_key = ValueKey { - account_id: account_id + 1, + account_id, collection: 0, - document_id: 0, - class: ValueClass::Blob(BlobOp::Reserve { - hash: BlobHash::default(), - until: 0, + document_id: u32::MAX, + class: ValueClass::Blob(BlobOp::Undelete { + hash: BlobHash::new_max(), + until: u64::MAX, }), }; @@ -89,18 +81,25 @@ impl Core { IterateParams::new(from_key, to_key).ascending(), |key, value| { let expires_at = key.deserialize_be_u64(key.len() - U64_LEN)?; - if value.len() == U32_LEN + U64_LEN + 1 && expires_at > now { + if expires_at > now { + let item = as Deserialize>::deserialize(value) + .and_then(|bytes| bytes.deserialize::()) + .add_context(|ctx| ctx.ctx(trc::Key::Key, key))?; + results.push(DeletedBlob { hash: BlobHash::try_from_hash_slice( - key.get(U32_LEN..U32_LEN + BLOB_HASH_LEN).ok_or_else(|| { - trc::Error::corrupted_key(key, value.into(), trc::location!()) - })?, + key.get(U32_LEN + 1..U32_LEN + 1 + BLOB_HASH_LEN) + .ok_or_else(|| { + trc::Error::corrupted_key( + key, + value.into(), + trc::location!(), + ) + })?, ) .unwrap(), - size: value.deserialize_be_u32(0)? as usize, - deleted_at: value.deserialize_be_u64(U32_LEN)?, expires_at, - collection: *value.last().unwrap(), + item, }); } Ok(true) diff --git a/crates/common/src/manager/backup.rs b/crates/common/src/manager/backup.rs index 0848b20a..f9b4e602 100644 --- a/crates/common/src/manager/backup.rs +++ b/crates/common/src/manager/backup.rs @@ -5,59 +5,33 @@ */ use crate::Core; -use ahash::{AHashMap, AHashSet}; +use ahash::AHashSet; +use lz4_flex::frame::FrameEncoder; use std::{ - collections::BTreeSet, io::{BufWriter, Write}, - ops::Range, path::{Path, PathBuf}, sync::mpsc::{self, SyncSender}, }; use store::{ - Deserialize, IndexKey, IterateParams, LogKey, SerializeInfallible, U32_LEN, U64_LEN, ValueKey, - write::{ - AnyKey, BlobOp, DirectoryClass, InMemoryClass, QueueClass, QueueEvent, ValueClass, - key::DeserializeBigEndian, - }, -}; -use types::{ - blob_hash::{BLOB_HASH_LEN, BlobHash}, - collection::Collection, - field::{Field, MailboxField}, -}; -use utils::{ - UnwrapFailure, - codec::leb128::{Leb128_, Leb128Reader}, - failed, + write::{AnyClass, AnyKey, ValueClass}, + *, }; +use types::blob_hash::{BLOB_HASH_LEN, BlobHash}; +use utils::{UnwrapFailure, codec::leb128::Leb128_}; pub(super) const MAGIC_MARKER: u8 = 123; -pub(super) const FILE_VERSION: u8 = 2; - -#[derive(Debug)] -pub(super) enum Op { - Family(Family), - AccountId(u32), - Collection(u8), - DocumentId(u32), - KeyValue((Vec, Vec)), -} #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] pub(super) enum Family { - Property = 0, - FtsIndex = 1, - Acl = 2, - Blob = 3, - Config = 4, - LookupValue = 5, - LookupCounter = 6, - Directory = 7, - Queue = 8, - Index = 9, - Bitmap = 10, - Log = 11, - None = 255, + Data = 0, + Directory = 1, + Blob = 2, + Config = 3, + Changelog = 4, + Queue = 5, + Report = 6, + Telemetry = 7, + Tasks = 8, } type TaskHandle = (tokio::task::JoinHandle<()>, std::thread::JoinHandle<()>); @@ -69,7 +43,7 @@ pub struct BackupParams { } impl Core { - pub async fn backup(&self, params: BackupParams) { + pub async fn backup(&self, mut params: BackupParams) { if !params.dest.exists() { std::fs::create_dir_all(¶ms.dest).failed("Failed to create backup directory"); } else if !params.dest.is_dir() { @@ -78,42 +52,44 @@ impl Core { } let mut sync_handles = Vec::new(); + let schema_version = self + .storage + .data + .get_value::(AnyKey { + subspace: SUBSPACE_PROPERTY, + key: vec![0u8], + }) + .await + .failed("Could not retrieve database schema version.") + .failed("Could not retrieve database schema version."); - for (async_handle, sync_handle) in [ - params - .has_family(Family::Property) - .then(|| self.backup_properties(¶ms.dest)), - params - .has_family(Family::FtsIndex) - .then(|| self.backup_fts_index(¶ms.dest)), - params - .has_family(Family::Acl) - .then(|| self.backup_acl(¶ms.dest)), - params - .has_family(Family::Blob) - .then(|| self.backup_blob(¶ms.dest)), - params - .has_family(Family::Config) - .then(|| self.backup_config(¶ms.dest)), - params - .has_family(Family::LookupValue) - .then(|| self.backup_lookup(¶ms.dest)), - params - .has_family(Family::Directory) - .then(|| self.backup_directory(¶ms.dest)), - params - .has_family(Family::Queue) - .then(|| self.backup_queue(¶ms.dest)), - params - .has_family(Family::Index) - .then(|| self.backup_index(¶ms.dest)), - params - .has_family(Family::Log) - .then(|| self.backup_logs(¶ms.dest)), - ] - .into_iter() - .flatten() + if params.families.is_empty() { + params.families = [ + Family::Data, + Family::Directory, + Family::Blob, + Family::Config, + Family::Changelog, + Family::Queue, + Family::Report, + Family::Telemetry, + Family::Tasks, + ] + .into_iter() + .collect(); + } + + for subspace in params + .families + .into_iter() + .flat_map(|f| f.subspaces()) + .copied() { + let (async_handle, sync_handle) = if subspace == SUBSPACE_BLOBS { + self.backup_blobs(¶ms.dest, subspace, schema_version) + } else { + self.backup_subspace(¶ms.dest, subspace, schema_version) + }; async_handle.await.failed("Task failed"); sync_handles.push(sync_handle); } @@ -123,483 +99,42 @@ impl Core { } } - fn backup_properties(&self, dest: &Path) -> TaskHandle { - let store = self.storage.data.clone(); - let (handle, writer) = spawn_writer(dest.join("property")); - ( - tokio::spawn(async move { - writer - .send(Op::Family(Family::Property)) - .failed("Failed to send family"); - - let mut keys = BTreeSet::new(); - - store - .iterate( - IterateParams::new( - ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::Property(0), - }, - ValueKey { - account_id: u32::MAX, - collection: u8::MAX, - document_id: u32::MAX, - class: ValueClass::Property(u8::MAX), - }, - ) - .no_values(), - |key, _| { - let account_id = key.deserialize_be_u32(0)?; - let collection = key.deserialize_u8(U32_LEN)?; - let field = key.deserialize_u8(U32_LEN + 1)?; - let document_id = key.deserialize_be_u32(U32_LEN + 2)?; - - keys.insert((account_id, collection, document_id, field)); - - Ok(true) - }, - ) - .await - .failed("Failed to iterate over data store"); - - let mut last_account_id = u32::MAX; - let mut last_collection = u8::MAX; - let mut last_document_id = u32::MAX; - - for (account_id, collection, document_id, field) in keys { - if account_id != last_account_id { - writer - .send(Op::AccountId(account_id)) - .failed("Failed to send account id"); - last_account_id = account_id; - } - - if collection != last_collection { - writer - .send(Op::Collection(collection)) - .failed("Failed to send collection"); - last_collection = collection; - } - - if document_id != last_document_id { - writer - .send(Op::DocumentId(document_id)) - .failed("Failed to send document id"); - last_document_id = document_id; - } - - // Obtain UID counter - if collection == u8::from(Collection::Mailbox) - && u8::from(Field::ARCHIVE) == field - { - let value = store - .get_counter(ValueKey { - account_id, - collection, - document_id, - class: MailboxField::UidCounter.into(), - }) - .await - .failed("Failed to get counter"); - if value != 0 { - writer - .send(Op::KeyValue(( - vec![u8::from(MailboxField::UidCounter)], - value.serialize(), - ))) - .failed("Failed to send key value"); - } - } - - // Write value - let value = store - .get_value::(ValueKey { - account_id, - collection, - document_id, - class: ValueClass::Property(field), - }) - .await - .failed("Failed to get value") - .failed("Expected value") - .0; - writer - .send(Op::KeyValue((vec![field], value))) - .failed("Failed to send key value"); - } - }), - handle, - ) - } - - fn backup_fts_index(&self, dest: &Path) -> TaskHandle { - /*let store = self.storage.data.clone(); - let (handle, writer) = spawn_writer(dest.join("fts_index")); - ( - tokio::spawn(async move { - writer - .send(Op::Family(Family::FtsIndex)) - .failed("Failed to send family"); - - let mut last_account_id = u32::MAX; - let mut last_collection = u8::MAX; - - store - .iterate( - IterateParams::new( - ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::FtsIndex(BitmapHash { - hash: [0; 8], - len: 1, - }), - }, - ValueKey { - account_id: u32::MAX, - collection: u8::MAX, - document_id: u32::MAX, - class: ValueClass::FtsIndex(BitmapHash { - hash: [u8::MAX; 8], - len: u8::MAX, - }), - }, - ), - |key, value| { - let account_id = key.deserialize_be_u32(0)?; - let collection = key.deserialize_u8(key.len() - U32_LEN - 1)?; - let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; - - if account_id != last_account_id { - writer - .send(Op::AccountId(account_id)) - .failed("Failed to send account id"); - last_account_id = account_id; - } - - if collection != last_collection { - writer - .send(Op::Collection(collection)) - .failed("Failed to send collection"); - last_collection = collection; - } - - writer - .send(Op::DocumentId(document_id)) - .failed("Failed to send document id"); - - writer - .send(Op::KeyValue(( - key.range(U32_LEN..key.len() - U32_LEN - 1)?.to_vec(), - value.to_vec(), - ))) - .failed("Failed to send key value"); - - Ok(true) - }, - ) - .await - .failed("Failed to iterate over data store"); - }), - handle, - )*/ - todo!() - } - - fn backup_acl(&self, dest: &Path) -> TaskHandle { - let store = self.storage.data.clone(); - let (handle, writer) = spawn_writer(dest.join("acl")); - ( - tokio::spawn(async move { - writer - .send(Op::Family(Family::Acl)) - .failed("Failed to send family"); - - let mut last_account_id = u32::MAX; - let mut last_collection = u8::MAX; - let mut last_document_id = u32::MAX; - - store - .iterate( - IterateParams::new( - ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::Acl(0), - }, - ValueKey { - account_id: u32::MAX, - collection: u8::MAX, - document_id: u32::MAX, - class: ValueClass::Acl(u32::MAX), - }, - ), - |key, value| { - let grant_account_id = key.deserialize_be_u32(0)?; - let account_id = key.deserialize_be_u32(U32_LEN)?; - let collection = key.deserialize_u8(U32_LEN * 2)?; - let document_id = key.deserialize_be_u32((U32_LEN * 2) + 1)?; - - if account_id != last_account_id { - writer - .send(Op::AccountId(account_id)) - .failed("Failed to send account id"); - last_account_id = account_id; - } - - if collection != last_collection { - writer - .send(Op::Collection(collection)) - .failed("Failed to send collection"); - last_collection = collection; - } - - if document_id != last_document_id { - writer - .send(Op::DocumentId(document_id)) - .failed("Failed to send document id"); - last_document_id = document_id; - } - - writer - .send(Op::KeyValue(( - grant_account_id.to_be_bytes().to_vec(), - value.to_vec(), - ))) - .failed("Failed to send key value"); - - Ok(true) - }, - ) - .await - .failed("Failed to iterate over data store"); - }), - handle, - ) - } - - fn backup_blob(&self, dest: &Path) -> TaskHandle { + fn backup_blobs(&self, dest: &Path, subspace: u8, schema_version: u32) -> TaskHandle { let store = self.storage.data.clone(); let blob_store = self.storage.blob.clone(); - let (handle, writer) = spawn_writer(dest.join("blob")); + let (handle, writer) = spawn_writer( + dest.join(format!("subspace_{}", char::from(subspace))), + subspace, + schema_version, + ); ( tokio::spawn(async move { - writer - .send(Op::Family(Family::Blob)) - .failed("Failed to send family"); - - let mut hashes = Vec::new(); - + let mut blobs = Vec::new(); + let mut last_hash = BlobHash::default(); store .iterate( IterateParams::new( - ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::Blob(BlobOp::Link { - hash: Default::default(), - }), + AnyKey { + subspace: SUBSPACE_BLOB_LINK, + key: vec![0u8], }, - ValueKey { - account_id: u32::MAX, - collection: u8::MAX, - document_id: u32::MAX, - class: ValueClass::Blob(BlobOp::Link { - hash: BlobHash::new_max(), - }), - }, - ), - |key, _| { - let account_id = key.deserialize_be_u32(BLOB_HASH_LEN)?; - let collection = key.deserialize_u8(BLOB_HASH_LEN + U32_LEN)?; - let document_id = - key.deserialize_be_u32(BLOB_HASH_LEN + U32_LEN + 1)?; - - let hash = key.range(0..BLOB_HASH_LEN)?.to_vec(); - - if account_id != u32::MAX && document_id != u32::MAX { - writer - .send(Op::AccountId(account_id)) - .failed("Failed to send account id"); - writer - .send(Op::Collection(collection)) - .failed("Failed to send collection"); - writer - .send(Op::DocumentId(document_id)) - .failed("Failed to send document id"); - writer - .send(Op::KeyValue((hash, vec![]))) - .failed("Failed to send key value"); - } else { - hashes.push(hash); - } - - Ok(true) - }, - ) - .await - .failed("Failed to iterate over data store"); - - if !hashes.is_empty() { - writer - .send(Op::AccountId(u32::MAX)) - .failed("Failed to send account id"); - writer - .send(Op::DocumentId(u32::MAX)) - .failed("Failed to send document id"); - for hash in hashes { - if let Some(value) = blob_store - .get_blob(&hash, 0..usize::MAX) - .await - .failed("Failed to get blob") - { - writer - .send(Op::KeyValue((hash, value))) - .failed("Failed to send key value"); - } else { - eprintln!( - "Warning: blob hash {hash:?} does not exist in blob store. Skipping." - ); - } - } - } - }), - handle, - ) - } - - fn backup_config(&self, dest: &Path) -> TaskHandle { - let store = self.storage.data.clone(); - let (handle, writer) = spawn_writer(dest.join("config")); - ( - tokio::spawn(async move { - writer - .send(Op::Family(Family::Config)) - .failed("Failed to send family"); - - store - .iterate( - IterateParams::new( - ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::Config(vec![0]), - }, - ValueKey { - account_id: u32::MAX, - collection: u8::MAX, - document_id: u32::MAX, - class: ValueClass::Config(vec![ - u8::MAX, - u8::MAX, - u8::MAX, - u8::MAX, - u8::MAX, - u8::MAX, - ]), - }, - ), - |key, value| { - writer - .send(Op::KeyValue((key.to_vec(), value.to_vec()))) - .failed("Failed to send key value"); - - Ok(true) - }, - ) - .await - .failed("Failed to iterate over data store"); - }), - handle, - ) - } - - fn backup_lookup(&self, dest: &Path) -> TaskHandle { - let store = self.storage.data.clone(); - let (handle, writer) = spawn_writer(dest.join("lookup")); - ( - tokio::spawn(async move { - writer - .send(Op::Family(Family::LookupValue)) - .failed("Failed to send family"); - - store - .iterate( - IterateParams::new( - ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::InMemory(InMemoryClass::Key(vec![0])), - }, - ValueKey { - account_id: u32::MAX, - collection: u8::MAX, - document_id: u32::MAX, - class: ValueClass::InMemory(InMemoryClass::Key(vec![ - u8::MAX, - u8::MAX, - u8::MAX, - u8::MAX, - u8::MAX, - u8::MAX, - ])), - }, - ), - |key, value| { - writer - .send(Op::KeyValue((key.to_vec(), value.to_vec()))) - .failed("Failed to send key value"); - - Ok(true) - }, - ) - .await - .failed("Failed to iterate over data store"); - - writer - .send(Op::Family(Family::LookupCounter)) - .failed("Failed to send family"); - - let mut counters = Vec::new(); - - store - .iterate( - IterateParams::new( - ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::InMemory(InMemoryClass::Counter(vec![0])), - }, - ValueKey { - account_id: u32::MAX, - collection: u8::MAX, - document_id: u32::MAX, - class: ValueClass::InMemory(InMemoryClass::Counter(vec![ - u8::MAX, - u8::MAX, - u8::MAX, - u8::MAX, - u8::MAX, - u8::MAX, - ])), + AnyKey { + subspace: SUBSPACE_BLOB_LINK, + key: vec![u8::MAX; 32], }, ) .no_values(), |key, _| { - if (key.len() != (U32_LEN * 2) + 2) - || key[U32_LEN + 1] != 84 - || key[U32_LEN] != 1 - { - counters.push(key.to_vec()); + let hash = BlobHash::try_from_hash_slice( + key.get(0..BLOB_HASH_LEN).ok_or_else(|| { + trc::Error::corrupted_key(key, None, trc::location!()) + })?, + ) + .unwrap(); + + if last_hash != hash { + blobs.push(hash.clone()); + last_hash = hash; } Ok(true) @@ -608,18 +143,15 @@ impl Core { .await .failed("Failed to iterate over data store"); - for key in counters { - let value = store - .get_counter(ValueKey::from(ValueClass::InMemory( - InMemoryClass::Counter(key.clone()), - ))) + for hash in blobs { + if let Some(blob) = blob_store + .get_blob(hash.as_slice(), 0..usize::MAX) .await - .failed("Failed to get counter"); - - if value != 0 { + .failed("Failed to get blob") + { writer - .send(Op::KeyValue((key, value.serialize()))) - .failed("Failed to send key value"); + .send((hash.as_slice().to_vec(), blob)) + .failed("Failed to send key"); } } }), @@ -627,338 +159,111 @@ impl Core { ) } - fn backup_directory(&self, dest: &Path) -> TaskHandle { + fn backup_subspace(&self, dest: &Path, subspace: u8, schema_version: u32) -> TaskHandle { let store = self.storage.data.clone(); - let (handle, writer) = spawn_writer(dest.join("directory")); + let (handle, writer) = spawn_writer( + dest.join(format!("subspace_{}", char::from(subspace))), + subspace, + schema_version, + ); ( tokio::spawn(async move { - writer - .send(Op::Family(Family::Directory)) - .failed("Failed to send family"); - - let mut principal_ids = Vec::new(); - - store - .iterate( - IterateParams::new( - ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::Directory(DirectoryClass::NameToId(vec![0])), - }, - ValueKey { - account_id: u32::MAX, - collection: u8::MAX, - document_id: u32::MAX, - class: ValueClass::Directory(DirectoryClass::Members { - principal_id: u32::MAX, - has_member: u32::MAX, - }), - }, - ), - |key, value| { - if key[0] == 2 { - principal_ids.push(key.range(1..usize::MAX)?.to_vec()); - } - - writer - .send(Op::KeyValue((key.to_vec(), value.to_vec()))) - .failed("Failed to send key value"); - - Ok(true) - }, - ) - .await - .failed("Failed to iterate over data store"); - - for principal_bytes in principal_ids { - let value = store - .get_counter(ValueKey::from(ValueClass::Directory( - DirectoryClass::UsedQuota( - principal_bytes - .as_slice() - .deserialize_leb128() - .failed("Failed to deserialize principal id"), + if !store.is_sql() || (subspace != SUBSPACE_COUNTER && subspace != SUBSPACE_QUOTA) { + store + .iterate( + IterateParams::new( + AnyKey { + subspace, + key: vec![0u8], + }, + AnyKey { + subspace, + key: vec![u8::MAX; 32], + }, ), - ))) - .await - .failed("Failed to get counter"); - if value != 0 { - let mut key = Vec::with_capacity(U32_LEN + 1); - key.push(4u8); - key.extend_from_slice(&principal_bytes); + |key, value| { + writer + .send((key.to_vec(), value.to_vec())) + .failed("Failed to send key"); - writer - .send(Op::KeyValue((key, value.serialize()))) - .failed("Failed to send key value"); - } - } - }), - handle, - ) - } - - fn backup_queue(&self, dest: &Path) -> TaskHandle { - let store = self.storage.data.clone(); - let (handle, writer) = spawn_writer(dest.join("queue")); - ( - tokio::spawn(async move { - writer - .send(Op::Family(Family::Queue)) - .failed("Failed to send family"); - - store - .iterate( - IterateParams::new( - ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::Queue(QueueClass::Message(0)), - }, - ValueKey { - account_id: u32::MAX, - collection: u8::MAX, - document_id: u32::MAX, - class: ValueClass::Queue(QueueClass::Message(u64::MAX)), - }, - ), - |key_, value| { - let mut key = Vec::with_capacity(U64_LEN + 1); - key.push(0); - key.extend_from_slice(key_); - - writer - .send(Op::KeyValue((key, value.to_vec()))) - .failed("Failed to send key value"); - - Ok(true) - }, - ) - .await - .failed("Failed to iterate over data store"); - - store - .iterate( - IterateParams::new( - ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { - due: 0, - queue_id: 0, - queue_name: [0; 8], - })), - }, - ValueKey { - account_id: u32::MAX, - collection: u8::MAX, - document_id: u32::MAX, - class: ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { - due: u64::MAX, - queue_id: u64::MAX, - queue_name: [u8::MAX; 8], - })), - }, - ), - |key_, value| { - let mut key = Vec::with_capacity(U64_LEN + 1); - key.push(1); - key.extend_from_slice(key_); - - writer - .send(Op::KeyValue((key, value.to_vec()))) - .failed("Failed to send key value"); - - Ok(true) - }, - ) - .await - .failed("Failed to iterate over data store"); - }), - handle, - ) - } - - fn backup_index(&self, dest: &Path) -> TaskHandle { - let store = self.storage.data.clone(); - let (handle, writer) = spawn_writer(dest.join("index")); - ( - tokio::spawn(async move { - writer - .send(Op::Family(Family::Index)) - .failed("Failed to send family"); - - let mut last_account_id = u32::MAX; - let mut last_collection = u8::MAX; - - store - .iterate( - IterateParams::new( - IndexKey { - account_id: 0, - collection: 0, - document_id: 0, - field: 0, - key: vec![0], - }, - IndexKey { - account_id: u32::MAX, - collection: u8::MAX, - document_id: u32::MAX, - field: u8::MAX, - key: vec![u8::MAX, u8::MAX, u8::MAX], + Ok(true) }, ) - .no_values(), - |key, _| { - let account_id = key.deserialize_be_u32(0)?; - let collection = key.deserialize_u8(U32_LEN)?; - let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; + .await + .failed("Failed to iterate over data store"); + } else { + let mut keys = Vec::with_capacity(128); + store + .iterate( + IterateParams::new( + AnyKey { + subspace, + key: vec![0u8], + }, + AnyKey { + subspace, + key: vec![u8::MAX; 32], + }, + ) + .no_values(), + |key, _| { + keys.push(key.to_vec()); - let key = key.range(U32_LEN + 1..key.len() - U32_LEN)?.to_vec(); - - if account_id != last_account_id { - writer - .send(Op::AccountId(account_id)) - .failed("Failed to send account id"); - last_account_id = account_id; - } - - if collection != last_collection { - writer - .send(Op::Collection(collection)) - .failed("Failed to send collection"); - last_collection = collection; - } - - writer - .send(Op::DocumentId(document_id)) - .failed("Failed to send document id"); - - writer - .send(Op::KeyValue((key, vec![]))) - .failed("Failed to send key value"); - - Ok(true) - }, - ) - .await - .failed("Failed to iterate over data store"); - }), - handle, - ) - } - - fn backup_logs(&self, dest: &Path) -> TaskHandle { - let store = self.storage.data.clone(); - let (handle, writer) = spawn_writer(dest.join("log")); - ( - tokio::spawn(async move { - writer - .send(Op::Family(Family::Log)) - .failed("Failed to send family"); - - let mut last_account_id = u32::MAX; - let mut last_collection = u8::MAX; - - store - .iterate( - IterateParams::new( - LogKey { - account_id: 0, - collection: 0, - change_id: 0, + Ok(true) }, - LogKey { - account_id: u32::MAX, - collection: u8::MAX, - change_id: u64::MAX, - }, - ), - |key, value| { - let account_id = key.deserialize_be_u32(0)?; - let collection = key.deserialize_u8(U32_LEN)?; - let key = key.range(U32_LEN + 1..usize::MAX)?.to_vec(); + ) + .await + .failed("Failed to iterate over data store"); - if key.len() != U64_LEN { - failed(&format!("Found invalid log entry {key:?} {value:?}")); - } - - if account_id != last_account_id { - writer - .send(Op::AccountId(account_id)) - .failed("Failed to send account id"); - last_account_id = account_id; - } - - if collection != last_collection { - writer - .send(Op::Collection(collection)) - .failed("Failed to send collection"); - last_collection = collection; - } - - writer - .send(Op::KeyValue((key, value.to_vec()))) - .failed("Failed to send key value"); - - Ok(true) - }, - ) - .await - .failed("Failed to iterate over data store"); + for key in keys { + let counter = store + .get_counter(ValueClass::Any(AnyClass { + subspace, + key: key.clone(), + })) + .await + .failed("Failed to get counter"); + writer + .send((key.to_vec(), (counter as u64).to_le_bytes().to_vec())) + .failed("Failed to send key"); + } + } }), handle, ) } } -fn spawn_writer(path: PathBuf) -> (std::thread::JoinHandle<()>, SyncSender) { - let (tx, rx) = mpsc::sync_channel(10); +#[allow(clippy::type_complexity)] +fn spawn_writer( + path: PathBuf, + subspace: u8, + version: u32, +) -> (std::thread::JoinHandle<()>, SyncSender<(Vec, Vec)>) { + let (tx, rx) = mpsc::sync_channel::<(Vec, Vec)>(10); let handle = std::thread::spawn(move || { println!("Exporting database to {}.", path.to_str().unwrap()); - let mut file = - BufWriter::new(std::fs::File::create(path).failed("Failed to create backup file")); - file.write_all(&[MAGIC_MARKER, FILE_VERSION]) + let mut file = FrameEncoder::new(BufWriter::new( + std::fs::File::create(path).failed("Failed to create backup file"), + )); + file.write_all(&[MAGIC_MARKER, subspace]) + .failed("Failed to write version"); + file.write_all(&version.to_le_bytes()) .failed("Failed to write version"); - while let Ok(op) = rx.recv() { - match op { - Op::Family(f) => { - file.write_all(&[0u8, f as u8]) - .failed("Failed to write family"); - } - Op::KeyValue((k, v)) => { - file.write_all(&[if !v.is_empty() { 1u8 } else { 2u8 }]) - .failed("Failed to write key"); - file.write_all(&(k.len() as u32).serialize()) - .failed("Failed to write key value"); - file.write_all(&k).failed("Failed to write key"); - if !v.is_empty() { - file.write_all(&(v.len() as u32).serialize()) - .failed("Failed to write key value"); - file.write_all(&v).failed("Failed to write key value"); - } - } - Op::AccountId(v) => { - file.write_all(&[3u8]).failed("Failed to write account id"); - file.write_all(&v.serialize()) - .failed("Failed to write account id"); - } - Op::Collection(v) => { - file.write_all(&[4u8, v]) - .failed("Failed to write collection"); - } - Op::DocumentId(v) => { - file.write_all(&[5u8]).failed("Failed to write document id"); - file.write_all(&v.serialize()) - .failed("Failed to write document id"); - } + while let Ok((key, value)) = rx.recv() { + key.len() + .to_leb128_writer(&mut file) + .failed("Failed to write key value"); + file.write_all(&key).failed("Failed to write key"); + value + .len() + .to_leb128_writer(&mut file) + .failed("Failed to write key value"); + if !value.is_empty() { + file.write_all(&value).failed("Failed to write key value"); } } @@ -968,31 +273,6 @@ fn spawn_writer(path: PathBuf) -> (std::thread::JoinHandle<()>, SyncSender) (handle, tx) } -pub(super) trait DeserializeBytes { - fn range(&self, range: Range) -> trc::Result<&[u8]>; - fn deserialize_u8(&self, offset: usize) -> trc::Result; - fn deserialize_leb128(&self) -> trc::Result; -} - -impl DeserializeBytes for &[u8] { - fn range(&self, range: Range) -> trc::Result<&[u8]> { - self.get(range.start..std::cmp::min(range.end, self.len())) - .ok_or_else(|| trc::StoreEvent::DataCorruption.caused_by(trc::location!())) - } - - fn deserialize_u8(&self, offset: usize) -> trc::Result { - self.get(offset) - .copied() - .ok_or_else(|| trc::StoreEvent::DataCorruption.caused_by(trc::location!())) - } - - fn deserialize_leb128(&self) -> trc::Result { - self.read_leb128::() - .map(|(v, _)| v) - .ok_or_else(|| trc::StoreEvent::DataCorruption.caused_by(trc::location!())) - } -} - impl BackupParams { pub fn new(dest: PathBuf) -> Self { let mut params = Self { @@ -1021,35 +301,41 @@ impl BackupParams { } } } - - fn has_family(&self, family: Family) -> bool { - self.families.is_empty() || self.families.contains(&family) - } } impl Family { + pub fn subspaces(&self) -> &'static [u8] { + match self { + Family::Data => &[ + SUBSPACE_ACL, + SUBSPACE_INDEXES, + SUBSPACE_QUOTA, + SUBSPACE_COUNTER, + SUBSPACE_PROPERTY, + ], + Family::Directory => &[SUBSPACE_DIRECTORY], + Family::Blob => &[SUBSPACE_BLOBS, SUBSPACE_BLOB_EXTRA, SUBSPACE_BLOB_LINK], + Family::Config => &[SUBSPACE_SETTINGS], + Family::Changelog => &[SUBSPACE_LOGS], + Family::Queue => &[SUBSPACE_QUEUE_MESSAGE, SUBSPACE_QUEUE_EVENT], + Family::Report => &[SUBSPACE_REPORT_OUT, SUBSPACE_REPORT_IN], + Family::Telemetry => &[SUBSPACE_TELEMETRY_SPAN, SUBSPACE_TELEMETRY_METRIC], + Family::Tasks => &[SUBSPACE_TASK_QUEUE], + } + } + pub fn parse(family: &str) -> Result { match family { - "property" => Ok(Family::Property), - "fts_index" => Ok(Family::FtsIndex), - "acl" => Ok(Family::Acl), + "data" => Ok(Family::Data), + "directory" => Ok(Family::Directory), "blob" => Ok(Family::Blob), "config" => Ok(Family::Config), - "lookup" => Ok(Family::LookupValue), - "directory" => Ok(Family::Directory), + "changelog" => Ok(Family::Changelog), "queue" => Ok(Family::Queue), - "index" => Ok(Family::Index), - "bitmap" => Ok(Family::Bitmap), - "log" => Ok(Family::Log), + "report" => Ok(Family::Report), + "telemetry" => Ok(Family::Telemetry), + "tasks" => Ok(Family::Tasks), _ => Err(format!("Unknown family {}", family)), } } } - -struct RawBytes(Vec); - -impl Deserialize for RawBytes { - fn deserialize(bytes: &[u8]) -> trc::Result { - Ok(Self(bytes.to_vec())) - } -} diff --git a/crates/common/src/manager/restore.rs b/crates/common/src/manager/restore.rs index 55f0e913..babd317d 100644 --- a/crates/common/src/manager/restore.rs +++ b/crates/common/src/manager/restore.rs @@ -4,32 +4,20 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::Core; -use ahash::AHashMap; +use super::backup::MAGIC_MARKER; +use crate::{Core, DATABASE_SCHEMA_VERSION}; +use lz4_flex::frame::FrameDecoder; use std::{ - io::ErrorKind, + fs::File, + io::{BufReader, ErrorKind, Read}, path::{Path, PathBuf}, }; use store::{ - BlobStore, Key, LogKey, SUBSPACE_LOGS, SerializeInfallible, Store, U32_LEN, - write::{ - AnyClass, BatchBuilder, BlobOp, DirectoryClass, InMemoryClass, Operation, SearchIndex, - TaskEpoch, TaskQueueClass, ValueClass, ValueOp, key::DeserializeBigEndian, now, - }, + BlobStore, SUBSPACE_BLOBS, SUBSPACE_COUNTER, SUBSPACE_QUOTA, Store, + write::{AnyClass, BatchBuilder, ValueClass}, }; -use store::{ - Deserialize, U64_LEN, - write::{QueueClass, QueueEvent}, -}; -use tokio::{ - fs::File, - io::{AsyncReadExt, BufReader}, -}; -use types::{blob_hash::BlobHash, collection::Collection, field::MailboxField}; use utils::{UnwrapFailure, failed}; -use super::backup::{DeserializeBytes, FILE_VERSION, Family, MAGIC_MARKER, Op}; - impl Core { pub async fn restore(&self, src: PathBuf) { // Backup the core @@ -60,258 +48,57 @@ impl Core { async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { println!("Importing database dump from {}.", path.to_str().unwrap()); - let mut reader = OpReader::new(path).await; - let mut account_id = u32::MAX; - let mut document_id = u32::MAX; - let mut collection = Collection::None; - let mut collection_raw = u8::MAX; - let mut family = Family::None; - let mut due = now(); - - let mut batch_size = 0; + let mut reader = KeyValueReader::new(path); let mut batch = BatchBuilder::new(); - let mut change_ids: AHashMap = AHashMap::new(); - - while let Some(op) = reader.next().await { - match op { - Op::Family(f) => family = f, - Op::AccountId(a) => { - account_id = a; - batch.with_account_id(account_id); + match reader.subspace { + SUBSPACE_BLOBS => { + while let Some((key, value)) = reader.next() { + blob_store + .put_blob(&key, &value) + .await + .failed("Failed to write blob"); } - Op::Collection(c) => { - collection_raw = c; - collection = Collection::from(c); - batch.with_collection(collection); - } - Op::DocumentId(d) => { - document_id = d; - batch.with_document(document_id); - } - Op::KeyValue((key, value)) => { - batch_size += key.len() + value.len() + U32_LEN * 2; - - match family { - Family::Property => { - let field = key - .as_slice() - .deserialize_u8(0) - .expect("Failed to deserialize field"); - if collection == Collection::Mailbox - && u8::from(MailboxField::UidCounter) == field - { - batch.add( - ValueClass::Property(field), - i64::deserialize(&value) - .expect("Failed to deserialize mailbox uidnext"), - ); - } else { - batch.set(ValueClass::Property(field), value); - } - } - Family::FtsIndex => { - if reader.version > 1 { - let mut hash = [0u8; 8]; - let (hash, len) = match key.len() { - 9 => { - hash[..8].copy_from_slice(&key[..8]); - (hash, key[key.len() - 1]) - } - len @ (1..=7) => { - hash[..len].copy_from_slice(&key[..len]); - (hash, len as u8) - } - invalid => { - panic!("Invalid text bitmap key length {invalid}"); - } - }; - - //batch.set(ValueClass::FtsIndex(BitmapHash { hash, len }), value); - } - } - Family::Acl => { - batch.set( - ValueClass::Acl( - key.as_slice() - .deserialize_be_u32(0) - .expect("Failed to deserialize acl"), - ), - value, - ); - } - Family::Blob => { - let hash = BlobHash::try_from_hash_slice(&key).expect("Invalid blob hash"); - - if account_id != u32::MAX && document_id != u32::MAX { - if reader.version == 1 && collection == Collection::Email { - batch.set( - ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { - due: TaskEpoch::from_inner(due), - index: SearchIndex::Email, - is_insert: true, - }), - 0u64.serialize(), - ); - due += 1; - } - batch.set(ValueClass::Blob(BlobOp::Link { hash }), vec![]); - } else { - batch_size -= value.len(); - blob_store - .put_blob(&key, &value) - .await - .expect("Failed to write blob"); - batch.set(ValueClass::Blob(BlobOp::Commit { hash }), vec![]); - } - } - Family::Config => { - batch.set(ValueClass::Config(key), value); - } - Family::LookupValue => { - batch.set(ValueClass::InMemory(InMemoryClass::Key(key)), value); - } - Family::LookupCounter => { - batch.add( - ValueClass::InMemory(InMemoryClass::Counter(key)), - i64::deserialize(&value).expect("Failed to deserialize counter"), - ); - } - Family::Directory => { - let key = key.as_slice(); - let class: DirectoryClass = - match key.first().expect("Failed to read directory key type") { - 0 => DirectoryClass::NameToId( - key.get(1..) - .expect("Failed to read directory string") - .to_vec(), - ), - 1 => DirectoryClass::EmailToId( - key.get(1..) - .expect("Failed to read directory string") - .to_vec(), - ), - 2 => DirectoryClass::Principal( - key.get(1..) - .expect("Failed to read range for principal id") - .deserialize_leb128::() - .expect("Failed to deserialize principal id"), - ), - 4 => { - batch.add( - ValueClass::Directory(DirectoryClass::UsedQuota( - key.get(1..) - .expect("Failed to read principal id") - .deserialize_leb128() - .expect("Failed to read principal id"), - )), - i64::deserialize(&value) - .expect("Failed to deserialize quota"), - ); - - continue; - } - 5 => DirectoryClass::MemberOf { - principal_id: key - .deserialize_be_u32(1) - .expect("Failed to read principal id"), - - member_of: key - .deserialize_be_u32(1 + U32_LEN) - .expect("Failed to read principal id"), - }, - 6 => DirectoryClass::Members { - principal_id: key - .deserialize_be_u32(1) - .expect("Failed to read principal id"), - - has_member: key - .deserialize_be_u32(1 + U32_LEN) - .expect("Failed to read principal id"), - }, - - _ => failed("Invalid directory key"), - }; - batch.set(ValueClass::Directory(class), value); - } - Family::Queue => { - let key = key.as_slice(); - - match key.first().expect("Failed to read queue key type") { - 0 => { - batch.set( - ValueClass::Queue(QueueClass::Message( - key.deserialize_be_u64(1) - .expect("Failed to deserialize queue message id"), - )), - value, - ); - } - 1 => { - batch.set( - ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { - due: key - .deserialize_be_u64(1) - .expect("Failed to deserialize queue message id"), - queue_id: key - .deserialize_be_u64(1 + U64_LEN) - .expect("Failed to deserialize queue message id"), - queue_name: key - .get(1 + U64_LEN + U64_LEN..) - .and_then(|bytes| bytes.try_into().ok()) - .unwrap_or_default(), - })), - value, - ); - } - _ => failed("Invalid queue key"), - } - } - Family::Index => { - batch.any_op(Operation::Index { - field: key.first().copied().expect("Failed to read index field"), - key: key.get(1..).expect("Failed to read index key").to_vec(), - set: true, - }); - } - Family::Bitmap => {} - Family::Log => { - let change_id = key - .as_slice() - .deserialize_be_u64(0) - .expect("Failed to deserialize change id"); - let change_ids = change_ids.entry(account_id).or_default(); - *change_ids = std::cmp::max(*change_ids, change_id); - - batch.any_op(Operation::Value { - class: ValueClass::Any(AnyClass { - subspace: SUBSPACE_LOGS, - key: LogKey { - account_id, - collection: collection_raw, - change_id, - } - .serialize(0), - }), - op: ValueOp::Set(value), - }); - } - Family::None => failed("No family specified in file"), + } + SUBSPACE_COUNTER | SUBSPACE_QUOTA => { + while let Some((key, value)) = reader.next() { + batch.add( + ValueClass::Any(AnyClass { + subspace: reader.subspace, + key, + }), + u64::from_le_bytes( + value + .try_into() + .expect("Failed to deserialize counter/quota"), + ) as i64, + ); + if batch.is_large_batch() { + store + .write(batch.build_all()) + .await + .failed("Failed to write batch"); + batch = BatchBuilder::new(); } } } - - if batch.len() >= 1000 || batch_size >= 5_000_000 { - store - .write(batch.build_all()) - .await - .failed("Failed to write batch"); - batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_collection(collection) - .with_document(document_id); - batch_size = 0; + _ => { + while let Some((key, value)) = reader.next() { + batch.set( + ValueClass::Any(AnyClass { + subspace: reader.subspace, + key, + }), + value, + ); + if batch.is_large_batch() { + store + .write(batch.build_all()) + .await + .failed("Failed to write batch"); + batch = BatchBuilder::new(); + } + } } } @@ -321,114 +108,87 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { .await .failed("Failed to write batch"); } - - if !change_ids.is_empty() { - let mut batch = BatchBuilder::new(); - - for (account_id, change_id) in change_ids { - batch - .with_account_id(account_id) - .add(ValueClass::ChangeId, change_id as i64); - } - - store - .write(batch.build_all()) - .await - .failed("Failed to write batch"); - } } -struct OpReader { - version: u8, - file: BufReader, +struct KeyValueReader { + subspace: u8, + file: FrameDecoder>, } -impl OpReader { - async fn new(path: &Path) -> Self { - let mut file = BufReader::new(File::open(&path).await.failed("Failed to open file")); +impl KeyValueReader { + fn new(path: &Path) -> Self { + let mut file = FrameDecoder::new(BufReader::new( + File::open(path).failed("Failed to open file"), + )); + let mut buf = [0u8; 1]; + file.read_exact(&mut buf) + .failed(&format!("Failed to read magic marker from {path:?}")); - if file - .read_u8() - .await - .failed(&format!("Failed to read magic marker from {path:?}")) - != MAGIC_MARKER - { + if buf[0] != MAGIC_MARKER { failed(&format!("Invalid magic marker in {path:?}")); } - let version = file - .read_u8() - .await - .failed(&format!("Failed to read version from {path:?}")); + file.read_exact(&mut buf) + .failed(&format!("Failed to read subspace from {path:?}")); + let subspace = buf[0]; - if version > FILE_VERSION { - failed(&format!("Invalid file version in {path:?}")); + let mut buf = [0u8; 4]; + file.read_exact(&mut buf) + .failed(&format!("Failed to read version from {path:?}")); + let version = u32::from_le_bytes(buf); + + if version != DATABASE_SCHEMA_VERSION { + failed(&format!( + "Invalid database schema version in {path:?}: Expected {DATABASE_SCHEMA_VERSION}, found {version}" + )); } - Self { file, version } + Self { file, subspace } } - async fn next(&mut self) -> Option { - match self.file.read_u8().await { - Ok(byte) => match byte { - 0 => Op::Family( - Family::try_from(self.expect_u8().await).failed("Failed to read family"), - ), - 1 => Op::KeyValue(( - self.expect_sized_bytes().await, - self.expect_sized_bytes().await, - )), - 2 => Op::KeyValue((self.expect_sized_bytes().await, vec![])), - 3 => Op::AccountId(self.expect_u32_be().await), - 4 => Op::Collection(self.expect_u8().await), - 5 => Op::DocumentId(self.expect_u32_be().await), - unknown => { - failed(&format!("Unknown op type {unknown}")); + fn next(&mut self) -> Option<(Vec, Vec)> { + let size = self.read_size()?; + + let mut key = vec![0; size as usize]; + self.file + .read_exact(&mut key) + .failed("Failed to read bytes"); + let value = self.expect_sized_bytes(); + + Some((key, value)) + } + + fn read_size(&mut self) -> Option { + let mut result = 0; + let mut buf = [0u8; 1]; + + for shift in [0, 7, 14, 21, 28] { + if let Err(err) = self.file.read_exact(&mut buf) { + if err.kind() == ErrorKind::UnexpectedEof { + return None; + } else { + failed(&format!("Failed to read file: {err:?}")); } } - .into(), - Err(err) if err.kind() == ErrorKind::UnexpectedEof => None, - Err(err) => failed(&format!("Failed to read file: {err:?}")), + + let byte = buf[0]; + if (byte & 0x80) == 0 { + result |= (byte as u32) << shift; + return Some(result); + } else { + result |= ((byte & 0x7F) as u32) << shift; + } } + + failed("Invalid leb128 sequence") } - async fn expect_u8(&mut self) -> u8 { - self.file.read_u8().await.failed("Failed to read u8") - } - - async fn expect_u32_be(&mut self) -> u32 { - self.file.read_u32().await.failed("Failed to read u32") - } - - async fn expect_sized_bytes(&mut self) -> Vec { - let len = self.expect_u32_be().await as usize; + fn expect_sized_bytes(&mut self) -> Vec { + let len = self.read_size().failed("Missing leb128 value sequence") as usize; let mut bytes = vec![0; len]; self.file .read_exact(&mut bytes) - .await .failed("Failed to read bytes"); bytes } } - -impl TryFrom for Family { - type Error = String; - - fn try_from(value: u8) -> Result { - match value { - 0 => Ok(Self::Property), - 1 => Ok(Self::FtsIndex), - 2 => Ok(Self::Acl), - 3 => Ok(Self::Blob), - 4 => Ok(Self::Config), - 5 => Ok(Self::LookupValue), - 6 => Ok(Self::LookupCounter), - 7 => Ok(Self::Directory), - 8 => Ok(Self::Queue), - 9 => Ok(Self::Index), - 10 => Ok(Self::Bitmap), - 11 => Ok(Self::Log), - other => Err(format!("Unknown family type {other}")), - } - } -} diff --git a/crates/common/src/sharing/document.rs b/crates/common/src/sharing/document.rs deleted file mode 100644 index 68e92b28..00000000 --- a/crates/common/src/sharing/document.rs +++ /dev/null @@ -1,112 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use crate::{Server, auth::AccessToken}; -use store::{ValueKey, query::acl::AclQuery, roaring::RoaringBitmap, write::ValueClass}; -use trc::AddContext; -use types::{acl::Acl, collection::Collection}; -use utils::map::bitmap::Bitmap; - -impl Server { - pub async fn shared_containers( - &self, - access_token: &AccessToken, - to_account_id: u32, - to_collection: Collection, - check_acls: impl IntoIterator, - match_any: bool, - ) -> trc::Result { - let check_acls = Bitmap::::from_iter(check_acls); - let mut document_ids = RoaringBitmap::new(); - let to_collection = u8::from(to_collection); - for &grant_account_id in [access_token.primary_id] - .iter() - .chain(access_token.member_of.clone().iter()) - { - for acl_item in self - .store() - .acl_query(AclQuery::SharedWith { - grant_account_id, - to_account_id, - to_collection, - }) - .await - .caused_by(trc::location!())? - { - let mut acls = Bitmap::::from(acl_item.permissions); - acls.intersection(&check_acls); - if acls == check_acls || (match_any && !acls.is_empty()) { - document_ids.insert(acl_item.to_document_id); - } - } - } - - Ok(document_ids) - } - - pub async fn has_access_to_document( - &self, - access_token: &AccessToken, - to_account_id: u32, - to_collection: impl Into, - to_document_id: u32, - check_acls: impl Into>, - ) -> trc::Result { - let to_collection = to_collection.into(); - let check_acls = check_acls.into(); - for grant_account_id in [access_token.primary_id] - .into_iter() - .chain(access_token.member_of.iter().copied()) - { - match self - .core - .storage - .data - .get_value::(ValueKey { - account_id: to_account_id, - collection: to_collection, - document_id: to_document_id, - class: ValueClass::Acl(grant_account_id), - }) - .await - { - Ok(Some(acls)) => { - let mut acls = Bitmap::::from(acls); - - acls.intersection(&check_acls); - if !acls.is_empty() { - return Ok(true); - } - } - Ok(None) => (), - Err(err) => { - return Err(err.caused_by(trc::location!())); - } - } - } - Ok(false) - } - - pub async fn document_acl( - &self, - grant_account_id: u32, - to_account_id: u32, - to_collection: impl Into, - to_document_id: u32, - ) -> trc::Result> { - self.core - .storage - .data - .get_value::(ValueKey { - account_id: to_account_id, - collection: to_collection.into(), - document_id: to_document_id, - class: ValueClass::Acl(grant_account_id), - }) - .await - .map(|v| v.map(Bitmap::::from).unwrap_or_default()) - } -} diff --git a/crates/common/src/sharing/mod.rs b/crates/common/src/sharing/mod.rs index 43421d47..5f14e145 100644 --- a/crates/common/src/sharing/mod.rs +++ b/crates/common/src/sharing/mod.rs @@ -10,7 +10,6 @@ use types::acl::{Acl, AclGrant, ArchivedAclGrant}; use utils::map::bitmap::Bitmap; pub mod acl; -pub mod document; pub mod notification; pub mod resources; diff --git a/crates/common/src/storage/index.rs b/crates/common/src/storage/index.rs index 3ea2b625..5020e73b 100644 --- a/crates/common/src/storage/index.rs +++ b/crates/common/src/storage/index.rs @@ -14,7 +14,7 @@ use std::{borrow::Cow, fmt::Debug}; use store::{ Serialize, SerializeInfallible, write::{ - Archive, Archiver, BatchBuilder, BlobOp, DirectoryClass, IntoOperations, Params, + Archive, Archiver, BatchBuilder, BlobLink, BlobOp, DirectoryClass, IntoOperations, Params, SearchIndex, TaskEpoch, TaskQueueClass, ValueClass, }, }; @@ -103,6 +103,14 @@ impl IndexItem<'_> { _ => false, } } + + pub fn is_none(&self) -> bool { + matches!(self, IndexItem::None) + } + + pub fn is_some(&self) -> bool { + !self.is_none() + } } impl PartialEq for IndexItem<'_> { @@ -420,17 +428,28 @@ fn build_index( ); } IndexValue::Property { field, value } => { - if set { - batch.set(field, value.into_owned()); - } else { - batch.clear(field); + if !value.is_none() { + if set { + batch.set(field, value.into_owned()); + } else { + batch.clear(field); + } } } IndexValue::Blob { value } => { if set { - batch.set(BlobOp::Link { hash: value }, vec![]); + batch.set( + BlobOp::Link { + hash: value, + to: BlobLink::Document, + }, + vec![], + ); } else { - batch.clear(BlobOp::Link { hash: value }); + batch.clear(BlobOp::Link { + hash: value, + to: BlobLink::Document, + }); } } IndexValue::Acl { value } => { @@ -566,12 +585,25 @@ fn merge_index( batch.clear(old_field); batch.set(new_field, new_value.into_owned()); } else if new_value != old_value { - batch.set(old_field, new_value.into_owned()); + if new_value.is_some() { + batch.set(old_field, new_value.into_owned()); + } else { + batch.clear(old_field); + } } } (IndexValue::Blob { value: old_hash }, IndexValue::Blob { value: new_hash }) => { - batch.clear(BlobOp::Link { hash: old_hash }); - batch.set(BlobOp::Link { hash: new_hash }, vec![]); + batch.clear(BlobOp::Link { + hash: old_hash, + to: BlobLink::Document, + }); + batch.set( + BlobOp::Link { + hash: new_hash, + to: BlobLink::Document, + }, + vec![], + ); } (IndexValue::Acl { value: old_acl }, IndexValue::Acl { value: new_acl }) => { let has_old_acl = !old_acl.is_empty(); diff --git a/crates/dav/src/file/update.rs b/crates/dav/src/file/update.rs index 6f4d39cb..93420c5e 100644 --- a/crates/dav/src/file/update.rs +++ b/crates/dav/src/file/update.rs @@ -161,11 +161,10 @@ impl FileUpdateRequestHandler for Server { } // Write blob - let blob_hash = self - .put_blob(account_id, &bytes, false) + let (blob_hash, blob_hold) = self + .put_temporary_blob(account_id, &bytes, 60) .await - .caused_by(trc::location!())? - .hash; + .caused_by(trc::location!())?; // Build node let mut new_node = node.deserialize::().caused_by(trc::location!())?; @@ -184,6 +183,7 @@ impl FileUpdateRequestHandler for Server { .with_account_id(account_id) .with_collection(Collection::FileNode) .with_document(document_id) + .clear(blob_hold) .custom( ObjectIndexBuilder::new() .with_current(node) @@ -241,11 +241,10 @@ impl FileUpdateRequestHandler for Server { } // Write blob - let blob_hash = self - .put_blob(account_id, &bytes, false) + let (blob_hash, blob_hold) = self + .put_temporary_blob(account_id, &bytes, 60) .await - .caused_by(trc::location!())? - .hash; + .caused_by(trc::location!())?; // Build node let now = now(); @@ -280,6 +279,7 @@ impl FileUpdateRequestHandler for Server { .with_account_id(account_id) .with_collection(Collection::FileNode) .with_document(document_id) + .clear(blob_hold) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(node) diff --git a/crates/directory/src/backend/internal/manage.rs b/crates/directory/src/backend/internal/manage.rs index 1db90b56..86f106aa 100644 --- a/crates/directory/src/backend/internal/manage.rs +++ b/crates/directory/src/backend/internal/manage.rs @@ -829,11 +829,6 @@ impl ManageDirectory for Store { } // SPDX-SnippetEnd - // Unlink all principal's blobs - self.blob_hash_unlink_account(principal_id) - .await - .caused_by(trc::location!())?; - // Revoke ACLs, obtain all changed principals let mut changed_principals = ChangedPrincipals::default(); @@ -849,11 +844,6 @@ impl ManageDirectory for Store { ); } - // Delete principal data - self.danger_destroy_account(principal_id) - .await - .caused_by(trc::location!())?; - // Delete principal batch .with_document(principal_id) diff --git a/crates/email/src/message/copy.rs b/crates/email/src/message/copy.rs index 156f26d4..d8bc52d6 100644 --- a/crates/email/src/message/copy.rs +++ b/crates/email/src/message/copy.rs @@ -153,21 +153,15 @@ impl EmailCopy for Server { // Assign IMAP UIDs let mut mailbox_ids = Vec::with_capacity(mailboxes.len()); email.imap_uids = Vec::with_capacity(mailboxes.len()); - for mailbox_id in &mailboxes { - let uid = self - .assign_imap_uid(account_id, *mailbox_id) - .await - .caused_by(trc::location!())?; - mailbox_ids.push(UidMailbox::new(*mailbox_id, uid)); - email.imap_uids.push(uid); - } - - // Obtain documentId - let document_id = self - .store() - .assign_document_ids(account_id, Collection::Email, 1) + let mut ids = self + .assign_email_ids(account_id, mailboxes.iter().copied(), true) .await .caused_by(trc::location!())?; + let document_id = ids.next().unwrap(); + for (uid, mailbox_id) in ids.zip(mailboxes.iter().copied()) { + mailbox_ids.push(UidMailbox::new(mailbox_id, uid)); + email.imap_uids.push(uid); + } // Prepare batch let mut batch = BatchBuilder::new(); diff --git a/crates/email/src/message/delete.rs b/crates/email/src/message/delete.rs index b10ea2b2..566f5f50 100644 --- a/crates/email/src/message/delete.rs +++ b/crates/email/src/message/delete.rs @@ -5,7 +5,6 @@ */ use super::metadata::MessageData; -use crate::{cache::MessageCacheFetch, mailbox::*}; use common::{KV_LOCK_PURGE_ACCOUNT, Server, storage::index::ObjectIndexBuilder}; use directory::backend::internal::manage::ManageDirectory; use groupware::calendar::storage::ItipAutoExpunge; @@ -20,7 +19,7 @@ use store::{ }; use trc::AddContext; use types::collection::{Collection, VanishedCollection}; -use types::field::EmailSubmissionField; +use types::field::{EmailField, EmailSubmissionField}; pub trait EmailDeletion: Sync + Send { fn emails_delete( @@ -213,27 +212,9 @@ impl EmailDeletion for Server { } async fn emails_auto_expunge(&self, account_id: u32, hold_period: u64) -> trc::Result<()> { - let trashed_ids = RoaringBitmap::from_iter( - self.get_cached_messages(account_id) - .await - .caused_by(trc::location!())? - .emails - .items - .iter() - .filter(|item| { - item.mailboxes - .iter() - .any(|id| id.mailbox_id == TRASH_ID || id.mailbox_id == JUNK_ID) - }) - .map(|item| item.document_id), - ); - if trashed_ids.is_empty() { - return Ok(()); - } - // Filter messages by received date - let todo = "fix"; - /*let mut destroy_ids = RoaringBitmap::new(); + let mut destroy_ids = RoaringBitmap::new(); + let cutoff = now().saturating_sub(hold_period); self.store() .iterate( IterateParams::new( @@ -241,33 +222,23 @@ impl EmailDeletion for Server { account_id, collection: Collection::Email.into(), document_id: 0, - class: ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: EmailField::ReceivedToSize.into(), - value: 0, - }), + class: ValueClass::Property(EmailField::DeletedAt.into()), }, ValueKey { account_id, collection: Collection::Email.into(), document_id: u32::MAX, - class: ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: EmailField::ReceivedToSize.into(), - value: now().saturating_sub(hold_period), - }), + class: ValueClass::Property(EmailField::DeletedAt.into()), }, ) - .ascending() - .no_values(), - |key, _| { - let document_id = key - .deserialize_be_u32(key.len() - U32_LEN) - .caused_by(trc::location!())?; - - if trashed_ids.contains(document_id) { - destroy_ids.insert(document_id); + .ascending(), + |key, value| { + let deleted_at = value.deserialize_be_u64(0)?; + if deleted_at <= cutoff { + destroy_ids.insert(key.deserialize_be_u32(key.len() - U32_LEN)?); } - Ok(trashed_ids.len() != destroy_ids.len()) + Ok(true) }, ) .await @@ -286,10 +257,16 @@ impl EmailDeletion for Server { // Delete messages let mut batch = BatchBuilder::new(); - self.emails_delete(account_id, &mut batch, destroy_ids) + let tenant_id = self + .store() + .get_principal(account_id) + .await + .caused_by(trc::location!())? + .and_then(|p| p.tenant()); + self.emails_delete(account_id, tenant_id, &mut batch, destroy_ids) .await?; self.commit_batch(batch).await?; - self.notify_task_queue();*/ + self.notify_task_queue(); Ok(()) } diff --git a/crates/email/src/message/index/metadata.rs b/crates/email/src/message/index/metadata.rs index 34390242..50eff879 100644 --- a/crates/email/src/message/index/metadata.rs +++ b/crates/email/src/message/index/metadata.rs @@ -20,7 +20,7 @@ use mail_parser::{ }; use store::{ Serialize, - write::{Archiver, BatchBuilder, BlobOp, IndexPropertyClass, ValueClass}, + write::{Archiver, BatchBuilder, BlobLink, BlobOp, IndexPropertyClass, ValueClass}, }; use trc::AddContext; use types::{blob_hash::BlobHash, field::EmailField}; @@ -38,6 +38,7 @@ impl MessageMetadata { .set( BlobOp::Link { hash: self.blob_hash.clone(), + to: BlobLink::Document, }, Vec::new(), ) @@ -46,6 +47,7 @@ impl MessageMetadata { batch .clear(BlobOp::Link { hash: self.blob_hash.clone(), + to: BlobLink::Document, }) .clear(EmailField::Metadata); } @@ -90,6 +92,7 @@ impl ArchivedMessageMetadata { })) .clear(BlobOp::Link { hash: BlobHash::from(&self.blob_hash), + to: BlobLink::Document, }); } } @@ -226,6 +229,7 @@ impl IndexMessage for BatchBuilder { self.set( BlobOp::Link { hash: metadata.blob_hash.clone(), + to: BlobLink::Document, }, Vec::new(), ) diff --git a/crates/email/src/message/index/mod.rs b/crates/email/src/message/index/mod.rs index 85e31ed1..c0c397db 100644 --- a/crates/email/src/message/index/mod.rs +++ b/crates/email/src/message/index/mod.rs @@ -4,9 +4,13 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::message::metadata::{ArchivedMessageData, MessageData}; -use common::storage::index::{IndexValue, IndexableObject}; -use types::{blob_hash::BlobHash, collection::SyncCollection}; +use crate::{ + mailbox::{JUNK_ID, TRASH_ID}, + message::metadata::{ArchivedMessageData, MessageData}, +}; +use common::storage::index::{IndexItem, IndexValue, IndexableObject}; +use store::write::now; +use types::{blob_hash::BlobHash, collection::SyncCollection, field::EmailField}; pub mod extractors; pub mod metadata; @@ -17,7 +21,23 @@ pub const PREVIEW_LENGTH: usize = 256; impl IndexableObject for MessageData { fn index_values(&self) -> impl Iterator> { + let mut mailboxes = Vec::with_capacity(self.mailboxes.len()); + let mut is_in_trash = false; + + for mailbox in &self.mailboxes { + mailboxes.push(mailbox.mailbox_id); + is_in_trash |= mailbox.mailbox_id == TRASH_ID || mailbox.mailbox_id == JUNK_ID; + } + [ + IndexValue::Property { + field: EmailField::DeletedAt.into(), + value: if is_in_trash { + IndexItem::from(now()) + } else { + IndexItem::None + }, + }, IndexValue::Quota { used: self.size }, IndexValue::LogItem { sync_collection: SyncCollection::Email, @@ -29,7 +49,7 @@ impl IndexableObject for MessageData { }, IndexValue::LogContainerProperty { sync_collection: SyncCollection::Email, - ids: self.mailboxes.iter().map(|m| m.mailbox_id).collect(), + ids: mailboxes, }, ] .into_iter() @@ -38,7 +58,24 @@ impl IndexableObject for MessageData { impl IndexableObject for &ArchivedMessageData { fn index_values(&self) -> impl Iterator> { + let mut mailboxes = Vec::with_capacity(self.mailboxes.len()); + let mut is_in_trash = false; + + for mailbox in self.mailboxes.iter() { + let mailbox_id = mailbox.mailbox_id.to_native(); + mailboxes.push(mailbox_id); + is_in_trash |= mailbox_id == TRASH_ID || mailbox_id == JUNK_ID; + } + [ + IndexValue::Property { + field: EmailField::DeletedAt.into(), + value: if is_in_trash { + IndexItem::from(now()) + } else { + IndexItem::None + }, + }, IndexValue::Quota { used: self.size.to_native(), }, @@ -52,11 +89,7 @@ impl IndexableObject for &ArchivedMessageData { }, IndexValue::LogContainerProperty { sync_collection: SyncCollection::Email, - ids: self - .mailboxes - .iter() - .map(|m| m.mailbox_id.to_native()) - .collect(), + ids: mailboxes, }, ] .into_iter() diff --git a/crates/email/src/message/ingest.rs b/crates/email/src/message/ingest.rs index 20679ba4..a59251bf 100644 --- a/crates/email/src/message/ingest.rs +++ b/crates/email/src/message/ingest.rs @@ -33,8 +33,8 @@ use store::{ IndexKeyPrefix, IterateParams, U32_LEN, ValueKey, ahash::{AHashMap, AHashSet}, write::{ - BatchBuilder, IndexPropertyClass, SearchIndex, TaskEpoch, TaskQueueClass, ValueClass, - key::DeserializeBigEndian, now, + AssignedId, AssignedIds, BatchBuilder, IndexPropertyClass, SearchIndex, TaskEpoch, + TaskQueueClass, ValueClass, key::DeserializeBigEndian, now, }, }; use trc::{AddContext, MessageIngestEvent}; @@ -93,11 +93,12 @@ pub trait EmailIngest: Sync + Send { thread_name: &str, message_ids: &[CheekyHash], ) -> impl Future> + Send; - fn assign_imap_uid( + fn assign_email_ids( &self, account_id: u32, - mailbox_id: u32, - ) -> impl Future> + Send; + mailbox_ids: impl IntoIterator + Sync + Send, + generate_email_id: bool, + ) -> impl Future + 'static>> + Send; fn email_bayes_can_train(&self, access_token: &AccessToken) -> bool; } @@ -536,24 +537,25 @@ impl EmailIngest for Server { } // Store blob - let blob_hash = if let Some(blob_hash) = params.blob_hash { - blob_hash.clone() + let (blob_hash, blob_hold) = if let Some(blob_hash) = params.blob_hash { + (blob_hash.clone(), None) } else { - self.put_blob(account_id, raw_message.as_ref(), false) + self.put_temporary_blob(account_id, raw_message.as_ref(), 60) .await + .map(|(hash, op)| (hash, Some(op))) .caused_by(trc::location!())? - .hash }; // Assign IMAP UIDs let mut mailbox_ids = Vec::with_capacity(params.mailbox_ids.len()); let mut imap_uids = Vec::with_capacity(params.mailbox_ids.len()); - for mailbox_id in ¶ms.mailbox_ids { - let uid = self - .assign_imap_uid(account_id, *mailbox_id) - .await - .caused_by(trc::location!())?; - mailbox_ids.push(UidMailbox::new(*mailbox_id, uid)); + let mut ids = self + .assign_email_ids(account_id, params.mailbox_ids.iter().copied(), true) + .await + .caused_by(trc::location!())?; + let document_id = ids.next().unwrap(); + for (uid, mailbox_id) in ids.zip(params.mailbox_ids.iter().copied()) { + mailbox_ids.push(UidMailbox::new(mailbox_id, uid)); imap_uids.push(uid); } @@ -565,13 +567,6 @@ impl EmailIngest for Server { .collect::>(); batch.with_account_id(account_id); - // Obtain document ID - let document_id = self - .store() - .assign_document_ids(account_id, Collection::Email, 1) - .await - .caused_by(trc::location!())?; - // Determine thread id let thread_id = if let Some(thread_id) = thread_result.thread_id { thread_id @@ -619,6 +614,10 @@ impl EmailIngest for Server { vec![], ); + if let Some(blob_hold) = blob_hold { + batch.clear(blob_hold); + } + // Merge threads if necessary if let Some(merge_threads) = MergeThreadIds::new(thread_result).serialize() { batch.set( @@ -792,20 +791,48 @@ impl EmailIngest for Server { } } - async fn assign_imap_uid(&self, account_id: u32, mailbox_id: u32) -> trc::Result { + async fn assign_email_ids( + &self, + account_id: u32, + mailbox_ids: impl IntoIterator + Sync + Send, + generate_email_id: bool, + ) -> trc::Result + 'static> { // Increment UID next let mut batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_collection(Collection::Mailbox) - .with_document(mailbox_id) - .add_and_get(MailboxField::UidCounter, 1); - self.core - .storage - .data - .write(batch.build_all()) - .await - .and_then(|v| v.last_counter_id().map(|id| id as u32)) + batch.with_account_id(account_id); + + let mut expected_ids = 0; + if generate_email_id { + batch + .with_collection(Collection::Email) + .add_and_get(ValueClass::DocumentId, 1); + expected_ids += 1; + } + + batch.with_collection(Collection::Mailbox); + + for mailbox_id in mailbox_ids { + batch + .with_document(mailbox_id) + .add_and_get(MailboxField::UidCounter, 1); + expected_ids += 1; + } + + let ids = if expected_ids > 0 { + self.core.storage.data.write(batch.build_all()).await? + } else { + AssignedIds::default() + }; + if ids.ids.len() == expected_ids { + Ok(ids.ids.into_iter().map(|id| match id { + AssignedId::Counter(id) => id as u32, + AssignedId::ChangeId(_) => unreachable!(), + })) + } else { + Err(trc::StoreEvent::UnexpectedError + .caused_by(trc::location!()) + .ctx(trc::Key::Reason, "No all document ids were generated")) + } } fn email_bayes_can_train(&self, access_token: &AccessToken) -> bool { diff --git a/crates/email/src/message/metadata.rs b/crates/email/src/message/metadata.rs index a82970f7..0c648fb8 100644 --- a/crates/email/src/message/metadata.rs +++ b/crates/email/src/message/metadata.rs @@ -994,6 +994,16 @@ impl ArchivedMetadataHeaderValue { _ => None, } } + + pub fn as_single_address(&self) -> Option<&ArchivedMetadataAddress> { + match self { + ArchivedMetadataHeaderValue::AddressList(list) => list.first(), + ArchivedMetadataHeaderValue::AddressGroup(groups) => { + groups.first().and_then(|g| g.addresses.first()) + } + _ => None, + } + } } impl ArchivedUidMailbox { diff --git a/crates/email/src/sieve/ingest.rs b/crates/email/src/sieve/ingest.rs index 6f828b21..fe4df68a 100644 --- a/crates/email/src/sieve/ingest.rs +++ b/crates/email/src/sieve/ingest.rs @@ -23,7 +23,9 @@ use store::{ Deserialize, Serialize, ValueKey, ahash::AHashMap, dispatch::lookup::KeyValue, - write::{AlignedBytes, Archive, ArchiveVersion, Archiver, BatchBuilder, BlobOp, ValueClass}, + write::{ + AlignedBytes, Archive, ArchiveVersion, Archiver, BatchBuilder, BlobLink, BlobOp, ValueClass, + }, }; use trc::{AddContext, SieveEvent}; use types::{ @@ -697,10 +699,9 @@ impl SieveScriptIngest for Server { updated_sieve_bytes.extend_from_slice(&compiled_bytes); // Store updated blob - let new_blob_hash = self - .put_blob(account_id, &updated_sieve_bytes, false) - .await? - .hash; + let (new_blob_hash, new_blob_hold) = self + .put_temporary_blob(account_id, &updated_sieve_bytes, 60) + .await?; let mut new_script_object = rkyv::deserialize(unarchived_script).caused_by(trc::location!())?; let blob_hash = @@ -718,13 +719,18 @@ impl SieveScriptIngest for Server { SieveField::Archive, new_archive.serialize().caused_by(trc::location!())?, ) - .clear(BlobOp::Link { hash: blob_hash }) + .clear(BlobOp::Link { + hash: blob_hash, + to: BlobLink::Document, + }) .set( BlobOp::Link { hash: new_blob_hash, + to: BlobLink::Document, }, Vec::new(), - ); + ) + .clear(new_blob_hold); self.store() .write(batch.build_all()) .await diff --git a/crates/http/src/form/mod.rs b/crates/http/src/form/mod.rs index cf0fde45..74532d2a 100644 --- a/crates/http/src/form/mod.rs +++ b/crates/http/src/form/mod.rs @@ -25,13 +25,8 @@ use mail_builder::{ }; use serde_json::json; use std::{borrow::Cow, fmt::Write, future::Future}; -use store::{ - SerializeInfallible, - write::{BatchBuilder, BlobOp, now}, -}; +use store::write::BatchBuilder; use trc::AddContext; -use types::blob_hash::BlobHash; -use x509_parser::nom::AsBytes; pub trait FormHandler: Sync + Send { fn handle_contact_form( @@ -175,22 +170,8 @@ impl FormHandler for Server { .unwrap_or_default(); // Reserve and write blob - let message_blob = BlobHash::generate(message.as_bytes()); - let message_size = message.len() as u64; - let mut batch = BatchBuilder::new(); - batch.set( - BlobOp::Reserve { - hash: message_blob.clone(), - until: now() + 120, - }, - 0u32.serialize(), - ); - self.store() - .write(batch.build_all()) - .await - .caused_by(trc::location!())?; - self.blob_store() - .put_blob(message_blob.as_slice(), message.as_ref()) + let (message_blob, blob_hold) = self + .put_temporary_blob(u32::MAX, &message, 60) .await .caused_by(trc::location!())?; @@ -200,7 +181,7 @@ impl FormHandler for Server { sender_authenticated: false, recipients: form.rcpt_to.clone(), message_blob, - message_size, + message_size: message.len() as u64, session_id: session.session_id, }) .await @@ -217,6 +198,14 @@ impl FormHandler for Server { } } + // Remove blob hold + let mut batch = BatchBuilder::new(); + batch.clear(blob_hold); + self.store() + .write(batch.build_all()) + .await + .caused_by(trc::location!())?; + // Suppress errors if there is at least one success if has_success { failure = None; diff --git a/crates/http/src/management/enterprise/undelete.rs b/crates/http/src/management/enterprise/undelete.rs index ec5398b5..5092e1b7 100644 --- a/crates/http/src/management/enterprise/undelete.rs +++ b/crates/http/src/management/enterprise/undelete.rs @@ -9,25 +9,24 @@ */ use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; -use common::{Server, enterprise::undelete::DeletedBlob}; +use common::{Server, enterprise::undelete::DeletedItemType}; use directory::backend::internal::manage::ManageDirectory; use email::{ mailbox::INBOX_ID, message::ingest::{EmailIngest, IngestEmail, IngestSource}, }; +use http_proto::{request::decode_path_element, *}; use hyper::Method; use mail_parser::{DateTime, MessageParser}; use serde_json::json; use std::future::Future; use std::str::FromStr; -use store::write::{BatchBuilder, BlobOp, ValueClass}; +use store::write::{BatchBuilder, BlobLink, BlobOp}; use trc::AddContext; use types::{blob_hash::BlobHash, collection::Collection}; use utils::url_params::UrlParams; -use http_proto::{request::decode_path_element, *}; - -#[derive(serde::Deserialize, serde::Serialize)] +#[derive(serde::Deserialize, serde::Serialize, Debug)] pub struct UndeleteRequest { pub hash: H, pub collection: C, @@ -47,6 +46,41 @@ pub enum UndeleteResponse { Error { reason: String }, } +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub struct DeletedBlobResponse { + pub hash: String, + pub size: u32, + #[serde(rename = "deletedAt")] + pub deleted_at: String, + #[serde(rename = "expiresAt")] + pub expires_at: String, + pub item: DeletedItemResponse, +} + +#[derive(Debug, serde::Serialize, serde::Deserialize)] +#[serde(tag = "type")] +#[serde(rename_all = "camelCase")] +pub enum DeletedItemResponse { + Email { + from: Box, + subject: Box, + received_at: String, + }, + FileNode { + name: Box, + }, + CalendarEvent { + title: Box, + start_time: String, + }, + ContactCard { + name: Box, + }, + SieveScript { + name: Box, + }, +} + pub trait UndeleteApi: Sync + Send { fn handle_undelete_api_request( &self, @@ -87,19 +121,46 @@ impl UndeleteApi for Server { // Sort ascending by deleted_at let total = deleted.len(); - deleted.sort_by(|a, b| a.deleted_at.cmp(&b.deleted_at)); + deleted.sort_by(|a, b| a.item.deleted_at.cmp(&b.item.deleted_at)); let mut results = Vec::with_capacity(if limit > 0 { limit } else { total }); for blob in deleted { if offset == 0 { - results.push(DeletedBlob { + results.push(DeletedBlobResponse { hash: URL_SAFE_NO_PAD.encode(blob.hash.as_slice()), - size: blob.size, - deleted_at: DateTime::from_timestamp(blob.deleted_at as i64) + size: blob.item.size, + deleted_at: DateTime::from_timestamp(blob.item.deleted_at as i64) .to_rfc3339(), expires_at: DateTime::from_timestamp(blob.expires_at as i64) .to_rfc3339(), - collection: Collection::from(blob.collection).to_string(), + item: match blob.item.typ { + DeletedItemType::Email { + from, + subject, + received_at, + } => DeletedItemResponse::Email { + from, + subject, + received_at: DateTime::from_timestamp(received_at as i64) + .to_rfc3339(), + }, + DeletedItemType::FileNode { name } => { + DeletedItemResponse::FileNode { name } + } + DeletedItemType::CalendarEvent { title, start_time } => { + DeletedItemResponse::CalendarEvent { + title, + start_time: DateTime::from_timestamp(start_time as i64) + .to_rfc3339(), + } + } + DeletedItemType::ContactCard { name } => { + DeletedItemResponse::ContactCard { name } + } + DeletedItemType::SieveScript { name } => { + DeletedItemResponse::SieveScript { name } + } + }, }); if results.len() == limit { break; @@ -169,8 +230,20 @@ impl UndeleteApi for Server { for blob in deleted { results.push(UndeleteRequest { hash: blob.hash, - collection: Collection::from(blob.collection), - time: blob.deleted_at, + collection: match blob.item.typ { + DeletedItemType::Email { .. } => Collection::Email, + DeletedItemType::FileNode { .. } => Collection::FileNode, + DeletedItemType::CalendarEvent { .. } => { + Collection::CalendarEvent + } + DeletedItemType::ContactCard { .. } => { + Collection::ContactCard + } + DeletedItemType::SieveScript { .. } => { + Collection::SieveScript + } + }, + time: blob.item.deleted_at, cancel_deletion: blob.expires_at.into(), }); } @@ -216,10 +289,17 @@ impl UndeleteApi for Server { Ok(_) => { results.push(UndeleteResponse::Success); if let Some(cancel_deletion) = request.cancel_deletion { - batch.clear(ValueClass::Blob(BlobOp::Reserve { - hash: request.hash, - until: cancel_deletion, - })); + batch + .clear(BlobOp::Link { + hash: request.hash.clone(), + to: BlobLink::Temporary { + until: cancel_deletion, + }, + }) + .clear(BlobOp::Undelete { + hash: request.hash, + until: cancel_deletion, + }); } } Err(mut err) diff --git a/crates/http/src/management/principal.rs b/crates/http/src/management/principal.rs index 7ea14714..d081698e 100644 --- a/crates/http/src/management/principal.rs +++ b/crates/http/src/management/principal.rs @@ -4,7 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{KV_BAYES_MODEL_USER, Server, auth::AccessToken}; +use crate::management::stores::destroy_account_data; +use common::{Server, auth::AccessToken}; use directory::{ DirectoryInner, Permission, PrincipalData, QueryBy, QueryParams, Type, backend::internal::{ @@ -20,7 +21,6 @@ use hyper::{Method, header}; use serde_json::json; use std::future::Future; use std::sync::Arc; -use store::{search::SearchQuery, write::SearchIndex}; use trc::AddContext; use utils::url_params::UrlParams; @@ -376,12 +376,6 @@ impl PrincipalManager for Server { if found { let server = self.clone(); tokio::spawn(async move { - let has_bayes = server - .core - .spam - .bayes - .as_ref() - .is_some_and(|c| c.account_classify); for principal in principals.items { // Delete account match server @@ -399,41 +393,14 @@ impl PrincipalManager for Server { } } - if matches!(typ, Type::Individual | Type::Group) { - // Remove search index - for index in [ - SearchIndex::Email, - SearchIndex::Contacts, - SearchIndex::Calendar, - ] { - if let Err(err) = server - .core - .storage - .fts - .unindex( - SearchQuery::new(index).with_account_id(principal.id()), - ) - .await - { - trc::error!(err.details("Failed to delete FTS index")); - } - } - - // Delete bayes model - if has_bayes { - let mut key = - Vec::with_capacity(std::mem::size_of::() + 1); - key.push(KV_BAYES_MODEL_USER); - key.extend_from_slice(&principal.id().to_be_bytes()); - - if let Err(err) = - server.in_memory_store().key_delete_prefix(&key).await - { - trc::error!( - err.details("Failed to delete user bayes model") - ); - } - } + if let Err(err) = destroy_account_data( + &server, + principal.id(), + matches!(typ, Type::Individual | Type::Group), + ) + .await + { + trc::error!(err.details("Failed to delete principal")); } } }); @@ -527,42 +494,14 @@ impl PrincipalManager for Server { .delete_principal(QueryBy::Id(account_id)) .await?; - if matches!(typ, Type::Individual | Type::Group) { - // Remove FTS index - for index in [ - SearchIndex::Email, - SearchIndex::Contacts, - SearchIndex::Calendar, - ] { - if let Err(err) = self - .core - .storage - .fts - .unindex(SearchQuery::new(index).with_account_id(account_id)) - .await - { - trc::error!(err.details("Failed to delete FTS index")); - } - } - - // Delete bayes model - if self - .core - .spam - .bayes - .as_ref() - .is_some_and(|c| c.account_classify) - { - let mut key = Vec::with_capacity(std::mem::size_of::() + 1); - key.push(KV_BAYES_MODEL_USER); - key.extend_from_slice(&account_id.to_be_bytes()); - - if let Err(err) = - self.in_memory_store().key_delete_prefix(&key).await - { - trc::error!(err.details("Failed to delete user bayes model")); - } - } + if let Err(err) = destroy_account_data( + self, + account_id, + matches!(typ, Type::Individual | Type::Group), + ) + .await + { + trc::error!(err.details("Failed to delete principal")); } // Increment revision diff --git a/crates/http/src/management/stores.rs b/crates/http/src/management/stores.rs index 3b1701ee..11519872 100644 --- a/crates/http/src/management/stores.rs +++ b/crates/http/src/management/stores.rs @@ -18,7 +18,11 @@ use directory::{ }; use email::{ cache::MessageCacheFetch, - message::{ingest::EmailIngest, metadata::MessageData}, + message::{ + ingest::EmailIngest, + metadata::{MessageData, MessageMetadata}, + }, + sieve::SieveScript, }; use groupware::{ calendar::{Calendar, CalendarEvent, CalendarEventNotification}, @@ -32,12 +36,14 @@ use services::task_manager::index::ReindexIndexTask; use std::future::Future; use store::{ Serialize, rand, - write::{Archiver, BatchBuilder, DirectoryClass, SearchIndex, ValueClass}, + search::SearchQuery, + write::{Archiver, BatchBuilder, BlobLink, BlobOp, DirectoryClass, SearchIndex, ValueClass}, }; use trc::AddContext; use types::{ + blob_hash::BlobHash, collection::Collection, - field::{EmailField, MailboxField}, + field::{EmailField, Field, MailboxField}, }; use utils::url_params::UrlParams; @@ -398,6 +404,138 @@ pub async fn recalculate_quota(server: &Server, account_id: u32) -> trc::Result< .map(|_| ()) } +pub async fn destroy_account_blobs(server: &Server, account_id: u32) -> trc::Result<()> { + let mut delete_keys = Vec::new(); + for (collection, field) in [ + (Collection::Email, u8::from(EmailField::Metadata)), + (Collection::FileNode, u8::from(Field::ARCHIVE)), + (Collection::SieveScript, u8::from(Field::ARCHIVE)), + ] { + server + .all_archives(account_id, collection, field, |document_id, archive| { + match collection { + Collection::Email => { + let message = archive.unarchive::()?; + delete_keys.push(( + collection, + document_id, + BlobHash::from(&message.blob_hash), + )); + } + Collection::FileNode => { + if let Some(file) = archive.unarchive::()?.file.as_ref() { + delete_keys.push(( + collection, + document_id, + BlobHash::from(&file.blob_hash), + )); + } + } + Collection::SieveScript => { + let sieve = archive.unarchive::()?; + delete_keys.push(( + collection, + document_id, + BlobHash::from(&sieve.blob_hash), + )); + } + _ => {} + } + Ok(()) + }) + .await + .caused_by(trc::location!())?; + } + + let mut batch = BatchBuilder::new(); + batch.with_account_id(account_id); + + for (collection, document_id, hash) in delete_keys { + if batch.is_large_batch() { + server + .store() + .write(batch.build_all()) + .await + .caused_by(trc::location!())?; + batch = BatchBuilder::new(); + batch.with_account_id(account_id); + } + batch + .with_collection(collection) + .with_document(document_id) + .clear(ValueClass::Blob(BlobOp::Link { + hash, + to: BlobLink::Document, + })); + } + + if !batch.is_empty() { + server + .store() + .write(batch.build_all()) + .await + .caused_by(trc::location!())?; + } + + Ok(()) +} + +pub async fn destroy_account_data( + server: &Server, + account_id: u32, + has_data: bool, +) -> trc::Result<()> { + // Unlink all accounts's blobs + if has_data { + destroy_account_blobs(server, account_id).await?; + } + + // Destroy account data + server + .store() + .danger_destroy_account(account_id) + .await + .caused_by(trc::location!())?; + + if has_data { + // Remove search index + for index in [ + SearchIndex::Email, + SearchIndex::Contacts, + SearchIndex::Calendar, + ] { + if let Err(err) = server + .core + .storage + .fts + .unindex(SearchQuery::new(index).with_account_id(account_id)) + .await + { + trc::error!(err.details("Failed to delete FTS index")); + } + } + + // Delete bayes model + if server + .core + .spam + .bayes + .as_ref() + .is_some_and(|c| c.account_classify) + { + let mut key = Vec::with_capacity(std::mem::size_of::() + 1); + key.push(KV_BAYES_MODEL_USER); + key.extend_from_slice(&account_id.to_be_bytes()); + + if let Err(err) = server.in_memory_store().key_delete_prefix(&key).await { + trc::error!(err.details("Failed to delete user bayes model")); + } + } + } + + Ok(()) +} + pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u32, u32)> { let mut mailbox_count = 0; let mut email_count = 0; @@ -455,11 +593,17 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u .deserialize::() .caused_by(trc::location!())?; - for uid_mailbox in &mut new_data.mailboxes { - uid_mailbox.uid = server - .assign_imap_uid(account_id, uid_mailbox.mailbox_id) - .await - .caused_by(trc::location!())?; + let ids = server + .assign_email_ids( + account_id, + new_data.mailboxes.iter().map(|m| m.mailbox_id), + false, + ) + .await + .caused_by(trc::location!())?; + + for (uid_mailbox, uid) in new_data.mailboxes.iter_mut().zip(ids) { + uid_mailbox.uid = uid; } // Prepare write batch diff --git a/crates/imap/src/op/copy_move.rs b/crates/imap/src/op/copy_move.rs index cdafaeb1..ebe2f38a 100644 --- a/crates/imap/src/op/copy_move.rs +++ b/crates/imap/src/op/copy_move.rs @@ -29,6 +29,7 @@ use store::{ roaring::RoaringBitmap, write::{AlignedBytes, Archive, BatchBuilder, TaskEpoch, TaskQueueClass, ValueClass}, }; +use trc::AddContext; use types::{ acl::Acl, collection::{Collection, VanishedCollection}, @@ -277,17 +278,28 @@ impl SessionData { } // Assign IMAP UIDs - for uid_mailbox in &mut new_data.mailboxes { - if uid_mailbox.uid == 0 { - let assigned_uid = self - .server - .assign_imap_uid(account_id, uid_mailbox.mailbox_id) - .await - .imap_ctx(&arguments.tag, trc::location!())?; - debug_assert!(assigned_uid > 0); - copied_ids.push((imap_id.uid, assigned_uid)); - uid_mailbox.uid = assigned_uid; - } + let ids = self + .server + .assign_email_ids( + account_id, + new_data + .mailboxes + .iter() + .filter(|m| m.uid == 0) + .map(|m| m.mailbox_id), + false, + ) + .await + .caused_by(trc::location!())?; + + for (uid_mailbox, uid) in new_data + .mailboxes + .iter_mut() + .filter(|m| m.uid == 0) + .zip(ids) + { + copied_ids.push((imap_id.uid, uid)); + uid_mailbox.uid = uid; } // Prepare write batch diff --git a/crates/jmap/Cargo.toml b/crates/jmap/Cargo.toml index b941e590..932a6197 100644 --- a/crates/jmap/Cargo.toml +++ b/crates/jmap/Cargo.toml @@ -45,7 +45,7 @@ tungstenite = "0.28" chrono = "0.4" rand = "0.9.0" pkcs8 = { version = "0.10.2", features = ["alloc", "std"] } -lz4_flex = { version = "0.11", default-features = false } +lz4_flex = { version = "0.12", default-features = false } aes-gcm = "0.10.1" aes-gcm-siv = "0.11.1" rsa = "0.9.2" diff --git a/crates/jmap/src/blob/copy.rs b/crates/jmap/src/blob/copy.rs index 2ad64780..b1abae9c 100644 --- a/crates/jmap/src/blob/copy.rs +++ b/crates/jmap/src/blob/copy.rs @@ -15,7 +15,7 @@ use jmap_proto::{ use std::future::Future; use store::{ SerializeInfallible, - write::{BatchBuilder, BlobOp, now}, + write::{BatchBuilder, BlobLink, BlobOp, now}, }; use trc::AddContext; use types::blob::{BlobClass, BlobId}; @@ -74,9 +74,9 @@ impl BlobCopy for Server { let mut batch = BatchBuilder::new(); let until = now() + self.core.jmap.upload_tmp_ttl; batch.with_account_id(account_id).set( - BlobOp::Reserve { - until, + BlobOp::Link { hash: blob_id.hash.clone(), + to: BlobLink::Temporary { until }, }, 0u32.serialize(), ); diff --git a/crates/jmap/src/blob/download.rs b/crates/jmap/src/blob/download.rs index c8b11c01..84c6630c 100644 --- a/crates/jmap/src/blob/download.rs +++ b/crates/jmap/src/blob/download.rs @@ -7,11 +7,15 @@ use common::{Server, auth::AccessToken}; use email::cache::MessageCacheFetch; use email::cache::email::MessageCacheAccess; +use email::message::metadata::MessageMetadata; +use groupware::cache::GroupwareCache; use std::future::Future; use trc::AddContext; use types::acl::Acl; use types::blob::{BlobClass, BlobId}; -use types::collection::Collection; +use types::collection::{Collection, SyncCollection}; +use types::field::EmailField; +use utils::chained_bytes::ChainedBytes; pub trait BlobDownload: Sync + Send { fn blob_download( @@ -34,64 +38,58 @@ impl BlobDownload for Server { blob_id: &BlobId, access_token: &AccessToken, ) -> trc::Result>> { - if !self - .core - .storage - .data - .blob_has_access(&blob_id.hash, &blob_id.class) - .await - .caused_by(trc::location!())? - { - return Ok(None); - } - - if !access_token.is_member(blob_id.class.account_id()) { - match &blob_id.class { - BlobClass::Linked { - account_id, - collection, - document_id, - } => { - if Collection::from(*collection) == Collection::Email { - if !self - .get_cached_messages(*account_id) - .await - .caused_by(trc::location!())? - .shared_messages(access_token, Acl::ReadItems) - .contains(*document_id) - { - return Ok(None); - } - } else { - match self - .has_access_to_document( - access_token, + if self.has_access_blob(blob_id, access_token).await? { + if let Some(section) = &blob_id.section { + self.get_blob_section(&blob_id.hash, section) + .await + .caused_by(trc::location!()) + } else { + let blob = self + .blob_store() + .get_blob(blob_id.hash.as_slice(), 0..usize::MAX) + .await + .caused_by(trc::location!()); + match (&blob_id.class, blob) { + ( + BlobClass::Linked { + account_id, + collection, + document_id, + }, + Ok(Some(data)), + ) if *collection == Collection::Email as u8 => { + let Some(archive) = self + .archive_by_property( *account_id, - *collection, + Collection::Email, *document_id, - Acl::Read, + EmailField::Metadata.into(), ) .await - { - Ok(has_access) if has_access => (), - _ => return Ok(None), + .caused_by(trc::location!())? + else { + return Ok(Some(data)); + }; + let metadata = archive + .to_unarchived::() + .caused_by(trc::location!())?; + let body_offset = metadata.inner.blob_body_offset.to_native(); + if metadata.inner.root_part().offset_body.to_native() != body_offset { + let raw_message = ChainedBytes::new( + metadata.inner.raw_headers.as_ref(), + ) + .with_last(data.get(body_offset as usize..).unwrap_or_default()); + Ok(Some(raw_message.to_bytes())) + } else { + Ok(Some(data)) } } - } - BlobClass::Reserved { .. } => { - return Ok(None); + (_, blob) => blob, } } - } - - if let Some(section) = &blob_id.section { - self.get_blob_section(&blob_id.hash, section).await } else { - self.blob_store() - .get_blob(blob_id.hash.as_slice(), 0..usize::MAX) - .await + Ok(None) } - .caused_by(trc::location!()) } async fn has_access_blob( @@ -100,9 +98,7 @@ impl BlobDownload for Server { access_token: &AccessToken, ) -> trc::Result { Ok(self - .core - .storage - .data + .store() .blob_has_access(&blob_id.hash, &blob_id.class) .await .caused_by(trc::location!())? @@ -112,26 +108,30 @@ impl BlobDownload for Server { collection, document_id, } => { - if Collection::from(*collection) == Collection::Email { - access_token.is_member(*account_id) - || self + if access_token.is_member(*account_id) { + true + } else { + match Collection::from(*collection) { + Collection::Email => self .get_cached_messages(*account_id) .await .caused_by(trc::location!())? .shared_messages(access_token, Acl::ReadItems) - .contains(*document_id) - } else { - access_token.is_member(*account_id) - || (access_token.has_access(*account_id, *collection) - && self - .has_access_to_document( - access_token, - *account_id, - *collection, - *document_id, - Acl::Read, - ) - .await?) + .contains(*document_id), + collection @ (Collection::FileNode + | Collection::ContactCard + | Collection::CalendarEvent) => self + .fetch_dav_resources( + access_token, + *account_id, + SyncCollection::from(collection), + ) + .await + .caused_by(trc::location!())? + .shared_items(access_token, [Acl::ReadItems], true) + .contains(*document_id), + _ => false, + } } } BlobClass::Reserved { account_id, .. } => access_token.is_member(*account_id), diff --git a/crates/jmap/src/blob/upload.rs b/crates/jmap/src/blob/upload.rs index 98eaa93f..e8f9ff53 100644 --- a/crates/jmap/src/blob/upload.rs +++ b/crates/jmap/src/blob/upload.rs @@ -195,7 +195,7 @@ impl BlobUpload for Server { response.created.insert( create_id, BlobUploadResponseObject { - id: self.put_blob(account_id, &data, true).await?, + id: self.put_jmap_blob(account_id, &data).await?, type_: upload_object.type_, size: data.len(), }, @@ -257,7 +257,7 @@ impl BlobUpload for Server { Ok(UploadResponse { account_id, blob_id: self - .put_blob(account_id.document_id(), data, true) + .put_jmap_blob(account_id.document_id(), data) .await .caused_by(trc::location!())?, c_type: content_type.to_string(), diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index 62b5c653..2ef1916a 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -974,13 +974,25 @@ impl EmailSet for Server { } // Obtain IMAP UIDs for added mailboxes - for uid_mailbox in &mut new_data.mailboxes { - if uid_mailbox.uid == 0 { - uid_mailbox.uid = self - .assign_imap_uid(account_id, uid_mailbox.mailbox_id) - .await - .caused_by(trc::location!())?; - } + let ids = self + .assign_email_ids( + account_id, + new_data + .mailboxes + .iter() + .filter(|m| m.uid == 0) + .map(|m| m.mailbox_id), + false, + ) + .await + .caused_by(trc::location!())?; + for (uid_mailbox, uid) in new_data + .mailboxes + .iter_mut() + .filter(|m| m.uid == 0) + .zip(ids) + { + uid_mailbox.uid = uid; } } diff --git a/crates/jmap/src/sieve/set.rs b/crates/jmap/src/sieve/set.rs index 5295163b..5d787455 100644 --- a/crates/jmap/src/sieve/set.rs +++ b/crates/jmap/src/sieve/set.rs @@ -117,7 +117,9 @@ impl SieveScriptSet for Server { Ok((mut builder, Some(blob))) => { // Store blob let sieve = &mut builder.changes_mut().unwrap(); - sieve.blob_hash = self.put_blob(account_id, &blob, false).await?.hash; + let (blob_hash, blob_hold) = + self.put_temporary_blob(account_id, &blob, 60).await?; + sieve.blob_hash = blob_hash; let blob_size = sieve.size as usize; let blob_hash = sieve.blob_hash.clone(); @@ -133,6 +135,7 @@ impl SieveScriptSet for Server { .with_document(document_id) .custom(builder.with_access_token(ctx.access_token)) .caused_by(trc::location!())? + .clear(blob_hold) .commit_point(); let mut result = Map::with_capacity(1) @@ -222,7 +225,10 @@ impl SieveScriptSet for Server { let blob_id = if let Some(blob) = blob { // Store blob let sieve = &mut builder.changes_mut().unwrap(); - sieve.blob_hash = self.put_blob(account_id, &blob, false).await?.hash; + let (blob_hash, blob_hold) = + self.put_temporary_blob(account_id, &blob, 60).await?; + sieve.blob_hash = blob_hash; + batch.clear(blob_hold); BlobId { hash: sieve.blob_hash.clone(), diff --git a/crates/jmap/src/vacation/set.rs b/crates/jmap/src/vacation/set.rs index 6b2b4738..1dccd288 100644 --- a/crates/jmap/src/vacation/set.rs +++ b/crates/jmap/src/vacation/set.rs @@ -274,14 +274,15 @@ impl VacationResponseSet for Server { // Create sieve script only if there are changes if build_script { // Upload new blob - obj.changes_mut().unwrap().blob_hash = self - .put_blob( + let (blob_hash, blob_hold) = self + .put_temporary_blob( account_id, &self.build_script(obj.changes_mut().unwrap())?, - false, + 60, ) - .await? - .hash; + .await?; + obj.changes_mut().unwrap().blob_hash = blob_hash; + batch.clear(blob_hold); }; batch.custom(obj).caused_by(trc::location!())?; diff --git a/crates/managesieve/src/op/putscript.rs b/crates/managesieve/src/op/putscript.rs index c27b5d0e..e52abf7d 100644 --- a/crates/managesieve/src/op/putscript.rs +++ b/crates/managesieve/src/op/putscript.rs @@ -116,12 +116,10 @@ impl Session { .caused_by(trc::location!())?; // Write script blob - let blob_hash = self + let (blob_hash, blob_hold) = self .server - .put_blob(account_id, &script_bytes, false) - .await - .caused_by(trc::location!())? - .hash; + .put_temporary_blob(account_id, &script_bytes, 60) + .await?; // Write record let mut batch = BatchBuilder::new(); @@ -141,7 +139,8 @@ impl Session { .with_current(script) .with_access_token(access_token), ) - .caused_by(trc::location!())?; + .caused_by(trc::location!())? + .clear(blob_hold); self.server .commit_batch(batch) @@ -158,11 +157,10 @@ impl Session { ); } else { // Write script blob - let blob_hash = self + let (blob_hash, blob_hold) = self .server - .put_blob(account_id, &script_bytes, false) - .await? - .hash; + .put_temporary_blob(account_id, &script_bytes, 60) + .await?; // Write record let mut batch = BatchBuilder::new(); @@ -184,7 +182,8 @@ impl Session { ) .with_access_token(access_token), ) - .caused_by(trc::location!())?; + .caused_by(trc::location!())? + .clear(blob_hold); self.server .commit_batch(batch) diff --git a/crates/migration/Cargo.toml b/crates/migration/Cargo.toml index bb139909..4845aafc 100644 --- a/crates/migration/Cargo.toml +++ b/crates/migration/Cargo.toml @@ -27,7 +27,7 @@ serde_json = "1.0" rkyv = { version = "0.8.10", features = ["little_endian"] } compact_str = "0.9.0" bincode = "1.3.3" -lz4_flex = { version = "0.11", default-features = false } +lz4_flex = { version = "0.12", default-features = false } base64 = "0.22" [features] diff --git a/crates/services/src/task_manager/index.rs b/crates/services/src/task_manager/index.rs index aee2607f..9bd2aa79 100644 --- a/crates/services/src/task_manager/index.rs +++ b/crates/services/src/task_manager/index.rs @@ -7,25 +7,25 @@ use crate::task_manager::{IndexAction, Task}; use common::{ Server, + auth::AccessToken, telemetry::tracers::store::{TracingStore, build_span_document}, }; use directory::{Type, backend::internal::manage::ManageDirectory}; -use email::message::metadata::MessageMetadata; +#[cfg(feature = "enterprise")] +use email::message::metadata::MESSAGE_RECEIVED_MASK; +use email::{cache::MessageCacheFetch, message::metadata::MessageMetadata}; use groupware::{cache::GroupwareCache, calendar::CalendarEvent, contact::ContactCard}; use std::cmp::Ordering; use store::{ - IterateParams, SerializeInfallible, U32_LEN, ValueKey, + SerializeInfallible, ahash::AHashMap, roaring::RoaringBitmap, search::{IndexDocument, SearchField, SearchFilter, SearchQuery}, - write::{ - BatchBuilder, BlobOp, SearchIndex, TaskEpoch, TaskQueueClass, ValueClass, - key::DeserializeBigEndian, - }, + write::{BatchBuilder, SearchIndex, TaskEpoch, TaskQueueClass, ValueClass}, }; use trc::{AddContext, TaskQueueEvent}; use types::{ - blob_hash::{BLOB_HASH_LEN, BlobHash}, + blob_hash::BlobHash, collection::{Collection, SyncCollection}, field::EmailField, }; @@ -305,58 +305,21 @@ impl ReindexIndexTask for Server { match index { SearchIndex::Email => { - // Validate linked blobs - let from_key = ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::Blob(BlobOp::Link { - hash: BlobHash::default(), - }), - }; - let to_key = ValueKey { - account_id: u32::MAX, - collection: u8::MAX, - document_id: u32::MAX, - class: ValueClass::Blob(BlobOp::Link { - hash: BlobHash::new_max(), - }), - }; - let mut document_ids: AHashMap> = AHashMap::new(); - self.core - .storage - .data - .iterate( - IterateParams::new(from_key, to_key).ascending().no_values(), - |key, _| { - let account_id = key.deserialize_be_u32(BLOB_HASH_LEN)?; - let collection = - *key.get(BLOB_HASH_LEN + U32_LEN).ok_or_else(|| { - trc::Error::corrupted_key(key, None, trc::location!()) - })?; - - if accounts.contains(account_id) - && collection == Collection::Email as u8 - { - document_ids - .entry(account_id) - .or_default() - .push(key.deserialize_be_u32(key.len() - U32_LEN)?); - } - - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - - for (account_id, document_ids) in document_ids { + for account_id in accounts { let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Email); - for document_id in document_ids { + for document_id in self + .get_cached_messages(account_id) + .await + .caused_by(trc::location!())? + .emails + .items + .iter() + .map(|v| v.document_id) + { batch.with_document(document_id).set( ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { due, @@ -382,16 +345,18 @@ impl ReindexIndexTask for Server { } SearchIndex::Calendar | SearchIndex::Contacts => { for account_id in accounts { - let Some(cache) = self.cached_dav_resources( - account_id, - if index == SearchIndex::Calendar { - SyncCollection::Calendar - } else { - SyncCollection::AddressBook - }, - ) else { - continue; - }; + let cache = self + .fetch_dav_resources( + &AccessToken::from_id(account_id).with_tenant_id(tenant_id), + account_id, + if index == SearchIndex::Calendar { + SyncCollection::Calendar + } else { + SyncCollection::AddressBook + }, + ) + .await + .caused_by(trc::location!())?; let mut batch = BatchBuilder::new(); batch.with_account_id(account_id); @@ -579,12 +544,77 @@ async fn delete_email_metadata( // Hold blob for undeletion #[cfg(feature = "enterprise")] - server.core.hold_undelete( - batch, - Collection::Email.into(), - &BlobHash::from(&metadata.blob_hash), - metadata.root_part().offset_end.to_native() as usize, - ); + { + use common::enterprise::undelete::DeletedItemType; + use email::message::metadata::ArchivedMetadataHeaderName; + + if let Some(undelete) = server + .core + .enterprise + .as_ref() + .and_then(|e| e.undelete.as_ref()) + { + use common::enterprise::undelete::DeletedItem; + use store::{ + Serialize, + write::{Archiver, BlobLink, BlobOp, now}, + }; + + let root_part = metadata.root_part(); + let from: Option> = root_part.headers.iter().find_map(|h| { + if let ArchivedMetadataHeaderName::From = &h.name { + h.value.as_single_address().and_then(|addr| { + match (addr.address.as_ref(), addr.name.as_ref()) { + (Some(address), Some(name)) => { + Some(format!("{} <{}>", name, address).into_boxed_str()) + } + (Some(address), None) => Some(address.as_ref().into()), + (None, Some(name)) => Some(name.as_ref().into()), + (None, None) => None, + } + }) + } else { + None + } + }); + let subject: Option> = root_part.headers.iter().rev().find_map(|h| { + if let ArchivedMetadataHeaderName::Subject = &h.name { + h.value.as_text().map(Into::into) + } else { + None + } + }); + let now = now(); + let until = now + undelete.retention.as_secs(); + let blob_hash = BlobHash::from(&metadata.blob_hash); + batch + .set( + BlobOp::Link { + hash: blob_hash.clone(), + to: BlobLink::Temporary { until }, + }, + vec![], + ) + .set( + BlobOp::Undelete { + hash: blob_hash, + until, + }, + Archiver::new(DeletedItem { + typ: DeletedItemType::Email { + from: from.unwrap_or_default(), + subject: subject.unwrap_or_default(), + received_at: metadata.rcvd_attach.to_native() + & MESSAGE_RECEIVED_MASK, + }, + size: root_part.offset_end.to_native(), + deleted_at: now, + }) + .serialize() + .caused_by(trc::location!())?, + ); + } + } // SPDX-SnippetEnd } diff --git a/crates/smtp/src/queue/spool.rs b/crates/smtp/src/queue/spool.rs index 1af0e9bb..59b891f1 100644 --- a/crates/smtp/src/queue/spool.rs +++ b/crates/smtp/src/queue/spool.rs @@ -24,10 +24,10 @@ use std::time::SystemTime; use store::write::key::DeserializeBigEndian; use store::write::serialize::rkyv_deserialize; use store::write::{ - AlignedBytes, Archive, Archiver, BatchBuilder, BlobOp, MergeResult, Params, QueueClass, - ValueClass, now, + AlignedBytes, Archive, Archiver, BatchBuilder, BlobLink, BlobOp, MergeResult, Params, + QueueClass, ValueClass, now, }; -use store::{Deserialize, IterateParams, Serialize, SerializeInfallible, U64_LEN, ValueKey}; +use store::{Deserialize, IterateParams, Serialize, U64_LEN, ValueKey}; use trc::{AddContext, ServerEvent}; use types::blob_hash::BlobHash; use utils::DomainPart; @@ -341,11 +341,13 @@ impl MessageWrapper { let mut batch = BatchBuilder::new(); let reserve_until = now() + 120; batch.set( - BlobOp::Reserve { + BlobOp::Link { hash: self.message.blob_hash.clone(), - until: reserve_until, + to: BlobLink::Temporary { + until: reserve_until, + }, }, - 0u32.serialize(), + vec![], ); if let Err(err) = server.store().write(batch.build_all()).await { trc::error!( @@ -424,14 +426,16 @@ impl MessageWrapper { } batch - .clear(BlobOp::Reserve { + .clear(BlobOp::Link { hash: self.message.blob_hash.clone(), - until: reserve_until, + to: BlobLink::Temporary { + until: reserve_until, + }, }) .set( - BlobOp::LinkId { + BlobOp::Link { hash: self.message.blob_hash.clone(), - id: self.queue_id, + to: BlobLink::Id { id: self.queue_id }, }, vec![], ) @@ -658,9 +662,9 @@ impl MessageWrapper { } batch - .clear(BlobOp::LinkId { + .clear(BlobOp::Link { hash: self.message.blob_hash.clone(), - id: self.queue_id, + to: BlobLink::Id { id: self.queue_id }, }) .clear(ValueClass::Queue(QueueClass::Message(self.queue_id))); diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 18d45502..f11ab426 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -32,7 +32,7 @@ parking_lot = "0.12" lru-cache = { version = "0.1.2", optional = true } num_cpus = { version = "1.17", optional = true } blake3 = "1.8" -lz4_flex = { version = "0.11", default-features = false } +lz4_flex = { version = "0.12", default-features = false } deadpool-postgres = { version = "0.14", optional = true } tokio-postgres = { version = "0.7.10", features = ["with-serde_json-1"], optional = true } tokio-rustls = { version = "0.26", optional = true, default-features = false, features = ["ring", "tls12"] } diff --git a/crates/store/src/backend/mysql/main.rs b/crates/store/src/backend/mysql/main.rs index 5e233f8a..a8ad6a34 100644 --- a/crates/store/src/backend/mysql/main.rs +++ b/crates/store/src/backend/mysql/main.rs @@ -110,7 +110,7 @@ impl MysqlStore { SUBSPACE_ACL, SUBSPACE_DIRECTORY, SUBSPACE_TASK_QUEUE, - SUBSPACE_BLOB_RESERVE, + SUBSPACE_BLOB_EXTRA, SUBSPACE_BLOB_LINK, SUBSPACE_IN_MEMORY_VALUE, SUBSPACE_PROPERTY, diff --git a/crates/store/src/backend/postgres/main.rs b/crates/store/src/backend/postgres/main.rs index 282fc06a..c2864ee9 100644 --- a/crates/store/src/backend/postgres/main.rs +++ b/crates/store/src/backend/postgres/main.rs @@ -102,7 +102,7 @@ impl PostgresStore { SUBSPACE_ACL, SUBSPACE_DIRECTORY, SUBSPACE_TASK_QUEUE, - SUBSPACE_BLOB_RESERVE, + SUBSPACE_BLOB_EXTRA, SUBSPACE_BLOB_LINK, SUBSPACE_IN_MEMORY_VALUE, SUBSPACE_PROPERTY, diff --git a/crates/store/src/backend/rocksdb/main.rs b/crates/store/src/backend/rocksdb/main.rs index fe4c4c2f..dba1cdca 100644 --- a/crates/store/src/backend/rocksdb/main.rs +++ b/crates/store/src/backend/rocksdb/main.rs @@ -61,7 +61,7 @@ impl RocksDbStore { SUBSPACE_ACL, SUBSPACE_DIRECTORY, SUBSPACE_TASK_QUEUE, - SUBSPACE_BLOB_RESERVE, + SUBSPACE_BLOB_EXTRA, SUBSPACE_BLOB_LINK, SUBSPACE_IN_MEMORY_VALUE, SUBSPACE_PROPERTY, diff --git a/crates/store/src/backend/sqlite/main.rs b/crates/store/src/backend/sqlite/main.rs index 2365c6ea..612f26fc 100644 --- a/crates/store/src/backend/sqlite/main.rs +++ b/crates/store/src/backend/sqlite/main.rs @@ -92,7 +92,7 @@ impl SqliteStore { SUBSPACE_ACL, SUBSPACE_DIRECTORY, SUBSPACE_TASK_QUEUE, - SUBSPACE_BLOB_RESERVE, + SUBSPACE_BLOB_EXTRA, SUBSPACE_BLOB_LINK, SUBSPACE_IN_MEMORY_VALUE, SUBSPACE_PROPERTY, diff --git a/crates/store/src/dispatch/store.rs b/crates/store/src/dispatch/store.rs index 42d08efd..c543b3f3 100644 --- a/crates/store/src/dispatch/store.rs +++ b/crates/store/src/dispatch/store.rs @@ -6,8 +6,8 @@ use super::DocumentSet; use crate::{ - Deserialize, IterateParams, Key, QueryResult, SUBSPACE_COUNTER, SUBSPACE_INDEXES, - SUBSPACE_LOGS, Store, U32_LEN, Value, ValueKey, + Deserialize, IterateParams, Key, QueryResult, SUBSPACE_BLOB_EXTRA, SUBSPACE_COUNTER, + SUBSPACE_INDEXES, SUBSPACE_LOGS, Store, U32_LEN, Value, ValueKey, write::{ AnyClass, AnyKey, AssignedIds, Batch, BatchBuilder, Operation, ReportClass, ValueClass, ValueOp, @@ -344,7 +344,12 @@ impl Store { } pub async fn danger_destroy_account(&self, account_id: u32) -> trc::Result<()> { - for subspace in [SUBSPACE_LOGS, SUBSPACE_INDEXES, SUBSPACE_COUNTER] { + for subspace in [ + SUBSPACE_LOGS, + SUBSPACE_INDEXES, + SUBSPACE_COUNTER, + SUBSPACE_BLOB_EXTRA, + ] { self.delete_range( AnyKey { subspace, diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index abf1db90..6834226b 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -87,7 +87,7 @@ pub const SUBSPACE_ACL: u8 = b'a'; pub const SUBSPACE_DIRECTORY: u8 = b'd'; pub const SUBSPACE_TASK_QUEUE: u8 = b'f'; pub const SUBSPACE_INDEXES: u8 = b'i'; -pub const SUBSPACE_BLOB_RESERVE: u8 = b'j'; +pub const SUBSPACE_BLOB_EXTRA: u8 = b'j'; pub const SUBSPACE_BLOB_LINK: u8 = b'k'; pub const SUBSPACE_BLOBS: u8 = b't'; pub const SUBSPACE_LOGS: u8 = b'l'; @@ -105,6 +105,13 @@ pub const SUBSPACE_TELEMETRY_SPAN: u8 = b'o'; pub const SUBSPACE_TELEMETRY_METRIC: u8 = b'x'; pub const SUBSPACE_SEARCH_INDEX: u8 = b'z'; +// TODO: Remove in v1.0 +pub const LEGACY_SUBSPACE_BITMAP_ID: u8 = b'b'; +pub const LEGACY_SUBSPACE_BITMAP_TAG: u8 = b'c'; +pub const LEGACY_SUBSPACE_BITMAP_TEXT: u8 = b'v'; +pub const LEGACY_SUBSPACE_FTS_INDEX: u8 = b'g'; +pub const LEGACY_SUBSPACE_TELEMETRY_INDEX: u8 = b'w'; + #[derive(Clone)] pub struct IterateParams { begin: T, diff --git a/crates/store/src/write/blob.rs b/crates/store/src/write/blob.rs index 5cf50bd5..568c6ae9 100644 --- a/crates/store/src/write/blob.rs +++ b/crates/store/src/write/blob.rs @@ -6,14 +6,13 @@ use super::{BlobOp, Operation, ValueClass, ValueOp, key::DeserializeBigEndian, now}; use crate::{ - BlobStore, Deserialize, IterateParams, Store, U32_LEN, U64_LEN, ValueKey, write::BatchBuilder, + BlobStore, IterateParams, Store, U32_LEN, U64_LEN, ValueKey, + write::{BatchBuilder, BlobLink}, }; -use ahash::AHashSet; use trc::AddContext; use types::{ blob::BlobClass, blob_hash::{BLOB_HASH_LEN, BlobHash}, - collection::Collection, }; #[derive(Debug, PartialEq, Eq)] @@ -42,7 +41,7 @@ impl Store { account_id, collection: 0, document_id: 0, - class: ValueClass::Blob(BlobOp::Reserve { + class: ValueClass::Blob(BlobOp::Quota { hash: BlobHash::default(), until: 0, }), @@ -51,9 +50,9 @@ impl Store { account_id: account_id + 1, collection: 0, document_id: 0, - class: ValueClass::Blob(BlobOp::Reserve { + class: ValueClass::Blob(BlobOp::Quota { hash: BlobHash::default(), - until: 0, + until: u64::MAX, }), }; @@ -64,8 +63,8 @@ impl Store { IterateParams::new(from_key, to_key).ascending(), |key, value| { let until = key.deserialize_be_u64(key.len() - U64_LEN)?; - if until > now && value.len() == U32_LEN { - let bytes = u32::deserialize(value)?; + if until > now { + let bytes = value.deserialize_be_u32(0)?; if bytes > 0 { quota.bytes += bytes as usize; quota.count += 1; @@ -93,9 +92,9 @@ impl Store { account_id: *account_id, collection: 0, document_id: 0, - class: ValueClass::Blob(BlobOp::Reserve { + class: ValueClass::Blob(BlobOp::Link { hash: hash.as_ref().clone(), - until: *expires, + to: BlobLink::Temporary { until: *expires }, }), }, BlobClass::Linked { @@ -108,6 +107,7 @@ impl Store { document_id: *document_id, class: ValueClass::Blob(BlobOp::Link { hash: hash.as_ref().clone(), + to: BlobLink::Document, }), }, _ => return Ok(false), @@ -117,54 +117,12 @@ impl Store { } pub async fn purge_blobs(&self, blob_store: BlobStore) -> trc::Result<()> { - // Remove expired temporary blobs - let from_key = ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::Blob(BlobOp::Reserve { - until: 0, - hash: BlobHash::default(), - }), - }; - let to_key = ValueKey { - account_id: u32::MAX, - collection: 0, - document_id: 0, - class: ValueClass::Blob(BlobOp::Reserve { - until: 0, - hash: BlobHash::default(), - }), - }; - let mut delete_keys = Vec::new(); - let mut active_hashes = AHashSet::new(); - let now = now(); - self.iterate( - IterateParams::new(from_key, to_key).ascending().no_values(), - |key, _| { - let hash = BlobHash::try_from_hash_slice( - key.get(U32_LEN..U32_LEN + BLOB_HASH_LEN) - .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?, - ) - .unwrap(); - let until = key.deserialize_be_u64(key.len() - U64_LEN)?; - if until <= now { - delete_keys.push((key.deserialize_be_u32(0)?, BlobOp::Reserve { until, hash })); - } else { - active_hashes.insert(hash); - } - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - // Validate linked blobs let from_key = ValueKey { account_id: 0, collection: 0, document_id: 0, - class: ValueClass::Blob(BlobOp::Link { + class: ValueClass::Blob(BlobOp::Commit { hash: BlobHash::default(), }), }; @@ -174,9 +132,16 @@ impl Store { document_id: u32::MAX, class: ValueClass::Blob(BlobOp::Link { hash: BlobHash::new_max(), + to: BlobLink::Document, }), }; + const TEMP_LINK: usize = BLOB_HASH_LEN + U32_LEN + U64_LEN; + const DOC_LINK: usize = BLOB_HASH_LEN + U64_LEN + 1; + let mut last_hash = BlobHash::default(); + let mut last_hash_is_linked = true; // Avoid deleting non-existing last_hash on first iteration + let mut delete_keys = Vec::new(); + let now = now(); self.iterate( IterateParams::new(from_key, to_key).ascending().no_values(), |key, _| { @@ -185,15 +150,64 @@ impl Store { .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?, ) .unwrap(); - let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; - if document_id != u32::MAX { - if last_hash != hash { + if last_hash != hash { + if !last_hash_is_linked { + delete_keys.push(( + None, + BlobOp::Commit { + hash: std::mem::replace(&mut last_hash, hash), + }, + )); + } else { last_hash = hash; } - } else if last_hash != hash && !active_hashes.contains(&hash) { - // Unlinked or expired blob, delete. - delete_keys.push((0, BlobOp::Commit { hash })); + last_hash_is_linked = false; + } + + match key.len() { + BLOB_HASH_LEN => { + // Main blob entry + } + TEMP_LINK => { + // Temporary link + let until = key.deserialize_be_u64(BLOB_HASH_LEN + U32_LEN)?; + if until <= now { + let account_id = key.deserialize_be_u32(BLOB_HASH_LEN)?; + delete_keys.push(( + Some(account_id), + BlobOp::Link { + hash: last_hash.clone(), + to: BlobLink::Temporary { until }, + }, + )); + if account_id != u32::MAX { + delete_keys.push(( + Some(account_id), + BlobOp::Quota { + hash: last_hash.clone(), + until, + }, + )); + delete_keys.push(( + Some(account_id), + BlobOp::Undelete { + hash: last_hash.clone(), + until, + }, + )); + } + } else { + last_hash_is_linked = true; + } + } + DOC_LINK => { + // Document link + last_hash_is_linked = true; + } + _ => { + return Err(trc::Error::corrupted_key(key, None, trc::location!())); + } } Ok(true) @@ -202,6 +216,10 @@ impl Store { .await .caused_by(trc::location!())?; + if !last_hash_is_linked { + delete_keys.push((None, BlobOp::Commit { hash: last_hash })); + } + // Delete expired or unlinked blobs for (_, op) in &delete_keys { if let BlobOp::Commit { hash } = op { @@ -214,96 +232,18 @@ impl Store { // Delete hashes let mut batch = BatchBuilder::new(); - let mut last_account_id = u32::MAX; - for (account_id, op) in delete_keys.into_iter() { - if batch.is_large_batch() { - last_account_id = u32::MAX; - self.write(batch.build_all()) - .await - .caused_by(trc::location!())?; - batch = BatchBuilder::new(); - } - if matches!(op, BlobOp::Reserve { .. }) && account_id != last_account_id { - batch.with_account_id(account_id); - last_account_id = account_id; - } - batch.any_op(Operation::Value { - class: ValueClass::Blob(op), - op: ValueOp::Clear, - }); - } - if !batch.is_empty() { - self.write(batch.build_all()) - .await - .caused_by(trc::location!())?; - } - - Ok(()) - } - - pub async fn blob_hash_unlink_account(&self, account_id: u32) -> trc::Result<()> { - // Validate linked blobs - let from_key = ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::Blob(BlobOp::Link { - hash: BlobHash::default(), - }), - }; - let to_key = ValueKey { - account_id: u32::MAX, - collection: u8::MAX, - document_id: u32::MAX, - class: ValueClass::Blob(BlobOp::Link { - hash: BlobHash::new_max(), - }), - }; - let mut delete_keys = Vec::new(); - self.iterate( - IterateParams::new(from_key, to_key).ascending().no_values(), - |key, _| { - let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; - - if document_id != u32::MAX && key.deserialize_be_u32(BLOB_HASH_LEN)? == account_id { - delete_keys.push(( - Collection::from(key[BLOB_HASH_LEN + U32_LEN]), - document_id, - BlobOp::Link { - hash: BlobHash::try_from_hash_slice( - key.get(0..BLOB_HASH_LEN).ok_or_else(|| { - trc::Error::corrupted_key(key, None, trc::location!()) - })?, - ) - .unwrap(), - }, - )); - } - - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - - // Unlink blobs - let mut batch = BatchBuilder::new(); - batch.with_account_id(account_id); - let mut last_collection = Collection::None; - for (collection, document_id, op) in delete_keys.into_iter() { + for (account_id, op) in delete_keys { if batch.is_large_batch() { self.write(batch.build_all()) .await .caused_by(trc::location!())?; batch = BatchBuilder::new(); + } + + if let Some(account_id) = account_id { batch.with_account_id(account_id); - last_collection = Collection::None; } - if collection != last_collection { - batch.with_collection(collection); - last_collection = collection; - } - batch.with_document(document_id); + batch.any_op(Operation::Value { class: ValueClass::Blob(op), op: ValueOp::Clear, diff --git a/crates/store/src/write/key.rs b/crates/store/src/write/key.rs index 1e2657ca..496dd371 100644 --- a/crates/store/src/write/key.rs +++ b/crates/store/src/write/key.rs @@ -9,14 +9,14 @@ use super::{ TaskQueueClass, TelemetryClass, ValueClass, }; use crate::{ - Deserialize, IndexKey, IndexKeyPrefix, Key, LogKey, SUBSPACE_ACL, SUBSPACE_BLOB_LINK, - SUBSPACE_BLOB_RESERVE, SUBSPACE_COUNTER, SUBSPACE_DIRECTORY, SUBSPACE_IN_MEMORY_COUNTER, + Deserialize, IndexKey, IndexKeyPrefix, Key, LogKey, SUBSPACE_ACL, SUBSPACE_BLOB_EXTRA, + SUBSPACE_BLOB_LINK, SUBSPACE_COUNTER, SUBSPACE_DIRECTORY, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_IN_MEMORY_VALUE, SUBSPACE_INDEXES, SUBSPACE_LOGS, SUBSPACE_PROPERTY, SUBSPACE_QUEUE_EVENT, SUBSPACE_QUEUE_MESSAGE, SUBSPACE_QUOTA, SUBSPACE_REPORT_IN, SUBSPACE_REPORT_OUT, SUBSPACE_SEARCH_INDEX, SUBSPACE_SETTINGS, SUBSPACE_TASK_QUEUE, SUBSPACE_TELEMETRY_METRIC, SUBSPACE_TELEMETRY_SPAN, U16_LEN, U32_LEN, U64_LEN, ValueKey, WITH_SUBSPACE, - write::{IndexPropertyClass, SearchIndex, SearchIndexId, SearchIndexType}, + write::{BlobLink, IndexPropertyClass, SearchIndex, SearchIndexId, SearchIndexType}, }; use std::convert::TryInto; use types::{blob_hash::BLOB_HASH_LEN, collection::SyncCollection}; @@ -329,25 +329,29 @@ impl ValueClass { .write(document_id), }, ValueClass::Blob(op) => match op { - BlobOp::Reserve { hash, until } => serializer + BlobOp::Commit { hash } => serializer.write::<&[u8]>(hash.as_ref()), + BlobOp::Link { hash, to } => match to { + BlobLink::Id { id } => serializer.write::<&[u8]>(hash.as_ref()).write(*id), + BlobLink::Document => serializer + .write::<&[u8]>(hash.as_ref()) + .write(account_id) + .write(collection) + .write(document_id), + BlobLink::Temporary { until } => serializer + .write::<&[u8]>(hash.as_ref()) + .write(account_id) + .write(*until), + }, + BlobOp::Quota { hash, until } => serializer .write(account_id) + .write(0u8) .write::<&[u8]>(hash.as_ref()) .write(*until), - BlobOp::Commit { hash } => serializer - .write::<&[u8]>(hash.as_ref()) - .write(u32::MAX) - .write(0u8) - .write(u32::MAX), - BlobOp::Link { hash } => serializer - .write::<&[u8]>(hash.as_ref()) + BlobOp::Undelete { hash, until } => serializer .write(account_id) - .write(collection) - .write(document_id), - BlobOp::LinkId { hash, id } => serializer + .write(1u8) .write::<&[u8]>(hash.as_ref()) - .write((*id >> 32) as u32) - .write(u8::MAX) - .write(*id as u32), + .write(*until), }, ValueClass::Config(key) => serializer.write(key.as_slice()), ValueClass::InMemory(lookup) => match lookup { @@ -561,9 +565,17 @@ impl ValueClass { DirectoryClass::Index { word, .. } => word.len() + U32_LEN, }, ValueClass::Blob(op) => match op { - BlobOp::Reserve { .. } => BLOB_HASH_LEN + U64_LEN + U32_LEN + 1, - BlobOp::Commit { .. } | BlobOp::Link { .. } | BlobOp::LinkId { .. } => { - BLOB_HASH_LEN + U32_LEN * 2 + 2 + BlobOp::Commit { .. } => BLOB_HASH_LEN, + BlobOp::Link { to, .. } => { + BLOB_HASH_LEN + + match to { + BlobLink::Id { .. } => U64_LEN, + BlobLink::Document => U32_LEN * 2 + 1, + BlobLink::Temporary { .. } => U32_LEN + U64_LEN, + } + } + BlobOp::Quota { .. } | BlobOp::Undelete { .. } => { + BLOB_HASH_LEN + U32_LEN + U64_LEN + 1 } }, ValueClass::TaskQueue(e) => match e { @@ -624,10 +636,8 @@ impl ValueClass { ValueClass::Acl(_) => SUBSPACE_ACL, ValueClass::TaskQueue { .. } => SUBSPACE_TASK_QUEUE, ValueClass::Blob(op) => match op { - BlobOp::Reserve { .. } => SUBSPACE_BLOB_RESERVE, - BlobOp::Commit { .. } | BlobOp::Link { .. } | BlobOp::LinkId { .. } => { - SUBSPACE_BLOB_LINK - } + BlobOp::Commit { .. } | BlobOp::Link { .. } => SUBSPACE_BLOB_LINK, + BlobOp::Quota { .. } | BlobOp::Undelete { .. } => SUBSPACE_BLOB_EXTRA, }, ValueClass::Config(_) => SUBSPACE_SETTINGS, ValueClass::InMemory(lookup) => match lookup { diff --git a/crates/store/src/write/mod.rs b/crates/store/src/write/mod.rs index ba55930c..ad2ff534 100644 --- a/crates/store/src/write/mod.rs +++ b/crates/store/src/write/mod.rs @@ -375,10 +375,17 @@ pub struct SetOperation { #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub enum BlobOp { - Reserve { hash: BlobHash, until: u64 }, Commit { hash: BlobHash }, - Link { hash: BlobHash }, - LinkId { hash: BlobHash, id: u64 }, + Link { hash: BlobHash, to: BlobLink }, + Quota { hash: BlobHash, until: u64 }, + Undelete { hash: BlobHash, until: u64 }, +} + +#[derive(Debug, PartialEq, Clone, Eq, Hash)] +pub enum BlobLink { + Id { id: u64 }, + Document, + Temporary { until: u64 }, } #[derive(Debug, PartialEq, Clone, Eq, Hash)] diff --git a/crates/types/src/field.rs b/crates/types/src/field.rs index 2786815f..3d22eb11 100644 --- a/crates/types/src/field.rs +++ b/crates/types/src/field.rs @@ -41,6 +41,7 @@ pub enum EmailField { Archive, Metadata, Threading, + DeletedAt, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -118,6 +119,7 @@ impl From for u8 { match value { EmailField::Metadata => 71, EmailField::Threading => 90, + EmailField::DeletedAt => 91, EmailField::Archive => ARCHIVE_FIELD, } } diff --git a/crates/types/src/keyword.rs b/crates/types/src/keyword.rs index d37855e8..16915496 100644 --- a/crates/types/src/keyword.rs +++ b/crates/types/src/keyword.rs @@ -38,6 +38,7 @@ pub const OTHER: usize = 12; #[serde(untagged)] #[rkyv(derive(PartialEq), compare(PartialEq))] pub enum Keyword { + Other(Box), #[serde(rename(serialize = "$seen"))] Seen, #[serde(rename(serialize = "$draft"))] @@ -63,7 +64,6 @@ pub enum Keyword { Forwarded, #[serde(rename(serialize = "$mdnsent"))] MdnSent, - Other(Box), } impl Keyword { diff --git a/tests/src/directory/internal.rs b/tests/src/directory/internal.rs index 7395562b..b60d092b 100644 --- a/tests/src/directory/internal.rs +++ b/tests/src/directory/internal.rs @@ -4,11 +4,14 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use std::sync::Arc; + use crate::{ directory::{DirectoryTest, IntoTestPrincipal, TestPrincipal}, - store::cleanup::store_destroy, + store::cleanup::{store_assert_is_empty, store_destroy}, }; use ahash::AHashSet; +use common::{Core, Inner, Server, config::storage::Storage}; use directory::{ Permission, QueryBy, QueryParams, Type, backend::{ @@ -20,9 +23,10 @@ use directory::{ }, }, }; +use http::management::stores::destroy_account_data; use mail_send::Credentials; use store::{ - IndexKeyPrefix, IterateParams, Store, ValueKey, + IterateParams, Store, ValueKey, write::{BatchBuilder, ValueClass}, }; use types::collection::Collection; @@ -678,7 +682,20 @@ async fn internal_directory() { } // Delete John's account and make sure his records are gone + let server = Server { + inner: Arc::new(Inner::default()), + core: Arc::new(Core { + storage: Storage { + data: store.clone(), + blob: store.clone().into(), + fts: store.clone().into(), + ..Default::default() + }, + ..Default::default() + }), + }; store.delete_principal(QueryBy::Id(john_id)).await.unwrap(); + destroy_account_data(&server, john_id, true).await.unwrap(); assert_eq!(store.get_principal_id("john.doe").await.unwrap(), None); assert_eq!( store.email_to_id("john.doe@example.org").await.unwrap(), @@ -746,6 +763,16 @@ async fn internal_directory() { .unwrap(), Some("hello".into()) ); + + // Clean up + destroy_account_data(&server, jane_id, true).await.unwrap(); + for principal_name in ["jane", "list", "sales", "support", "example.org"] { + store + .delete_principal(QueryBy::Name(principal_name)) + .await + .unwrap(); + } + store_assert_is_empty(&store, store.clone().into(), true).await; } } @@ -1011,15 +1038,17 @@ async fn account_has_emails(store: &Store, account_id: u32) -> bool { store .iterate( IterateParams::new( - IndexKeyPrefix { + ValueKey { account_id, collection: Collection::Email.into(), - field: 0, + document_id: 0, + class: ValueClass::Property(0), }, - IndexKeyPrefix { + ValueKey { account_id, collection: Collection::Email.into(), - field: u8::MAX, + document_id: u32::MAX, + class: ValueClass::Property(u8::MAX), }, ) .no_values(), diff --git a/tests/src/jmap/auth/permissions.rs b/tests/src/jmap/auth/permissions.rs index 424b5a50..a390fd18 100644 --- a/tests/src/jmap/auth/permissions.rs +++ b/tests/src/jmap/auth/permissions.rs @@ -610,11 +610,10 @@ pub async fn test(params: &JMAPTest) { ); // John should not be allowed to receive email - let message_blob = server - .put_blob(tenant_user_id, TEST_MESSAGE.as_bytes(), false) + let (message_blob, _) = server + .put_temporary_blob(tenant_user_id, TEST_MESSAGE.as_bytes(), 60) .await - .unwrap() - .hash; + .unwrap(); assert_eq!( server .deliver_message(IngestMessage { diff --git a/tests/src/jmap/mail/delivery.rs b/tests/src/jmap/mail/delivery.rs index 75feb864..0ec756cd 100644 --- a/tests/src/jmap/mail/delivery.rs +++ b/tests/src/jmap/mail/delivery.rs @@ -15,7 +15,7 @@ use email::{ }; use groupware::DavResourceName; use jmap::blob::download::BlobDownload; -use std::time::Duration; +use std::{sync::Arc, time::Duration}; use tokio::{ io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines, ReadHalf, WriteHalf}, net::TcpStream, @@ -31,7 +31,11 @@ use utils::chained_bytes::ChainedBytes; pub async fn test(params: &mut JMAPTest) { println!("Running message delivery tests..."); - let todo = "enable delivered to for test"; + // Enable delivered to + let old_core = params.server.core.clone(); + let mut new_core = old_core.as_ref().clone(); + new_core.smtp.session.data.add_delivered_to = true; + params.server.inner.shared_core.store(Arc::new(new_core)); // Create a domain name and a test account let server = params.server.clone(); @@ -277,37 +281,48 @@ END:VCARD .unwrap() .unwrap(); let metadata = archive.to_unarchived::().unwrap(); - let body = server - .blob_download( - &BlobId { - hash: BlobHash::from(&metadata.inner.blob_hash), - class: BlobClass::Linked { - account_id, - collection: Collection::Email.into(), - document_id, - }, - section: None, - }, - &access_token, - ) + let partial_message = server + .store() + .get_blob(metadata.inner.blob_hash.0.as_ref(), 0..usize::MAX) .await .unwrap() .unwrap(); assert_ne!(metadata.inner.blob_body_offset.to_native(), 0); - let raw_message = ChainedBytes::new(metadata.inner.raw_headers.as_ref()).with_last( - body.get(metadata.inner.blob_body_offset.to_native() as usize..) - .unwrap_or_default(), - ); - let full_message = String::from_utf8(raw_message.to_bytes()).unwrap(); + let expected_full_message = String::from_utf8( + ChainedBytes::new(metadata.inner.raw_headers.as_ref()) + .with_last( + partial_message + .get(metadata.inner.blob_body_offset.to_native() as usize..) + .unwrap_or_default(), + ) + .to_bytes(), + ) + .unwrap(); assert!( - full_message.contains("Delivered-To:") && full_message.contains("Subject:"), - "for {account_id}: {full_message}" - ); - println!( - "full message for {}:\n{}", - account.id_string(), - full_message + expected_full_message.contains("Delivered-To:") + && expected_full_message.contains("Subject:"), + "for {account_id}: {expected_full_message}" ); + let full_message = String::from_utf8( + server + .blob_download( + &BlobId { + hash: BlobHash::from(&metadata.inner.blob_hash), + class: BlobClass::Linked { + account_id, + collection: Collection::Email.into(), + document_id, + }, + section: None, + }, + &access_token, + ) + .await + .unwrap() + .unwrap(), + ) + .unwrap(); + assert_eq!(full_message, expected_full_message, "for {account_id}"); } } @@ -317,6 +332,9 @@ END:VCARD } params.assert_is_empty().await; + // Restore core + params.server.inner.shared_core.store(old_core); + // Check webhook events params.webhook.assert_contains(&[ "message-ingest.", diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 2a467376..449e1d57 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -127,6 +127,8 @@ async fn jmap_tests() { server::purge::test(&mut params).await; server::enterprise::test(&mut params).await; + assert_is_empty(¶ms.server).await; + if delete { params.temp_dir.delete(); } @@ -266,7 +268,7 @@ pub async fn assert_is_empty(server: &Server) { .unwrap(); // Assert is empty - store_assert_is_empty(server.store(), server.core.storage.blob.clone()).await; + store_assert_is_empty(server.store(), server.core.storage.blob.clone(), false).await; search_store_destroy(server.search_store()).await; // Clean caches diff --git a/tests/src/jmap/server/enterprise.rs b/tests/src/jmap/server/enterprise.rs index 49266dff..999736a2 100644 --- a/tests/src/jmap/server/enterprise.rs +++ b/tests/src/jmap/server/enterprise.rs @@ -25,14 +25,16 @@ use common::{ core::BuildServer, enterprise::{ Enterprise, MetricStore, TraceStore, Undelete, config::parse_metric_alerts, - license::LicenseKey, undelete::DeletedBlob, + license::LicenseKey, }, telemetry::{ metrics::store::{Metric, MetricsStore, SharedMetricHistory}, tracers::store::TracingStore, }, }; -use http::management::enterprise::undelete::{UndeleteRequest, UndeleteResponse}; +use http::management::enterprise::undelete::{ + DeletedBlobResponse, DeletedItemResponse, UndeleteRequest, UndeleteResponse, +}; use imap_proto::ResponseType; use nlp::language::Language; use std::{sync::Arc, time::Duration}; @@ -435,13 +437,22 @@ async fn undelete(params: &mut JMAPTest) { wait_for_index(¶ms.server).await; tokio::time::sleep(Duration::from_millis(200)).await; let deleted = api - .get::>>("/api/store/undelete/jdoe@example.com") + .get::>("/api/store/undelete/jdoe@example.com") .await .unwrap() .unwrap_data() .items; assert_eq!(deleted.len(), 1); let deleted = deleted.into_iter().next().unwrap(); + match deleted.item { + DeletedItemResponse::Email { from, subject, .. } => { + assert_eq!(subject.as_ref(), "undelete test"); + assert_eq!(from.as_ref(), "john@example.com"); + } + other => { + panic!("Unexpected deleted item response: {:?}", other); + } + } // Undelete let result = api @@ -449,7 +460,7 @@ async fn undelete(params: &mut JMAPTest) { "/api/store/undelete/jdoe@example.com", &vec![UndeleteRequest { hash: deleted.hash, - collection: deleted.collection, + collection: "email".to_string(), time: deleted.deleted_at, cancel_deletion: deleted.expires_at.into(), }], diff --git a/tests/src/jmap/server/purge.rs b/tests/src/jmap/server/purge.rs index 65f2e7e6..461f5103 100644 --- a/tests/src/jmap/server/purge.rs +++ b/tests/src/jmap/server/purge.rs @@ -16,12 +16,9 @@ use email::{ mailbox::{INBOX_ID, JUNK_ID, TRASH_ID}, message::delete::EmailDeletion, }; +use http::management::stores::destroy_account_data; use imap_proto::ResponseType; -use store::{ - IterateParams, LogKey, U32_LEN, U64_LEN, - search::SearchQuery, - write::{SearchIndex, key::DeserializeBigEndian}, -}; +use store::{IterateParams, LogKey, U32_LEN, U64_LEN, write::key::DeserializeBigEndian}; use types::id::Id; pub async fn test(params: &mut JMAPTest) { @@ -153,25 +150,13 @@ pub async fn test(params: &mut JMAPTest) { // Delete account server - .core - .storage - .data + .store() .delete_principal(QueryBy::Id(account.id().document_id())) .await .unwrap(); - for index in [ - SearchIndex::Email, - SearchIndex::Contacts, - SearchIndex::Calendar, - ] { - server - .core - .storage - .fts - .unindex(SearchQuery::new(index).with_account_id(account.id().document_id())) - .await - .unwrap(); - } + destroy_account_data(&server, account.id().document_id(), true) + .await + .unwrap(); params.assert_is_empty().await; } diff --git a/tests/src/smtp/inbound/data.rs b/tests/src/smtp/inbound/data.rs index b42fb970..aaf78f75 100644 --- a/tests/src/smtp/inbound/data.rs +++ b/tests/src/smtp/inbound/data.rs @@ -234,5 +234,5 @@ async fn data() { // Make sure store is empty qr.clear_queue(&test.server).await; - store_assert_is_empty(test.server.store(), test.server.blob_store().clone()).await; + store_assert_is_empty(test.server.store(), test.server.blob_store().clone(), false).await; } diff --git a/tests/src/smtp/queue/concurrent.rs b/tests/src/smtp/queue/concurrent.rs index e22ec4e2..701d54f4 100644 --- a/tests/src/smtp/queue/concurrent.rs +++ b/tests/src/smtp/queue/concurrent.rs @@ -152,5 +152,10 @@ async fn concurrent_queue() { assert_eq!(remote_messages.len(), NUM_MESSAGES); // Make sure local store is queue - store_assert_is_empty(&core.core.storage.data, core.core.storage.blob.clone()).await; + store_assert_is_empty( + &core.core.storage.data, + core.core.storage.blob.clone(), + false, + ) + .await; } diff --git a/tests/src/smtp/queue/virtualq.rs b/tests/src/smtp/queue/virtualq.rs index 5c1be93a..c3ca66cf 100644 --- a/tests/src/smtp/queue/virtualq.rs +++ b/tests/src/smtp/queue/virtualq.rs @@ -208,5 +208,10 @@ async fn virtual_queue() { assert_eq!(remote_messages.len(), NUM_MESSAGES * 2); // Make sure local store is queue - store_assert_is_empty(&core.core.storage.data, core.core.storage.blob.clone()).await; + store_assert_is_empty( + &core.core.storage.data, + core.core.storage.blob.clone(), + false, + ) + .await; } diff --git a/tests/src/store/blob.rs b/tests/src/store/blob.rs index 5c123669..eac90d3e 100644 --- a/tests/src/store/blob.rs +++ b/tests/src/store/blob.rs @@ -4,15 +4,18 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use ahash::AHashMap; -use store::{ - BlobStore, SerializeInfallible, Stores, - write::{BatchBuilder, BlobOp, blob::BlobQuota, now}, -}; -use types::{blob::BlobClass, blob_hash::BlobHash, collection::Collection}; -use utils::config::Config; - use crate::store::{CONFIG, TempDir, cleanup::store_destroy}; +use ahash::AHashMap; +use common::{Core, Inner, Server, config::storage::Storage}; +use email::message::metadata::MessageMetadata; +use http::management::stores::destroy_account_blobs; +use std::sync::Arc; +use store::{ + BlobStore, Serialize, SerializeInfallible, Stores, + write::{Archiver, BatchBuilder, BlobLink, BlobOp, ValueClass, blob::BlobQuota, now}, +}; +use types::{blob::BlobClass, blob_hash::BlobHash, collection::Collection, field::EmailField}; +use utils::config::Config; #[tokio::test] pub async fn blob_tests() { @@ -34,6 +37,17 @@ pub async fn blob_tests() { // Test internal blob store let blob_store: BlobStore = store.clone().into(); + let server = Server { + inner: Arc::new(Inner::default()), + core: Arc::new(Core { + storage: Storage { + data: store.clone(), + blob: blob_store.clone(), + ..Default::default() + }, + ..Default::default() + }), + }; // Blob hash exists let hash = BlobHash::generate(b"abc".as_slice()); @@ -46,8 +60,8 @@ pub async fn blob_tests() { BatchBuilder::new() .with_account_id(0) .set( - BlobOp::Reserve { - until, + BlobOp::Link { + to: BlobLink::Temporary { until }, hash: hash.clone(), }, 1024u32.serialize(), @@ -165,26 +179,54 @@ pub async fn blob_tests() { .enumerate() { let hash = BlobHash::generate(blob.as_slice()); - let blob_op = if let Some(until) = expiry_times.get(blob) { - BlobOp::Reserve { - until: *until, - hash: hash.clone(), + let mut batch = BatchBuilder::new(); + batch + .with_account_id(if document_id > 0 { 0 } else { 1 }) + .with_collection(Collection::Email) + .with_document(document_id as u32); + if let Some(until) = expiry_times.get(blob) { + if !blob_value.is_empty() { + batch.set( + BlobOp::Quota { + hash: hash.clone(), + until: *until, + }, + blob_value, + ); } + batch.set( + BlobOp::Link { + hash: hash.clone(), + to: BlobLink::Temporary { until: *until }, + }, + vec![], + ); } else { - BlobOp::Link { hash: hash.clone() } + batch + .set( + BlobOp::Link { + hash: hash.clone(), + to: BlobLink::Document, + }, + vec![], + ) + .set( + ValueClass::Property(EmailField::Metadata.into()), + Archiver::new(MessageMetadata { + contents: Default::default(), + rcvd_attach: Default::default(), + blob_hash: hash.clone(), + blob_body_offset: Default::default(), + preview: Default::default(), + raw_headers: Default::default(), + }) + .serialize() + .unwrap(), + ); }; - store - .write( - BatchBuilder::new() - .with_account_id(if document_id > 0 { 0 } else { 1 }) - .with_collection(Collection::Email) - .with_document(document_id as u32) - .set(blob_op, blob_value) - .set(BlobOp::Commit { hash: hash.clone() }, vec![]) - .build_all(), - ) - .await - .unwrap(); + batch.set(BlobOp::Commit { hash: hash.clone() }, vec![]); + + store.write(batch.build_all()).await.unwrap(); blob_store .put_blob(hash.as_ref(), blob.as_slice()) .await @@ -294,6 +336,7 @@ pub async fn blob_tests() { .with_document(2) .clear(BlobOp::Link { hash: BlobHash::generate(b"789".as_slice()), + to: BlobLink::Document, }) .build_all(), ) @@ -360,7 +403,7 @@ pub async fn blob_tests() { } // Unlink all blobs from accountId 1 and purge - store.blob_hash_unlink_account(1).await.unwrap(); + destroy_account_blobs(&server, 1).await.unwrap(); store.purge_blobs(blob_store.clone()).await.unwrap(); // Make sure only accountId 0's blobs are left diff --git a/tests/src/store/cleanup.rs b/tests/src/store/cleanup.rs index 70fd667d..d3025c7e 100644 --- a/tests/src/store/cleanup.rs +++ b/tests/src/store/cleanup.rs @@ -10,6 +10,7 @@ use store::{ *, }; use trc::AddContext; +use types::blob_hash::{BLOB_HASH_LEN, BlobHash}; pub async fn store_destroy(store: &Store) { store_destroy_sql_indexes(store).await; @@ -19,7 +20,7 @@ pub async fn store_destroy(store: &Store) { SUBSPACE_DIRECTORY, SUBSPACE_TASK_QUEUE, SUBSPACE_INDEXES, - SUBSPACE_BLOB_RESERVE, + SUBSPACE_BLOB_EXTRA, SUBSPACE_BLOB_LINK, SUBSPACE_LOGS, SUBSPACE_IN_MEMORY_COUNTER, @@ -102,18 +103,17 @@ pub async fn store_blob_expire_all(store: &Store) { account_id: 0, collection: 0, document_id: 0, - class: ValueClass::Blob(BlobOp::Reserve { - hash: types::blob_hash::BlobHash::default(), - until: 0, + class: ValueClass::Blob(BlobOp::Commit { + hash: BlobHash::default(), }), }; let to_key = ValueKey { account_id: u32::MAX, - collection: 0, - document_id: 0, - class: ValueClass::Blob(BlobOp::Reserve { - hash: types::blob_hash::BlobHash::default(), - until: 0, + collection: u8::MAX, + document_id: u32::MAX, + class: ValueClass::Blob(BlobOp::Link { + hash: BlobHash::new_max(), + to: BlobLink::Document, }), }; let mut batch = BatchBuilder::new(); @@ -122,25 +122,31 @@ pub async fn store_blob_expire_all(store: &Store) { .iterate( IterateParams::new(from_key, to_key).ascending().no_values(), |key, _| { - let account_id = key.deserialize_be_u32(0).caused_by(trc::location!())?; - if account_id != last_account_id { - last_account_id = account_id; - batch.with_account_id(account_id); - } + if key.len() == BLOB_HASH_LEN + U32_LEN + U64_LEN { + let account_id = key + .deserialize_be_u32(BLOB_HASH_LEN) + .caused_by(trc::location!())?; + if account_id != last_account_id { + last_account_id = account_id; + batch.with_account_id(account_id); + } + let hash = + BlobHash::try_from_hash_slice(key.get(..BLOB_HASH_LEN).unwrap()).unwrap(); + let until = key + .deserialize_be_u64(BLOB_HASH_LEN + U32_LEN) + .caused_by(trc::location!())?; - batch.any_op(Operation::Value { - class: ValueClass::Blob(BlobOp::Reserve { - hash: types::blob_hash::BlobHash::try_from_hash_slice( - key.get(U32_LEN..U32_LEN + types::blob_hash::BLOB_HASH_LEN) - .unwrap(), - ) - .unwrap(), - until: key - .deserialize_be_u64(key.len() - U64_LEN) - .caused_by(trc::location!())?, - }), - op: ValueOp::Clear, - }); + batch + .clear(ValueClass::Blob(BlobOp::Link { + hash: hash.clone(), + to: BlobLink::Temporary { until }, + })) + .clear(ValueClass::Blob(BlobOp::Quota { + hash: hash.clone(), + until, + })) + .clear(ValueClass::Blob(BlobOp::Undelete { hash, until })); + } Ok(true) }, @@ -211,7 +217,7 @@ pub async fn store_lookup_expire_all(store: &Store) { } #[allow(unused_variables)] -pub async fn store_assert_is_empty(store: &Store, blob_store: BlobStore) { +pub async fn store_assert_is_empty(store: &Store, blob_store: BlobStore, include_directory: bool) { store_blob_expire_all(store).await; store_lookup_expire_all(store).await; store.purge_blobs(blob_store).await.unwrap(); @@ -222,7 +228,7 @@ pub async fn store_assert_is_empty(store: &Store, blob_store: BlobStore) { for (subspace, with_values) in [ (SUBSPACE_ACL, true), - //(SUBSPACE_DIRECTORY, true), + (SUBSPACE_DIRECTORY, true), (SUBSPACE_TASK_QUEUE, true), (SUBSPACE_IN_MEMORY_VALUE, true), (SUBSPACE_IN_MEMORY_COUNTER, false), @@ -232,7 +238,7 @@ pub async fn store_assert_is_empty(store: &Store, blob_store: BlobStore) { (SUBSPACE_QUEUE_EVENT, true), (SUBSPACE_REPORT_OUT, true), (SUBSPACE_REPORT_IN, true), - (SUBSPACE_BLOB_RESERVE, true), + (SUBSPACE_BLOB_EXTRA, true), (SUBSPACE_BLOB_LINK, true), (SUBSPACE_BLOBS, true), (SUBSPACE_COUNTER, false), @@ -242,7 +248,9 @@ pub async fn store_assert_is_empty(store: &Store, blob_store: BlobStore) { (SUBSPACE_TELEMETRY_METRIC, true), (SUBSPACE_SEARCH_INDEX, true), ] { - if subspace == SUBSPACE_SEARCH_INDEX && store.is_pg_or_mysql() { + if (subspace == SUBSPACE_SEARCH_INDEX && store.is_pg_or_mysql()) + || (subspace == SUBSPACE_DIRECTORY && !include_directory) + { continue; } diff --git a/tests/src/store/import_export.rs b/tests/src/store/import_export.rs index 3f0ab3df..d28e30fd 100644 --- a/tests/src/store/import_export.rs +++ b/tests/src/store/import_export.rs @@ -4,13 +4,16 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::store::TempDir; +use crate::store::{ + TempDir, + cleanup::{store_assert_is_empty, store_destroy}, +}; use ahash::AHashSet; -use common::{Core, manager::backup::BackupParams}; +use common::{Core, DATABASE_SCHEMA_VERSION, manager::backup::BackupParams}; use store::{ rand, write::{ - AnyKey, BatchBuilder, BlobOp, DirectoryClass, InMemoryClass, Operation, QueueClass, + AnyClass, AnyKey, BatchBuilder, BlobLink, BlobOp, DirectoryClass, Operation, QueueClass, QueueEvent, ValueClass, }, *, @@ -29,11 +32,18 @@ pub async fn test(db: Store) { core.storage.lookup = db.clone().into(); // Make sure the store is empty - db.assert_is_empty(db.clone().into()).await; + store_assert_is_empty(&db, db.clone().into(), true).await; // Create blobs println!("Creating blobs..."); let mut batch = BatchBuilder::new(); + batch.set( + ValueClass::Any(AnyClass { + subspace: SUBSPACE_PROPERTY, + key: vec![0u8], + }), + DATABASE_SCHEMA_VERSION.serialize(), + ); let mut blob_hashes = Vec::new(); for blob_size in [16, 128, 1024, 2056, 102400] { let data = random_bytes(blob_size); @@ -82,13 +92,6 @@ pub async fn test(db: Store) { batch.set(ValueClass::Property(idx as u8), random_bytes(value_size)); } - for value_size in [1, 4, 7, 8, 9, 16] { - batch.set( - ValueClass::FtsIndex(BitmapHash::new(random_bytes(value_size))), - random_bytes(value_size * 2), - ); - } - for grant_account_id in 0u32..10u32 { if account_id != grant_account_id { batch.set( @@ -100,50 +103,17 @@ pub async fn test(db: Store) { for hash in &blob_hashes { batch.set( - ValueClass::Blob(BlobOp::Link { hash: hash.clone() }), + ValueClass::Blob(BlobOp::Link { + hash: hash.clone(), + to: BlobLink::Document, + }), vec![], ); } batch.log_item_insert(SyncCollection::from(collection), None); - /*batch.any_op(Operation::ChangeId { - change_id: document_id as u64 + account_id as u64 + collection as u64, - }); - - batch.any_op(Operation::Log { - set: MaybeDynamicValue::Static(vec![ - account_id as u8, - collection, - document_id as u8, - ]), - });*/ - for field in 0..5 { - batch.any_op(Operation::Bitmap { - class: BitmapClass::Tag { - field, - value: TagValue::Id(rand::random()), - }, - set: true, - }); - - batch.any_op(Operation::Bitmap { - class: BitmapClass::Tag { - field, - value: TagValue::Text(random_bytes(field as usize + 2)), - }, - set: true, - }); - - batch.any_op(Operation::Bitmap { - class: BitmapClass::Text { - field, - token: BitmapHash::new(random_bytes(field as usize + 2)), - }, - set: true, - }); - batch.any_op(Operation::Index { field, key: random_bytes(field as usize + 2), @@ -172,14 +142,14 @@ pub async fn test(db: Store) { })), random_bytes(idx), ); - batch.set( + /*batch.set( ValueClass::InMemory(InMemoryClass::Key(random_bytes(idx))), random_bytes(idx), ); batch.add( ValueClass::InMemory(InMemoryClass::Counter(random_bytes(idx))), rand::random(), - ); + );*/ batch.set( ValueClass::Config(random_bytes(idx + 10)), random_bytes(idx + 10), @@ -246,8 +216,8 @@ pub async fn test(db: Store) { // Destroy store println!("Destroying store..."); - db.destroy().await; - db.assert_is_empty(db.clone().into()).await; + store_destroy(&db).await; + store_assert_is_empty(&db, db.clone().into(), true).await; // Import store println!("Importing store..."); @@ -259,7 +229,7 @@ pub async fn test(db: Store) { println!(" GREAT SUCCESS!"); // Destroy store - db.destroy().await; + store_destroy(&db).await; temp_dir.delete(); } @@ -286,7 +256,7 @@ impl Snapshot { (SUBSPACE_DIRECTORY, true), (SUBSPACE_TASK_QUEUE, true), (SUBSPACE_INDEXES, false), - (SUBSPACE_BLOB_RESERVE, true), + (SUBSPACE_BLOB_EXTRA, true), (SUBSPACE_BLOB_LINK, true), (SUBSPACE_BLOBS, true), (SUBSPACE_LOGS, true), diff --git a/tests/src/store/lookup.rs b/tests/src/store/lookup.rs index c2d6598b..5f226bc1 100644 --- a/tests/src/store/lookup.rs +++ b/tests/src/store/lookup.rs @@ -68,7 +68,7 @@ pub async fn lookup_tests() { store.purge_in_memory_store().await.unwrap(); if let InMemoryStore::Store(store) = &store { - store_assert_is_empty(store, store.clone().into()).await; + store_assert_is_empty(store, store.clone().into(), false).await; } // Test counter @@ -126,7 +126,7 @@ pub async fn lookup_tests() { tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; store.purge_in_memory_store().await.unwrap(); if let InMemoryStore::Store(store) = &store { - store_assert_is_empty(store, store.clone().into()).await; + store_assert_is_empty(store, store.clone().into(), false).await; } // Test locking @@ -152,7 +152,7 @@ pub async fn lookup_tests() { } store.purge_in_memory_store().await.unwrap(); if let InMemoryStore::Store(store) = &store { - store_assert_is_empty(store, store.clone().into()).await; + store_assert_is_empty(store, store.clone().into(), false).await; } // Test prefix delete @@ -284,7 +284,7 @@ pub async fn lookup_tests() { ); if let InMemoryStore::Store(store) = &store { - store_assert_is_empty(store, store.clone().into()).await; + store_assert_is_empty(store, store.clone().into(), false).await; } } } diff --git a/tests/src/store/mod.rs b/tests/src/store/mod.rs index 5390e795..49878393 100644 --- a/tests/src/store/mod.rs +++ b/tests/src/store/mod.rs @@ -5,8 +5,8 @@ */ pub mod blob; -//pub mod import_export; pub mod cleanup; +pub mod import_export; pub mod lookup; pub mod ops; pub mod query; @@ -45,7 +45,7 @@ pub async fn store_tests() { store_destroy(&store).await; } - //import_export::test(store.clone()).await; + import_export::test(store.clone()).await; ops::test(store.clone()).await; if insert { diff --git a/tests/src/store/ops.rs b/tests/src/store/ops.rs index e1c67a3a..e749a46d 100644 --- a/tests/src/store/ops.rs +++ b/tests/src/store/ops.rs @@ -473,6 +473,6 @@ pub async fn test(db: Store) { db.write(batch.build_all()).await.unwrap(); // Make sure everything is deleted - store_assert_is_empty(&db, db.clone().into()).await; + store_assert_is_empty(&db, db.clone().into(), false).await; } }