diff --git a/crates/imap/src/core/mailbox.rs b/crates/imap/src/core/mailbox.rs index 35878435..ebeb1b80 100644 --- a/crates/imap/src/core/mailbox.rs +++ b/crates/imap/src/core/mailbox.rs @@ -11,10 +11,7 @@ use jmap_proto::{ types::{acl::Acl, collection::Collection, id::Id, property::Property, value::Value}, }; use parking_lot::Mutex; -use store::{ - query::log::{Change, Query}, - StoreRead, -}; +use store::query::log::{Change, Query}; use tokio::io::AsyncRead; use utils::{listener::limiter::InFlight, map::mutex_map::MutexMap}; diff --git a/crates/imap/src/core/message.rs b/crates/imap/src/core/message.rs index 844d21af..11638919 100644 --- a/crates/imap/src/core/message.rs +++ b/crates/imap/src/core/message.rs @@ -32,7 +32,7 @@ use jmap_proto::types::{collection::Collection, property::Property}; use store::{ roaring::RoaringBitmap, write::{assert::HashedValue, now, BatchBuilder, ToBitmaps, F_VALUE}, - Deserialize, Serialize, StoreRead, StoreWrite, + Deserialize, Serialize, }; use utils::codec::leb128::{Leb128Iterator, Leb128Vec}; diff --git a/crates/imap/src/op/fetch.rs b/crates/imap/src/op/fetch.rs index bdcb58a6..17c82264 100644 --- a/crates/imap/src/op/fetch.rs +++ b/crates/imap/src/op/fetch.rs @@ -41,8 +41,8 @@ use jmap::{email::metadata::MessageMetadata, Bincode}; use jmap_proto::{ error::method::MethodError, types::{ - acl::Acl, blob::BlobId, collection::Collection, id::Id, keyword::Keyword, - property::Property, state::StateChange, type_state::DataType, + acl::Acl, collection::Collection, id::Id, keyword::Keyword, property::Property, + state::StateChange, type_state::DataType, }, }; use mail_parser::{Address, GetHeader, HeaderName, Message, PartType}; @@ -298,15 +298,14 @@ impl SessionData { // Fetch and parse blob let raw_message = if needs_blobs { // Retrieve raw message if needed - let blob_id = BlobId::maildir(account_id, id); - match self.jmap.get_blob(&blob_id.kind, 0..u32::MAX).await { + match self.jmap.get_blob(&email.blob_hash, 0..u32::MAX).await { Ok(Some(raw_message)) => raw_message.into(), Ok(None) => { tracing::warn!(event = "not-found", account_id = account_id, collection = ?Collection::Email, document_id = id, - blob_id = ?blob_id, + blob_id = ?email.blob_hash, "Blob not found"); continue; } diff --git a/crates/imap/src/op/search.rs b/crates/imap/src/op/search.rs index e627a873..3f8ec0df 100644 --- a/crates/imap/src/op/search.rs +++ b/crates/imap/src/op/search.rs @@ -36,12 +36,7 @@ use jmap_proto::types::{collection::Collection, id::Id, keyword::Keyword, proper use mail_parser::HeaderName; use nlp::language::Language; use store::{ - query::{ - self, - log::Query, - sort::{Pagination, StoreSort}, - ResultSet, - }, + query::{self, log::Query, sort::Pagination, ResultSet}, roaring::RoaringBitmap, write::now, }; diff --git a/crates/imap/src/op/status.rs b/crates/imap/src/op/status.rs index eead1dc7..0095a422 100644 --- a/crates/imap/src/op/status.rs +++ b/crates/imap/src/op/status.rs @@ -30,8 +30,8 @@ use imap_proto::{ Command, ResponseCode, StatusResponse, }; use jmap_proto::types::{collection::Collection, id::Id, keyword::Keyword, property::Property}; +use store::roaring::RoaringBitmap; use store::Deserialize; -use store::{roaring::RoaringBitmap, StoreRead}; use tokio::io::AsyncRead; use crate::core::{Mailbox, Session, SessionData}; diff --git a/crates/imap/src/op/thread.rs b/crates/imap/src/op/thread.rs index d65aca13..9c5d6721 100644 --- a/crates/imap/src/op/thread.rs +++ b/crates/imap/src/op/thread.rs @@ -34,7 +34,7 @@ use imap_proto::{ }; use jmap_proto::types::{collection::Collection, property::Property}; -use store::{write::ValueClass, StoreRead, ValueKey}; +use store::{write::ValueClass, ValueKey}; use tokio::io::AsyncRead; use crate::core::{SelectedMailbox, Session, SessionData}; diff --git a/crates/jmap-proto/src/object/index.rs b/crates/jmap-proto/src/object/index.rs index 3a3262f6..8b8a546d 100644 --- a/crates/jmap-proto/src/object/index.rs +++ b/crates/jmap-proto/src/object/index.rs @@ -144,6 +144,10 @@ impl ObjectIndexBuilder { self.changes.as_ref() } + pub fn changes_mut(&mut self) -> Option<&mut Object> { + self.changes.as_mut() + } + pub fn current(&self) -> Option<&HashedValue>> { self.current.as_ref() } diff --git a/crates/jmap-proto/src/types/blob.rs b/crates/jmap-proto/src/types/blob.rs index eaee0442..c34344c0 100644 --- a/crates/jmap-proto/src/types/blob.rs +++ b/crates/jmap-proto/src/types/blob.rs @@ -24,9 +24,8 @@ use std::{borrow::Borrow, io::Write}; use store::{ - rand::{self, Rng}, - write::{now, DeserializeFrom, SerializeInto}, - BlobKind, + write::{DeserializeFrom, SerializeInto}, + BlobClass, BlobHash, }; use utils::codec::{ base32_custom::{Base32Reader, Base32Writer}, @@ -35,15 +34,13 @@ use utils::codec::{ use crate::parser::{base32::JsonBase32Reader, json::Parser, JsonObjectParser}; -use super::collection::Collection; - const B_LINKED: u8 = 0x10; -const B_LINKED_MAILDIR: u8 = 0x20; -const B_TEMPORARY: u8 = 0x40; +const B_RESERVED: u8 = 0x20; -#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[derive(Clone, Debug, PartialEq, Eq, Hash, Default)] pub struct BlobId { - pub kind: BlobKind, + pub hash: BlobHash, + pub class: BlobClass, pub section: Option, } @@ -54,53 +51,6 @@ pub struct BlobSection { pub encoding: u8, } -impl BlobId { - pub fn maildir(account_id: u32, document_id: u32) -> Self { - Self { - kind: BlobKind::LinkedMaildir { - account_id, - document_id, - }, - section: None, - } - } - - pub fn linked(account_id: u32, collection: Collection, document_id: u32) -> Self { - Self { - kind: BlobKind::Linked { - account_id, - collection: collection.into(), - document_id, - }, - section: None, - } - } - - pub fn temporary(account_id: u32) -> Self { - Self { - kind: BlobKind::Temporary { - account_id, - timestamp: now(), - seq: rand::thread_rng().gen_range(0u32..=u32::MAX), - }, - section: None, - } - } - - pub fn account_id(&self) -> u32 { - match &self.kind { - BlobKind::Linked { account_id, .. } => *account_id, - BlobKind::LinkedMaildir { account_id, .. } => *account_id, - BlobKind::Temporary { account_id, .. } => *account_id, - } - } - - pub fn with_section_size(mut self, size: usize) -> Self { - self.section.get_or_insert_with(Default::default).size = size; - self - } -} - impl JsonObjectParser for BlobId { fn parse(parser: &mut Parser<'_>) -> crate::parser::Result where @@ -112,13 +62,38 @@ impl JsonObjectParser for BlobId { } impl BlobId { - pub fn new(kind: BlobKind) -> Self { + pub fn new(hash: BlobHash, class: BlobClass) -> Self { BlobId { - kind, + hash, + class, section: None, } } + pub fn new_section( + hash: BlobHash, + class: BlobClass, + offset_start: usize, + offset_end: usize, + encoding: impl Into, + ) -> Self { + BlobId { + hash, + class, + section: BlobSection { + offset_start, + size: offset_end - offset_start, + encoding: encoding.into(), + } + .into(), + } + } + + pub fn with_section_size(mut self, size: usize) -> Self { + self.section.get_or_insert_with(Default::default).size = size; + self + } + pub fn from_base32(value: impl AsRef<[u8]>) -> Option { BlobId::from_iter(&mut Base32Reader::new(value.as_ref())) } @@ -129,26 +104,26 @@ impl BlobId { T: Iterator + Leb128Iterator, U: Borrow, { - let kind = *it.next()?.borrow(); - let encoding = kind & 0x0F; + let class = *it.next()?.borrow(); + let encoding = class & 0x0F; + + let mut hash = BlobHash::default(); + for byte in hash.as_mut().iter_mut() { + *byte = *it.next()?.borrow(); + } + + let account_id: u32 = it.next_leb128()?; BlobId { - kind: match kind & 0xF0 { - B_LINKED => BlobKind::Linked { - account_id: it.next_leb128()?, + hash, + class: if (class & B_LINKED) != 0 { + BlobClass::Linked { + account_id, collection: *it.next()?.borrow(), document_id: it.next_leb128()?, - }, - B_LINKED_MAILDIR => BlobKind::LinkedMaildir { - account_id: it.next_leb128()?, - document_id: it.next_leb128()?, - }, - B_TEMPORARY => BlobKind::Temporary { - account_id: it.next_leb128()?, - timestamp: it.next_leb128()?, - seq: it.next_leb128()?, - }, - _ => return None, + } + } else { + BlobClass::Reserved { account_id } }, section: if encoding != 0 { BlobSection { @@ -165,39 +140,38 @@ impl BlobId { } fn serialize_as(&self, writer: &mut (impl Write + Leb128Writer)) { - let kind = self + let marker = self .section .as_ref() - .map_or(0, |section| section.encoding + 1); - match &self.kind { - BlobKind::Linked { + .map_or(0, |section| section.encoding + 1) + | if matches!( + self, + BlobId { + class: BlobClass::Linked { .. }, + .. + } + ) { + B_LINKED + } else { + B_RESERVED + }; + + let _ = writer.write(&[marker]); + let _ = writer.write(self.hash.as_ref()); + + match &self.class { + BlobClass::Reserved { account_id } => { + let _ = writer.write_leb128(*account_id); + } + BlobClass::Linked { account_id, collection, document_id, } => { - let _ = writer.write(&[kind | B_LINKED]); let _ = writer.write_leb128(*account_id); let _ = writer.write(&[*collection]); let _ = writer.write_leb128(*document_id); } - BlobKind::LinkedMaildir { - account_id, - document_id, - } => { - let _ = writer.write(&[kind | B_LINKED_MAILDIR]); - let _ = writer.write_leb128(*account_id); - let _ = writer.write_leb128(*document_id); - } - BlobKind::Temporary { - account_id, - timestamp, - seq, - } => { - let _ = writer.write(&[kind | B_TEMPORARY]); - let _ = writer.write_leb128(*account_id); - let _ = writer.write_leb128(*timestamp); - let _ = writer.write_leb128(*seq); - } } if let Some(section) = &self.section { @@ -206,23 +180,6 @@ impl BlobId { } } - pub fn new_section( - kind: BlobKind, - offset_start: usize, - offset_end: usize, - encoding: impl Into, - ) -> Self { - BlobId { - kind, - section: BlobSection { - offset_start, - size: offset_end - offset_start, - encoding: encoding.into(), - } - .into(), - } - } - pub fn start_offset(&self) -> usize { if let Some(section) = &self.section { section.offset_start @@ -232,30 +189,6 @@ impl BlobId { } } -impl Default for BlobId { - fn default() -> Self { - BlobId { - kind: store::BlobKind::LinkedMaildir { - account_id: u32::MAX, - document_id: u32::MAX, - }, - section: None, - } - } -} - -impl From<&BlobKind> for BlobId { - fn from(kind: &BlobKind) -> Self { - BlobId::new(*kind) - } -} - -impl From for BlobId { - fn from(id: BlobKind) -> Self { - BlobId::new(id) - } -} - impl serde::Serialize for BlobId { fn serialize(&self, serializer: S) -> Result where diff --git a/crates/jmap-proto/src/types/value.rs b/crates/jmap-proto/src/types/value.rs index b5d7b1e3..59fdce87 100644 --- a/crates/jmap-proto/src/types/value.rs +++ b/crates/jmap-proto/src/types/value.rs @@ -25,7 +25,6 @@ use std::{borrow::Cow, fmt::Display}; use mail_parser::{Addr, DateTime, Group}; use serde::Serialize; -use store::BlobKind; use crate::{ object::Object, @@ -269,6 +268,13 @@ impl Value { } } + pub fn as_blob_id(&self) -> Option<&BlobId> { + match self { + Value::BlobId(id) => Some(id), + _ => None, + } + } + pub fn as_list(&self) -> Option<&Vec> { match self { Value::List(l) => Some(l), @@ -414,12 +420,6 @@ impl From for Value { } } -impl From for Value { - fn from(value: BlobKind) -> Self { - Value::BlobId(BlobId::new(value)) - } -} - impl From for Value { fn from(value: Id) -> Self { Value::Id(value) diff --git a/crates/jmap/src/api/admin.rs b/crates/jmap/src/api/admin.rs index bf2d593b..aeebbacf 100644 --- a/crates/jmap/src/api/admin.rs +++ b/crates/jmap/src/api/admin.rs @@ -21,60 +21,34 @@ * for more details. */ -use jmap_proto::{ - object::{index::ObjectIndexBuilder, Object}, - types::{collection::Collection, property::Property, value::Value}, -}; -use store::{ - write::{assert::HashedValue, BatchBuilder, ValueClass}, - BitmapKey, Serialize, StorePurge, StoreRead, StoreWrite, ValueKey, -}; +use jmap_proto::types::collection::Collection; +use store::{write::BatchBuilder, Serialize}; -use crate::{mailbox::set::SCHEMA, NamedKey, JMAP}; +use crate::{NamedKey, JMAP}; impl JMAP { pub async fn delete_account(&self, account_name: &str, account_id: u32) -> store::Result<()> { - // Delete blobs - self.store.delete_account_blobs(account_id).await?; + let test = true; - // Delete mailboxes + // Unlink all account's blobs + self.store.blob_hash_unlink_account(account_id).await?; + + // Revoke ACLs + self.store.acl_revoke_all(account_id).await?; + + // Delete account data + self.store.purge_account(account_id).await?; + + // Delete account let mut batch = BatchBuilder::new(); batch .with_account_id(u32::MAX) .with_collection(Collection::Principal) .clear(NamedKey::Name(account_name)) .clear(NamedKey::Id::<&[u8]>(account_id)) - .clear(NamedKey::Quota::<&[u8]>(account_id)) - .with_account_id(account_id) - .with_collection(Collection::Mailbox); - for mailbox_id in self - .store - .get_bitmap(BitmapKey::document_ids(account_id, Collection::Mailbox)) - .await? - .unwrap_or_default() - { - let mailbox = self - .store - .get_value::>>(ValueKey { - account_id, - collection: Collection::Mailbox.into(), - document_id: mailbox_id, - class: ValueClass::Property(Property::Value.into()), - }) - .await? - .ok_or_else(|| { - store::Error::InternalError(format!("Mailbox {} not found", mailbox_id)) - })?; - batch - .delete_document(mailbox_id) - .custom(ObjectIndexBuilder::new(SCHEMA).with_current(mailbox)); - } - if !batch.is_empty() { - self.store.write(batch.build()).await?; - } + .clear(NamedKey::Quota::<&[u8]>(account_id)); - // Delete account - self.store.purge_account(account_id).await?; + self.store.write(batch.build()).await?; Ok(()) } @@ -85,10 +59,6 @@ impl JMAP { account_name: &str, account_id: u32, ) -> store::Result<()> { - // Delete blobs - self.store.delete_account_blobs(account_id).await?; - - // Delete mailboxes let mut batch = BatchBuilder::new(); batch .with_account_id(u32::MAX) diff --git a/crates/jmap/src/api/http.rs b/crates/jmap/src/api/http.rs index 7671fc4c..56806f90 100644 --- a/crates/jmap/src/api/http.rs +++ b/crates/jmap/src/api/http.rs @@ -365,7 +365,7 @@ pub async fn parse_jmap_request( }; } ("blob", "purge", &Method::GET) => { - return match jmap.store.purge_tmp_blobs(jmap.config.upload_tmp_ttl).await { + return match jmap.store.blob_hash_purge(jmap.blob_store.clone()).await { Ok(_) => { JsonResponse::new(Value::String("success".into())).into_http_response() } diff --git a/crates/jmap/src/auth/acl.rs b/crates/jmap/src/auth/acl.rs index 74cd437a..5c3c9cb5 100644 --- a/crates/jmap/src/auth/acl.rs +++ b/crates/jmap/src/auth/acl.rs @@ -32,13 +32,10 @@ use jmap_proto::{ }, }; use store::{ + query::acl::AclQuery, roaring::RoaringBitmap, - write::{ - assert::HashedValue, - key::{AclKey, DeserializeBigEndian}, - ValueClass, - }, - Deserialize, Error, StoreRead, ValueKey, + write::{assert::HashedValue, ValueClass}, + ValueKey, }; use utils::map::bitmap::{Bitmap, BitmapItem}; @@ -52,32 +49,30 @@ impl JMAP { .iter() .chain(access_token.member_of.clone().iter()) { - let from_key = ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::Acl(grant_account_id), - }; - let to_key = ValueKey { - account_id: u32::MAX, - collection: u8::MAX, - document_id: u32::MAX, - class: ValueClass::Acl(grant_account_id), - }; - match self + for acl_item in self .store - .iterate(from_key, to_key, false, true, |key, value| { - let acl_key = AclKey::deserialize(key)?; - if access_token.is_member(acl_key.to_account_id) { - return Ok(true); - } - - let acl = Bitmap::::from(u64::deserialize(value)?); - let collection = Collection::from(acl_key.to_collection); + .acl_query(AclQuery::HasAccess { grant_account_id }) + .await + .map_err(|err| { + tracing::error!( + event = "error", + context = "update_access_token", + error = ?err, + "Failed to iterate ACLs."); + }) + .ok()? + { + if !access_token.is_member(acl_item.to_account_id) { + let acl = Bitmap::::from(acl_item.permissions); + let collection = Collection::from(acl_item.to_collection); if !collection.is_valid() { - return Err(Error::InternalError(format!( - "Found corrupted collection in key {key:?}" - ))); + tracing::warn!( + event = "error", + context = "update_access_token", + error = ?acl_item, + "Found corrupted collection in key" + ); + return None; } let mut collections: Bitmap = Bitmap::new(); @@ -94,28 +89,15 @@ impl JMAP { if let Some((_, sharing)) = access_token .access_to .iter_mut() - .find(|(account_id, _)| *account_id == acl_key.to_account_id) + .find(|(account_id, _)| *account_id == acl_item.to_account_id) { sharing.union(&collections); } else { access_token .access_to - .push((acl_key.to_account_id, collections)); + .push((acl_item.to_account_id, collections)); } } - - Ok(true) - }) - .await - { - Ok(_) => {} - Err(err) => { - tracing::error!( - event = "error", - context = "shared_accounts", - error = ?err, - "Failed to iterate ACLs."); - return None; } } } @@ -136,39 +118,28 @@ impl JMAP { .iter() .chain(access_token.member_of.clone().iter()) { - let from_key = ValueKey { - account_id: to_account_id, - collection: to_collection, - document_id: 0, - class: ValueClass::Acl(grant_account_id), - }; - let mut to_key = from_key.clone(); - to_key.document_id = u32::MAX; - - match self + for acl_item in self .store - .iterate(from_key, to_key, false, true, |key, value| { - let mut acls = Bitmap::::from(u64::deserialize(value)?); - - acls.intersection(&check_acls); - if !acls.is_empty() { - document_ids.insert( - key.deserialize_be_u32(key.len() - std::mem::size_of::())?, - ); - } - - Ok(true) + .acl_query(AclQuery::SharedWith { + grant_account_id, + to_account_id, + to_collection, }) .await - { - Ok(_) => (), - Err(err) => { + .map_err(|err| { tracing::error!( - event = "error", - context = "shared_accounts", - error = ?err, - "Failed to iterate ACLs."); - return Err(MethodError::ServerPartialFail); + event = "error", + context = "shared_documents", + error = ?err, + "Failed to iterate ACLs."); + MethodError::ServerPartialFail + })? + { + let mut acls = Bitmap::::from(acl_item.permissions); + + acls.intersection(&check_acls); + if !acls.is_empty() { + document_ids.insert(acl_item.to_document_id); } } } diff --git a/crates/jmap/src/auth/authenticate.rs b/crates/jmap/src/auth/authenticate.rs index 4c4b49fd..8003b67d 100644 --- a/crates/jmap/src/auth/authenticate.rs +++ b/crates/jmap/src/auth/authenticate.rs @@ -34,7 +34,7 @@ use jmap_proto::{ }; use mail_parser::decoders::base64::base64_decode; use mail_send::Credentials; -use store::{write::BatchBuilder, Serialize, StoreRead, StoreWrite}; +use store::{write::BatchBuilder, Serialize}; use utils::{listener::limiter::InFlight, map::ttl_dashmap::TtlMap}; use crate::{NamedKey, JMAP}; diff --git a/crates/jmap/src/blob/copy.rs b/crates/jmap/src/blob/copy.rs index 69e6a1d3..66d93b8a 100644 --- a/crates/jmap/src/blob/copy.rs +++ b/crates/jmap/src/blob/copy.rs @@ -30,6 +30,10 @@ use jmap_proto::{ types::blob::BlobId, }; +use store::{ + write::{now, BatchBuilder, BlobOp}, + BlobClass, +}; use utils::map::vec_map::VecMap; use crate::{auth::AccessToken, JMAP}; @@ -50,44 +54,29 @@ impl JMAP { for blob_id in request.blob_ids { if self.has_access_blob(&blob_id, access_token).await? { - let dest_blob_id = BlobId::temporary(account_id); - match self - .store - .copy_blob( - &blob_id.kind, - &dest_blob_id.kind, - blob_id - .section - .as_ref() - .map(|s| (s.offset_start as u32)..((s.offset_start + s.size) as u32)), - ) - .await - { - Ok(success) => { - if success { - response.copied.append(blob_id, dest_blob_id); - } else { - response.not_copied.append( - blob_id, - SetError::new(SetErrorType::BlobNotFound) - .with_description("blobId does not exist."), - ); - } - } - Err(err) => { - tracing::error!( - context = "copy_blob", - event = "error", - reason = %err, - "Failed to copy blob"); - return Err(MethodError::ServerPartialFail); - } - } + let mut batch = BatchBuilder::new(); + batch.with_account_id(account_id).blob( + blob_id.hash.clone(), + BlobOp::Reserve { + until: now() + self.config.upload_tmp_ttl, + size: 0, + }, + 0, + ); + self.write_batch(batch).await?; + let dest_blob_id = BlobId { + hash: blob_id.hash.clone(), + class: BlobClass::Reserved { account_id }, + section: blob_id.section.clone(), + }; + + response.copied.append(blob_id, dest_blob_id); } else { response.not_copied.append( blob_id, - SetError::forbidden() - .with_description("You do not have access to this blobId."), + SetError::new(SetErrorType::BlobNotFound).with_description( + "blobId does not exist or not enough permissions to access it.", + ), ); } } diff --git a/crates/jmap/src/blob/download.rs b/crates/jmap/src/blob/download.rs index 466db936..f27d2132 100644 --- a/crates/jmap/src/blob/download.rs +++ b/crates/jmap/src/blob/download.rs @@ -28,74 +28,91 @@ use jmap_proto::{ types::{ acl::Acl, blob::{BlobId, BlobSection}, + collection::Collection, }, }; use mail_parser::{ decoders::{base64::base64_decode, quoted_printable::quoted_printable_decode}, Encoding, }; -use store::BlobKind; +use store::{BlobClass, BlobHash}; use crate::{auth::AccessToken, JMAP}; impl JMAP { + #[allow(clippy::blocks_in_if_conditions)] pub async fn blob_download( &self, blob_id: &BlobId, access_token: &AccessToken, ) -> Result>, MethodError> { - if !access_token.is_member(blob_id.account_id()) { - match &blob_id.kind { - BlobKind::Linked { + if !self + .store + .blob_hash_can_read(&blob_id.hash, &blob_id.class) + .await + .map_err(|err| { + tracing::error!(event = "error", + context = "blob_download", + error = ?err, + "Failed to validate blob access"); + MethodError::ServerPartialFail + })? + { + return Ok(None); + } + + if !access_token.is_member(blob_id.class.account_id()) { + match &blob_id.class { + BlobClass::Linked { account_id, collection, document_id, } => { - match self - .has_access_to_document( - access_token, - *account_id, - *collection, - *document_id, - Acl::Read, - ) - .await - { - Ok(has_access) if has_access => (), - _ => return Ok(None), + if Collection::from(*collection) == Collection::Email { + match self + .shared_messages(access_token, *account_id, Acl::ReadItems) + .await + { + Ok(shared_messages) if shared_messages.contains(*document_id) => (), + _ => return Ok(None), + } + } else { + match self + .has_access_to_document( + access_token, + *account_id, + *collection, + *document_id, + Acl::Read, + ) + .await + { + Ok(has_access) if has_access => (), + _ => return Ok(None), + } } } - BlobKind::LinkedMaildir { - account_id, - document_id, - } => { - match self - .shared_messages(access_token, *account_id, Acl::ReadItems) - .await - { - Ok(shared_messages) if shared_messages.contains(*document_id) => (), - _ => return Ok(None), - } + BlobClass::Reserved { .. } => { + return Ok(None); } - BlobKind::Temporary { .. } => return Ok(None), } } if let Some(section) = &blob_id.section { - self.get_blob_section(&blob_id.kind, section).await + self.get_blob_section(&blob_id.hash, section).await } else { - self.get_blob(&blob_id.kind, 0..u32::MAX).await + self.get_blob(&blob_id.hash, 0..u32::MAX).await } } pub async fn get_blob_section( &self, - kind: &BlobKind, + hash: &BlobHash, section: &BlobSection, ) -> Result>, MethodError> { Ok(self .get_blob( - kind, + hash, (section.offset_start as u32) ..(section.offset_start.saturating_add(section.size) as u32), ) @@ -109,15 +126,15 @@ impl JMAP { pub async fn get_blob( &self, - kind: &BlobKind, + hash: &BlobHash, range: Range, ) -> Result>, MethodError> { - match self.store.get_blob(kind, range).await { + match self.blob_store.get_blob(hash.as_ref(), range).await { Ok(blob) => Ok(blob), Err(err) => { tracing::error!(event = "error", context = "blob_store", - blob_id = ?kind, + blob_id = ?hash, error = ?err, "Failed to retrieve blob"); Err(MethodError::ServerPartialFail) @@ -130,35 +147,44 @@ impl JMAP { blob_id: &BlobId, access_token: &AccessToken, ) -> Result { - Ok(match &blob_id.kind { - BlobKind::Linked { - account_id, - collection, - document_id, - } => { - access_token.is_member(*account_id) - || (access_token.has_access(*account_id, *collection) - && self - .has_access_to_document( - access_token, - *account_id, - *collection, - *document_id, - Acl::Read, - ) - .await?) - } - BlobKind::LinkedMaildir { - account_id, - document_id, - } => { - access_token.is_member(*account_id) - || self - .shared_messages(access_token, *account_id, Acl::ReadItems) - .await? - .contains(*document_id) - } - BlobKind::Temporary { account_id, .. } => access_token.is_member(*account_id), - }) + Ok(self + .store + .blob_hash_can_read(&blob_id.hash, &blob_id.class) + .await + .map_err(|err| { + tracing::error!(event = "error", + context = "has_access_blob", + error = ?err, + "Failed to validate blob access"); + MethodError::ServerPartialFail + })? + && match &blob_id.class { + BlobClass::Linked { + account_id, + collection, + document_id, + } => { + if Collection::from(*collection) == Collection::Email { + access_token.is_member(*account_id) + || self + .shared_messages(access_token, *account_id, Acl::ReadItems) + .await? + .contains(*document_id) + } else { + access_token.is_member(*account_id) + || (access_token.has_access(*account_id, *collection) + && self + .has_access_to_document( + access_token, + *account_id, + *collection, + *document_id, + Acl::Read, + ) + .await?) + } + } + BlobClass::Reserved { account_id } => access_token.is_member(*account_id), + }) } } diff --git a/crates/jmap/src/blob/get.rs b/crates/jmap/src/blob/get.rs index d3d65760..95106d76 100644 --- a/crates/jmap/src/blob/get.rs +++ b/crates/jmap/src/blob/get.rs @@ -40,7 +40,7 @@ use jmap_proto::{ use mail_builder::encoders::base64::base64_encode; use sha1::{Digest, Sha1}; use sha2::{Sha256, Sha512}; -use store::BlobKind; +use store::BlobClass; use utils::map::vec_map::VecMap; use crate::{auth::AccessToken, JMAP}; @@ -208,69 +208,64 @@ impl JMAP { MaybeUnparsable::Value(id) => { let mut matched_ids = VecMap::new(); - match &id.kind { - BlobKind::Linked { + match &id.class { + BlobClass::Linked { account_id, collection, document_id, } if *account_id == req_account_id => { - if *account_id != req_account_id { - response.not_found.push(MaybeUnparsable::Value(id)); - continue; - } - - match DataType::try_from(Collection::from(*collection)) { - Ok(data_type) if type_names.contains(&data_type) => { - matched_ids.append(data_type, vec![Id::from(*document_id)]); + let collection = Collection::from(*collection); + if collection == Collection::Email { + if include_email || include_thread { + if let Some(thread_id) = self + .get_property::( + req_account_id, + Collection::Email, + *document_id, + Property::ThreadId, + ) + .await? + { + if include_email { + matched_ids.append( + DataType::Email, + vec![Id::from_parts(thread_id, *document_id)], + ); + } + if include_thread { + matched_ids.append( + DataType::Thread, + vec![Id::from(thread_id)], + ); + } + } } - _ => (), - } - } - BlobKind::LinkedMaildir { - account_id, - document_id, - } if *account_id == req_account_id => { - if include_email || include_thread { - if let Some(thread_id) = self - .get_property::( - req_account_id, - Collection::Email, - *document_id, - Property::ThreadId, - ) - .await? - { - if include_email { + if include_mailbox { + if let Some(mailboxes) = self + .get_property::>( + req_account_id, + Collection::Email, + *document_id, + Property::MailboxIds, + ) + .await? + { matched_ids.append( - DataType::Email, - vec![Id::from_parts(thread_id, *document_id)], + DataType::Mailbox, + mailboxes.into_iter().map(Id::from).collect::>(), ); } - if include_thread { - matched_ids - .append(DataType::Thread, vec![Id::from(thread_id)]); + } + } else { + match DataType::try_from(collection) { + Ok(data_type) if type_names.contains(&data_type) => { + matched_ids.append(data_type, vec![Id::from(*document_id)]); } - } - } - if include_mailbox { - if let Some(mailboxes) = self - .get_property::>( - req_account_id, - Collection::Email, - *document_id, - Property::MailboxIds, - ) - .await? - { - matched_ids.append( - DataType::Mailbox, - mailboxes.into_iter().map(Id::from).collect::>(), - ); + _ => (), } } } - BlobKind::Temporary { account_id, .. } if *account_id == req_account_id => { - } + BlobClass::Reserved { account_id } if *account_id == req_account_id => (), _ => { response.not_found.push(MaybeUnparsable::Value(id)); continue; diff --git a/crates/jmap/src/blob/upload.rs b/crates/jmap/src/blob/upload.rs index 69f7ef85..4f55954a 100644 --- a/crates/jmap/src/blob/upload.rs +++ b/crates/jmap/src/blob/upload.rs @@ -31,7 +31,10 @@ use jmap_proto::{ request::reference::MaybeReference, types::{blob::BlobId, id::Id}, }; -use store::BlobKind; +use store::{ + write::{now, BatchBuilder, BlobOp}, + BlobClass, BlobHash, +}; use crate::{auth::AccessToken, JMAP}; @@ -96,7 +99,7 @@ impl JMAP { .map(|length| (length as u32).saturating_add(offset)) .unwrap_or(u32::MAX); let bytes = if let Some(section) = &id.section { - self.get_blob_section(&id.kind, section) + self.get_blob_section(&id.hash, section) .await? .map(|bytes| { if offset == 0 && length == u32::MAX { @@ -112,7 +115,7 @@ impl JMAP { } }) } else { - self.get_blob(&id.kind, offset..length).await? + self.get_blob(&id.hash, offset..length).await? }; if let Some(bytes) = bytes { bytes @@ -152,9 +155,9 @@ impl JMAP { } // Enforce quota - let (total_files, total_bytes) = self + let used = self .store - .get_tmp_blob_usage(account_id, self.config.upload_tmp_ttl) + .blob_hash_quota(account_id) .await .map_err(|err| { tracing::error!(event = "error", @@ -166,9 +169,9 @@ impl JMAP { })?; if ((self.config.upload_tmp_quota_size > 0 - && total_bytes + data.len() > self.config.upload_tmp_quota_size) + && used.bytes + data.len() > self.config.upload_tmp_quota_size) || (self.config.upload_tmp_quota_amount > 0 - && total_files + 1 > self.config.upload_tmp_quota_amount)) + && used.count + 1 > self.config.upload_tmp_quota_amount)) && !access_token.is_super_user() { response.not_created.append( @@ -182,29 +185,14 @@ impl JMAP { } // Write blob - let blob_id = BlobId::temporary(account_id); - match self.store.put_blob(&blob_id.kind, &data).await { - Ok(_) => { - response.created.insert( - create_id, - BlobUploadResponseObject { - id: blob_id, - type_: upload_object.type_, - size: data.len(), - }, - ); - } - Err(err) => { - tracing::error!(event = "error", - context = "blob_store", - account_id = account_id, - blob_id = ?blob_id, - size = data.len(), - error = ?err, - "Failed to upload blob"); - return Err(MethodError::ServerPartialFail); - } - } + response.created.insert( + create_id, + BlobUploadResponseObject { + id: self.put_blob(account_id, &data, true).await?, + type_: upload_object.type_, + size: data.len(), + }, + ); } Ok(response) @@ -229,9 +217,9 @@ impl JMAP { } // Enforce quota - let (total_files, total_bytes) = self + let used = self .store - .get_tmp_blob_usage(account_id.document_id(), self.config.upload_tmp_ttl) + .blob_hash_quota(account_id.document_id()) .await .map_err(|err| { tracing::error!(event = "error", @@ -243,9 +231,9 @@ impl JMAP { })?; if ((self.config.upload_tmp_quota_size > 0 - && total_bytes + data.len() > self.config.upload_tmp_quota_size) + && used.bytes + data.len() > self.config.upload_tmp_quota_size) || (self.config.upload_tmp_quota_amount > 0 - && total_files + 1 > self.config.upload_tmp_quota_amount)) + && used.count + 1 > self.config.upload_tmp_quota_amount)) && !access_token.is_super_user() { let err = Err(RequestError::over_blob_quota( @@ -262,49 +250,69 @@ impl JMAP { return err; } - let blob_id = BlobId::temporary(account_id.document_id()); - - match self.store.put_blob(&blob_id.kind, data).await { - Ok(_) => Ok(UploadResponse { - account_id, - blob_id, - c_type: content_type.to_string(), - size: data.len(), - }), - Err(err) => { - tracing::error!(event = "error", - context = "blob_store", - account_id = account_id.document_id(), - blob_id = ?blob_id, - size = data.len(), - error = ?err, - "Failed to upload blob"); - Err(RequestError::internal_server_error()) - } - } - } - - pub async fn put_blob(&self, kind: &BlobKind, data: &[u8]) -> Result<(), MethodError> { - self.store.put_blob(kind, data).await.map_err(|err| { - tracing::error!( - event = "error", - context = "blob_put", - kind = ?kind, - error = ?err, - "Failed to store blob."); - MethodError::ServerPartialFail + Ok(UploadResponse { + account_id, + blob_id: self + .put_blob(account_id.document_id(), data, true) + .await + .map_err(|_| RequestError::internal_server_error())?, + c_type: content_type.to_string(), + size: data.len(), }) } - pub async fn delete_blob(&self, kind: &BlobKind) -> Result { - self.store.delete_blob(kind).await.map_err(|err| { + #[allow(clippy::blocks_in_if_conditions)] + pub async fn put_blob( + &self, + account_id: u32, + data: &[u8], + set_quota: bool, + ) -> Result { + // First reserve the hash + let hash = BlobHash::from(data); + let mut batch = BatchBuilder::new(); + + batch.with_account_id(account_id).blob( + hash.clone(), + BlobOp::Reserve { + until: now() + self.config.upload_tmp_ttl, + size: if set_quota { data.len() } else { 0 }, + }, + 0, + ); + self.write_batch(batch).await?; + + if !self.store.blob_hash_exists(&hash).await.map_err(|err| { tracing::error!( - event = "error", - context = "delete_blob", - kind = ?kind, - error = ?err, - "Failed to delete blob."); + event = "error", + context = "put_blob", + error = ?err, + "Failed to verify blob hash existence."); MethodError::ServerPartialFail + })? { + // Upload blob to store + self.blob_store + .put_blob(hash.as_ref(), data) + .await + .map_err(|err| { + tracing::error!( + event = "error", + context = "put_blob", + error = ?err, + "Failed to store blob."); + MethodError::ServerPartialFail + })?; + + // Commit blob + let mut batch = BatchBuilder::new(); + batch.blob(hash.clone(), BlobOp::Commit, 0); + self.write_batch(batch).await?; + } + + Ok(BlobId { + hash, + class: BlobClass::Reserved { account_id }, + section: None, }) } } diff --git a/crates/jmap/src/changes/get.rs b/crates/jmap/src/changes/get.rs index e2bf7924..4488716a 100644 --- a/crates/jmap/src/changes/get.rs +++ b/crates/jmap/src/changes/get.rs @@ -26,7 +26,7 @@ use jmap_proto::{ method::changes::{ChangesRequest, ChangesResponse, RequestArguments}, types::{collection::Collection, property::Property, state::State}, }; -use store::query::log::{Change, Changes, Query, StoreLog}; +use store::query::log::{Change, Changes, Query}; use crate::{auth::AccessToken, JMAP}; diff --git a/crates/jmap/src/changes/state.rs b/crates/jmap/src/changes/state.rs index 14f10a3d..08167e94 100644 --- a/crates/jmap/src/changes/state.rs +++ b/crates/jmap/src/changes/state.rs @@ -25,7 +25,6 @@ use jmap_proto::{ error::method::MethodError, types::{collection::Collection, state::State}, }; -use store::StoreRead; use crate::JMAP; diff --git a/crates/jmap/src/changes/write.rs b/crates/jmap/src/changes/write.rs index 995c3e38..70e9b791 100644 --- a/crates/jmap/src/changes/write.rs +++ b/crates/jmap/src/changes/write.rs @@ -22,10 +22,7 @@ */ use jmap_proto::error::method::MethodError; -use store::{ - write::{log::ChangeLogBuilder, BatchBuilder}, - StoreId, StoreWrite, -}; +use store::write::{log::ChangeLogBuilder, BatchBuilder}; use crate::JMAP; diff --git a/crates/jmap/src/email/body.rs b/crates/jmap/src/email/body.rs index c4dfd306..685bdc9f 100644 --- a/crates/jmap/src/email/body.rs +++ b/crates/jmap/src/email/body.rs @@ -69,7 +69,8 @@ impl ToBodyPart for Vec> { Property::BlobId if multipart.is_none() => { let base_offset = blob_id.start_offset(); BlobId::new_section( - blob_id.kind, + blob_id.hash.clone(), + blob_id.class.clone(), part.offset_body + base_offset, part.offset_end + base_offset, part.encoding as u8, @@ -181,7 +182,8 @@ impl ToBodyPart for MessageMetadataContents<'_> { Property::BlobId if multipart.is_none() => { let base_offset = blob_id.start_offset(); BlobId::new_section( - blob_id.kind, + blob_id.hash.clone(), + blob_id.class.clone(), part.offset_body + base_offset, part.offset_end + base_offset, part.encoding as u8, diff --git a/crates/jmap/src/email/copy.rs b/crates/jmap/src/email/copy.rs index 95ad64b2..0d31b94e 100644 --- a/crates/jmap/src/email/copy.rs +++ b/crates/jmap/src/email/copy.rs @@ -49,7 +49,7 @@ use jmap_proto::{ use mail_parser::{parsers::fields::thread::thread_name, HeaderName, HeaderValue}; use store::{ write::{BatchBuilder, F_BITMAP, F_VALUE}, - BlobKind, StoreWrite, + BlobClass, }; use utils::map::vec_map::VecMap; @@ -354,40 +354,22 @@ impl JMAP { None }; - // Copy blob + // Assign id let message_id = self .assign_document_id(account_id, Collection::Email) .await?; let mut email = IngestedEmail { - blob_id: BlobId::new(BlobKind::LinkedMaildir { - account_id, - document_id: message_id, - }), + blob_id: BlobId::new( + metadata.blob_hash.clone(), + BlobClass::Linked { + account_id, + collection: Collection::Email.into(), + document_id: message_id, + }, + ), size: metadata.size, ..Default::default() }; - self.store - .copy_blob( - &BlobKind::LinkedMaildir { - account_id: from_account_id, - document_id: from_message_id, - }, - &email.blob_id.kind, - None, - ) - .await - .map_err(|err| { - tracing::error!( - event = "error", - context = "email_copy", - from_account_id = from_account_id, - from_message_id = from_message_id, - account_id = account_id, - message_id = message_id, - error = ?err, - "Failed to copy blob."); - MethodError::ServerPartialFail - })?; // Prepare batch let mut batch = BatchBuilder::new(); diff --git a/crates/jmap/src/email/get.rs b/crates/jmap/src/email/get.rs index 4daa60bf..c47e6b9d 100644 --- a/crates/jmap/src/email/get.rs +++ b/crates/jmap/src/email/get.rs @@ -37,6 +37,7 @@ use jmap_proto::{ }, }; use mail_parser::HeaderName; +use store::BlobClass; use crate::{auth::AccessToken, email::headers::HeaderToValue, Bincode, JMAP}; @@ -174,26 +175,21 @@ impl JMAP { }; // Retrieve raw message if needed - let blob_id = BlobId::maildir(account_id, id.document_id()); let raw_message = if needs_body || needs_headers { let offset = if !needs_body { - blob_id - .section - .as_ref() - .map(|s| s.offset_start as u32) - .unwrap_or(u32::MAX) + metadata.contents.parts[0].offset_body as u32 } else { u32::MAX }; - if let Some(raw_message) = self.get_blob(&blob_id.kind, 0..offset).await? { + if let Some(raw_message) = self.get_blob(&metadata.blob_hash, 0..offset).await? { raw_message } else { tracing::warn!(event = "not-found", account_id = account_id, collection = ?Collection::Email, document_id = id.document_id(), - blob_id = ?blob_id, + blob_id = ?metadata.blob_hash, "Blob not found"); response.not_found.push(id.into()); continue; @@ -201,6 +197,15 @@ impl JMAP { } else { vec![] }; + let blob_id = BlobId { + hash: metadata.blob_hash.clone(), + class: BlobClass::Linked { + account_id, + collection: Collection::Email.into(), + document_id: id.document_id(), + }, + section: None, + }; // Prepare response let mut email = Object::with_capacity(properties.len()); diff --git a/crates/jmap/src/email/index.rs b/crates/jmap/src/email/index.rs index 39725b72..db352ad0 100644 --- a/crates/jmap/src/email/index.rs +++ b/crates/jmap/src/email/index.rs @@ -31,7 +31,10 @@ use mail_parser::{ PartType, }; use nlp::language::Language; -use store::write::{BatchBuilder, IntoOperations, F_BITMAP, F_CLEAR, F_INDEX, F_VALUE}; +use store::{ + write::{BatchBuilder, BlobOp, IntoOperations, F_BITMAP, F_CLEAR, F_INDEX, F_VALUE}, + BlobHash, +}; use crate::{Bincode, NamedKey}; @@ -53,6 +56,7 @@ pub(super) trait IndexMessage { fn index_message( &mut self, message: Message, + blob_hash: BlobHash, keywords: Vec, mailbox_ids: Vec, received_at: u64, @@ -69,6 +73,7 @@ impl IndexMessage for BatchBuilder { fn index_message( &mut self, message: Message, + blob_hash: BlobHash, keywords: Vec, mailbox_ids: Vec, received_at: u64, @@ -142,6 +147,9 @@ impl IndexMessage for BatchBuilder { self.tag(Property::HasAttachment, (), 0); } + // Link blob + self.blob(blob_hash.clone(), BlobOp::Link, 0); + // Store message metadata self.value( Property::BodyStructure, @@ -151,6 +159,7 @@ impl IndexMessage for BatchBuilder { contents: message.into(), received_at, has_attachments, + blob_hash, }), F_VALUE, ); @@ -432,7 +441,12 @@ impl<'x> IntoOperations for EmailIndexBuilder<'x> { if metadata.has_attachments { batch.tag(Property::HasAttachment, (), options); } + + // Index headers batch.index_headers(&metadata.contents.parts[0].headers, options); + + // Link blob + batch.blob(metadata.blob_hash.clone(), BlobOp::Link, options); } } diff --git a/crates/jmap/src/email/ingest.rs b/crates/jmap/src/email/ingest.rs index ac873979..72f92136 100644 --- a/crates/jmap/src/email/ingest.rs +++ b/crates/jmap/src/email/ingest.rs @@ -35,12 +35,12 @@ use mail_parser::{ }; use store::{ ahash::AHashSet, - query::{filter::StoreQuery, Filter}, + query::Filter, write::{ log::ChangeLogBuilder, now, BatchBuilder, BitmapClass, TagValue, ValueClass, F_BITMAP, F_CLEAR, F_VALUE, }, - BitmapKey, StoreId, StoreRead, StoreWrite, ValueKey, + BitmapKey, BlobClass, ValueKey, }; use utils::map::vec_map::VecMap; @@ -250,9 +250,8 @@ impl JMAP { })?; // Store blob - let blob_id = BlobId::maildir(params.account_id, document_id); - self.store - .put_blob(&blob_id.kind, raw_message.as_ref()) + let blob_id = self + .put_blob(params.account_id, raw_message.as_ref(), false) .await .map_err(|err| { tracing::error!( @@ -303,6 +302,7 @@ impl JMAP { .create_document(document_id) .index_message( message, + blob_id.hash.clone(), params.keywords, params.mailbox_ids, params.received_at.unwrap_or_else(now), @@ -330,7 +330,15 @@ impl JMAP { Ok(IngestedEmail { id, change_id, - blob_id, + blob_id: BlobId { + hash: blob_id.hash, + class: BlobClass::Linked { + account_id: params.account_id, + collection: Collection::Email.into(), + document_id, + }, + section: blob_id.section, + }, size: raw_message_len as usize, }) } diff --git a/crates/jmap/src/email/metadata.rs b/crates/jmap/src/email/metadata.rs index 22f707af..21b5f39a 100644 --- a/crates/jmap/src/email/metadata.rs +++ b/crates/jmap/src/email/metadata.rs @@ -32,10 +32,12 @@ use mail_parser::{ MessagePartId, MimeHeaders, PartType, }; use serde::{Deserialize, Serialize}; +use store::BlobHash; #[derive(Serialize, Deserialize)] pub struct MessageMetadata<'x> { pub contents: MessageMetadataContents<'x>, + pub blob_hash: BlobHash, pub size: usize, pub received_at: u64, pub preview: String, diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index bed60f84..af367159 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -56,7 +56,7 @@ use store::{ assert::HashedValue, log::ChangeLogBuilder, BatchBuilder, DeserializeFrom, SerializeInto, ToBitmaps, ValueClass, F_BITMAP, F_CLEAR, F_VALUE, }, - BlobKind, Serialize, StoreWrite, + Serialize, }; use crate::{auth::AccessToken, Bincode, IngestError, JMAP}; @@ -1226,22 +1226,6 @@ impl JMAP { } } - // Delete blob - self.store - .delete_blob(&BlobKind::LinkedMaildir { - account_id, - document_id, - }) - .await - .map_err(|err| { - tracing::error!( - event = "error", - context = "email_delete", - error = ?err, - "Failed to delete blob."); - MethodError::ServerPartialFail - })?; - Ok(Ok(changes)) } } diff --git a/crates/jmap/src/email/snippet.rs b/crates/jmap/src/email/snippet.rs index 8884efb5..d14fab01 100644 --- a/crates/jmap/src/email/snippet.rs +++ b/crates/jmap/src/email/snippet.rs @@ -31,7 +31,7 @@ use jmap_proto::{ }; use mail_parser::{decoders::html::html_to_text, MessageParser, PartType}; use nlp::language::{stemmer::Stemmer, Language}; -use store::BlobKind; +use store::BlobHash; use crate::{auth::AccessToken, JMAP}; @@ -128,7 +128,7 @@ impl JMAP { self.get_term_index::(account_id, Collection::Email, document_id) .await?, self.get_blob( - &BlobKind::LinkedMaildir { + &BlobHash::LinkedMaildir { account_id, document_id, }, diff --git a/crates/jmap/src/lib.rs b/crates/jmap/src/lib.rs index 84492379..0808467f 100644 --- a/crates/jmap/src/lib.rs +++ b/crates/jmap/src/lib.rs @@ -48,17 +48,12 @@ use services::{ }; use smtp::core::SMTP; use store::{ - backend::sqlite::SqliteStore, + backend::{fs::FsStore, sqlite::SqliteStore}, parking_lot::Mutex, - query::{ - filter::StoreQuery, - sort::{Pagination, StoreSort}, - Comparator, Filter, ResultSet, SortedResultSet, - }, + query::{sort::Pagination, Comparator, Filter, ResultSet, SortedResultSet}, roaring::RoaringBitmap, write::{key::KeySerializer, BatchBuilder, BitmapClass, TagValue, ToBitmaps, ValueClass}, - BitmapKey, Deserialize, Key, Serialize, StoreId, StoreInit, StoreRead, StoreWrite, ValueKey, - SUBSPACE_VALUES, + BitmapKey, BlobStore, Deserialize, Key, Serialize, Store, ValueKey, SUBSPACE_VALUES, }; use tokio::sync::mpsc; use utils::{ @@ -88,7 +83,8 @@ pub mod websocket; pub const LONG_SLUMBER: Duration = Duration::from_secs(60 * 60 * 24); pub struct JMAP { - pub store: SqliteStore, + pub store: Store, + pub blob_store: Arc, pub config: Config, pub directory: Arc, @@ -201,9 +197,16 @@ impl JMAP { config.value_require("jmap.directory")? )) .clone(), - store: SqliteStore::open(config) - .await - .failed("Unable to open database"), + store: Store::SQLite(Arc::new( + SqliteStore::open(config) + .await + .failed("Unable to open database"), + )), + blob_store: Arc::new( + FsStore::open(config) + .await + .failed("Unable to open blob store"), + ), config: Config::new(config).failed("Invalid configuration file"), sessions: TtlDashMap::with_capacity( config.property("jmap.session.cache.size")?.unwrap_or(100), diff --git a/crates/jmap/src/mailbox/set.rs b/crates/jmap/src/mailbox/set.rs index 3218f718..9ab4412e 100644 --- a/crates/jmap/src/mailbox/set.rs +++ b/crates/jmap/src/mailbox/set.rs @@ -47,7 +47,6 @@ use store::{ query::Filter, roaring::RoaringBitmap, write::{assert::HashedValue, log::ChangeLogBuilder, BatchBuilder, F_BITMAP, F_CLEAR, F_VALUE}, - StoreWrite, }; use crate::{ diff --git a/crates/jmap/src/push/get.rs b/crates/jmap/src/push/get.rs index 93310560..ca228208 100644 --- a/crates/jmap/src/push/get.rs +++ b/crates/jmap/src/push/get.rs @@ -30,7 +30,7 @@ use jmap_proto::{ }; use store::{ write::{now, ValueClass}, - BitmapKey, StoreRead, ValueKey, + BitmapKey, ValueKey, }; use utils::map::bitmap::Bitmap; diff --git a/crates/jmap/src/services/housekeeper.rs b/crates/jmap/src/services/housekeeper.rs index d070c84a..8d5504dd 100644 --- a/crates/jmap/src/services/housekeeper.rs +++ b/crates/jmap/src/services/housekeeper.rs @@ -23,7 +23,6 @@ use std::{sync::Arc, time::Instant}; -use store::StorePurge; use tokio::sync::mpsc; use utils::{ config::{cron::SimpleCron, Config}, @@ -113,9 +112,9 @@ pub fn spawn_housekeeper(core: Arc, settings: &Config, mut rx: mpsc::Recei TASK_PURGE_BLOBS => { tracing::info!("Purging temporary blobs.",); if let Err(err) = - core.store.purge_tmp_blobs(core.config.upload_tmp_ttl).await + core.store.blob_hash_purge(core.blob_store.clone()).await { - tracing::error!("Error while purging bitmaps: {}", err); + tracing::error!("Error while purging blobs: {}", err); } } TASK_PURGE_SESSIONS => { diff --git a/crates/jmap/src/sieve/get.rs b/crates/jmap/src/sieve/get.rs index d33f5fcf..3b16acde 100644 --- a/crates/jmap/src/sieve/get.rs +++ b/crates/jmap/src/sieve/get.rs @@ -27,10 +27,14 @@ use jmap_proto::{ error::method::MethodError, method::get::{GetRequest, GetResponse, RequestArguments}, object::Object, - types::{blob::BlobId, collection::Collection, property::Property, value::Value}, + types::{collection::Collection, property::Property, value::Value}, }; use sieve::Sieve; -use store::{query::Filter, BlobKind, Deserialize, Serialize}; +use store::{ + query::Filter, + write::{assert::HashedValue, BatchBuilder, BlobOp, F_CLEAR}, + Deserialize, Serialize, +}; use crate::{sieve::SeenIds, Bincode, JMAP}; @@ -95,18 +99,7 @@ impl JMAP { Property::Id => { result.append(Property::Id, Value::Id(id)); } - Property::BlobId => { - if let Some(Value::UnsignedInt(blob_size)) = - push.properties.remove(&Property::Size) - { - result.append( - Property::BlobId, - BlobId::linked(account_id, Collection::SieveScript, document_id) - .with_section_size(blob_size as usize), - ); - } - } - Property::Name | Property::IsActive => { + Property::Name | Property::BlobId | Property::IsActive => { result.append(property.clone(), push.remove(property)); } property => { @@ -192,7 +185,7 @@ impl JMAP { ) -> Result<(Sieve, Object), MethodError> { // Obtain script object let script_object = self - .get_property::>( + .get_property::>>( account_id, Collection::SieveScript, document_id, @@ -212,32 +205,27 @@ impl JMAP { })?; // Obtain the sieve script length - let script_offset = script_object + let (script_offset, blob_id) = script_object + .inner .properties - .get(&Property::Size) - .and_then(|value| value.as_uint()) + .get(&Property::BlobId) + .and_then(|v| v.as_blob_id()) + .and_then(|v| (v.section.as_ref()?.size, v).into()) .ok_or_else(|| { tracing::warn!( context = "sieve_script_compile", event = "error", account_id = account_id, document_id = document_id, - "Failed to obtain sieve script offset" + "Failed to obtain sieve script blobId" ); MethodError::ServerPartialFail - })? as usize; + })?; // Obtain the sieve script blob let script_bytes = self - .get_blob( - &BlobKind::Linked { - account_id, - collection: Collection::SieveScript.into(), - document_id, - }, - 0..u32::MAX, - ) + .get_blob(&blob_id.hash, 0..u32::MAX) .await? .ok_or(MethodError::ServerPartialFail)?; @@ -246,7 +234,7 @@ impl JMAP { .get(script_offset..) .and_then(|bytes| Bincode::::deserialize(bytes).ok()) { - Ok((sieve.inner, script_object)) + Ok((sieve.inner, script_object.inner)) } else { // Deserialization failed, probably because the script compiler version changed match self @@ -270,18 +258,29 @@ impl JMAP { Vec::with_capacity(script_offset + compiled_bytes.len()); updated_sieve_bytes.extend_from_slice(&script_bytes[0..script_offset]); updated_sieve_bytes.extend_from_slice(&compiled_bytes); - let _ = self - .put_blob( - &BlobKind::Linked { - account_id, - collection: Collection::SieveScript.into(), - document_id, - }, - &updated_sieve_bytes, - ) - .await; - Ok((sieve.inner, script_object)) + // Store updated blob + let mut new_blob_id = blob_id.clone(); + new_blob_id.hash = self + .put_blob(account_id, &updated_sieve_bytes, false) + .await? + .hash; + let mut new_script_object = script_object.inner.clone(); + new_script_object.set(Property::BlobId, new_blob_id.clone()); + + // Update script object + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .with_collection(Collection::SieveScript) + .update_document(document_id) + .assert_value(Property::Value, &script_object) + .set(Property::Value, (&new_script_object).serialize()) + .blob(blob_id.hash.clone(), BlobOp::Link, F_CLEAR) + .blob(new_blob_id.hash, BlobOp::Link, 0); + self.write_batch(batch).await?; + + Ok((sieve.inner, new_script_object)) } Err(error) => { tracing::warn!( diff --git a/crates/jmap/src/sieve/set.rs b/crates/jmap/src/sieve/set.rs index 85f91d20..dbf00cdd 100644 --- a/crates/jmap/src/sieve/set.rs +++ b/crates/jmap/src/sieve/set.rs @@ -46,8 +46,8 @@ use sieve::compiler::ErrorType; use store::{ query::Filter, rand::{distributions::Alphanumeric, thread_rng, Rng}, - write::{assert::HashedValue, log::ChangeLogBuilder, BatchBuilder, F_CLEAR, F_VALUE}, - BlobKind, StoreWrite, + write::{assert::HashedValue, log::ChangeLogBuilder, BatchBuilder, BlobOp, F_CLEAR, F_VALUE}, + BlobClass, }; use crate::{auth::AccessToken, NamedKey, JMAP}; @@ -96,16 +96,22 @@ impl JMAP { for (id, object) in request.unwrap_create() { if sieve_ids.len() as usize <= self.config.sieve_max_scripts { match self.sieve_set_item(object, None, &ctx).await? { - Ok((builder, Some(blob))) => { + Ok((mut builder, Some(blob))) => { // Obtain document id let document_id = self .assign_document_id(account_id, Collection::SieveScript) .await?; // Store blob - let blob_id = - BlobId::linked(account_id, Collection::SieveScript, document_id); - self.put_blob(&blob_id.kind, &blob).await?; + let blob_id = builder.changes_mut().unwrap().blob_id_mut().unwrap(); + blob_id.hash = self.put_blob(account_id, &blob, false).await?.hash; + blob_id.class = BlobClass::Linked { + account_id, + collection: Collection::SieveScript.into(), + document_id, + }; + let script_size = blob_id.section.as_ref().unwrap().size; + let blob_id = blob_id.clone(); // Write record let mut batch = BatchBuilder::new(); @@ -113,10 +119,8 @@ impl JMAP { .with_account_id(account_id) .with_collection(Collection::SieveScript) .create_document(document_id) - .add( - NamedKey::Quota::<&[u8]>(account_id), - builder.changes().unwrap().script_size(), - ) + .add(NamedKey::Quota::<&[u8]>(account_id), script_size as i64) + .blob(blob_id.hash.clone(), BlobOp::Link, 0) .custom(builder); sieve_ids.insert(document_id); self.write_batch(batch).await?; @@ -127,10 +131,7 @@ impl JMAP { id, Object::with_capacity(1) .with_property(Property::Id, Value::Id(document_id.into())) - .with_property( - Property::BlobId, - blob_id.with_section_size(blob.len()), - ), + .with_property(Property::BlobId, blob_id), ); } Err(err) => { @@ -166,41 +167,72 @@ impl JMAP { ) .await? { - let prev_size = sieve.inner.script_size(); + let prev_blob_id = sieve + .inner + .blob_id() + .ok_or_else(|| { + tracing::warn!( + event = "error", + context = "sieve_set", + account_id = account_id, + document_id = document_id, + "Sieve does not contain a blobId." + ); + MethodError::ServerPartialFail + })? + .clone(); match self .sieve_set_item(object, (document_id, sieve).into(), &ctx) .await? { - Ok((builder, blob)) => { - // Store blob - let (update_quota, blob_id) = if let Some(blob) = blob { - let blob_id = - BlobId::linked(account_id, Collection::SieveScript, document_id); - self.put_blob(&blob_id.kind, &blob).await?; - let blob_size = builder.changes().unwrap().script_size(); - ( - match blob_size.cmp(&prev_size) { - std::cmp::Ordering::Greater => blob_size - prev_size, - std::cmp::Ordering::Less => -prev_size + blob_size, - std::cmp::Ordering::Equal => 0, - }, - Some(blob_id.with_section_size(blob.len())), - ) - } else { - (0, None) - }; - - // Write record + Ok((mut builder, blob)) => { + // Prepare write batch let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::SieveScript) - .update_document(document_id) - .custom(builder); - if update_quota != 0 { - batch.add(NamedKey::Quota::<&[u8]>(account_id), update_quota); - } + .update_document(document_id); + + let blob_id = if let Some(blob) = blob { + // Store blob + let blob_id = builder.changes_mut().unwrap().blob_id_mut().unwrap(); + blob_id.hash = self.put_blob(account_id, &blob, false).await?.hash; + blob_id.class = BlobClass::Linked { + account_id, + collection: Collection::SieveScript.into(), + document_id, + }; + let script_size = blob_id.section.as_ref().unwrap().size as i64; + let prev_script_size = + prev_blob_id.section.as_ref().unwrap().size as i64; + let blob_id = blob_id.clone(); + + // Update quota + let update_quota = match script_size.cmp(&prev_script_size) { + std::cmp::Ordering::Greater => script_size - prev_script_size, + std::cmp::Ordering::Less => -prev_script_size + script_size, + std::cmp::Ordering::Equal => 0, + }; + if update_quota != 0 { + batch.add(NamedKey::Quota::<&[u8]>(account_id), update_quota); + } + + // Update blobId + batch.blob(prev_blob_id.hash, BlobOp::Link, F_CLEAR).blob( + blob_id.hash.clone(), + BlobOp::Link, + 0, + ); + + blob_id.into() + } else { + None + }; + + // Write record + batch.custom(builder); + if !batch.is_empty() { changes.log_update(Collection::SieveScript, document_id); match self.store.write(batch.build()).await { @@ -344,24 +376,28 @@ impl JMAP { // Delete record let mut batch = BatchBuilder::new(); + let blob_id = obj.inner.blob_id().ok_or_else(|| { + tracing::warn!( + event = "error", + context = "sieve_script_delete", + account_id = account_id, + document_id = document_id, + "Sieve does not contain a blobId." + ); + MethodError::ServerPartialFail + })?; batch .with_account_id(account_id) .with_collection(Collection::SieveScript) .delete_document(document_id) .value(Property::EmailIds, (), F_VALUE | F_CLEAR) + .blob(blob_id.hash.clone(), BlobOp::Link, F_CLEAR) .add( NamedKey::Quota::<&[u8]>(account_id), - -(obj.inner.script_size()), + -(blob_id.section.as_ref().unwrap().size as i64), ) .custom(ObjectIndexBuilder::new(SCHEMA).with_current(obj)); self.write_batch(batch).await?; - let _ = self - .delete_blob(&BlobKind::Linked { - account_id, - collection: Collection::SieveScript.into(), - document_id, - }) - .await; Ok(true) } @@ -469,9 +505,7 @@ impl JMAP { let blob_update = if let Some(blob_id) = blob_id { if update.as_ref().map_or(true, |(document_id, _)| { - !blob_id - .kind - .is_document(ctx.account_id, Collection::SieveScript, *document_id) + !matches!(blob_id.class, BlobClass::Linked { account_id, collection, document_id: d } if account_id == ctx.account_id && collection == u8::from(Collection::SieveScript) && *document_id == d) }) { // Check access if let Some(mut bytes) = self.blob_download(&blob_id, ctx.access_token).await? { @@ -486,7 +520,7 @@ impl JMAP { // Compile script match self.sieve_compiler.compile(&bytes) { Ok(script) => { - changes.set(Property::Size, Value::UnsignedInt(bytes.len() as u64)); + changes.set(Property::BlobId, BlobId::default().with_section_size(bytes.len())); bytes.extend(bincode::serialize(&script).unwrap_or_default()); bytes.into() } @@ -626,15 +660,24 @@ impl JMAP { } } -pub trait ScriptSize { - fn script_size(&self) -> i64; +pub trait ObjectBlobId { + fn blob_id(&self) -> Option<&BlobId>; + fn blob_id_mut(&mut self) -> Option<&mut BlobId>; } -impl ScriptSize for Object { - fn script_size(&self) -> i64 { +impl ObjectBlobId for Object { + fn blob_id(&self) -> Option<&BlobId> { self.properties - .get(&Property::Size) - .and_then(|v| v.as_uint()) - .unwrap_or_default() as i64 + .get(&Property::BlobId) + .and_then(|v| v.as_blob_id()) + } + + fn blob_id_mut(&mut self) -> Option<&mut BlobId> { + self.properties + .get_mut(&Property::BlobId) + .and_then(|v| match v { + Value::BlobId(blob_id) => Some(blob_id), + _ => None, + }) } } diff --git a/crates/jmap/src/submission/set.rs b/crates/jmap/src/submission/set.rs index 3e9b28f3..ce9e63f9 100644 --- a/crates/jmap/src/submission/set.rs +++ b/crates/jmap/src/submission/set.rs @@ -53,10 +53,7 @@ use smtp::{ queue, }; use smtp_proto::{request::parser::Rfc5321Parser, MailFrom, RcptTo}; -use store::{ - write::{assert::HashedValue, log::ChangeLogBuilder, now, BatchBuilder}, - BlobKind, -}; +use store::write::{assert::HashedValue, log::ChangeLogBuilder, now, BatchBuilder}; use tokio::sync::oneshot; use utils::{listener::ServerInstance, map::vec_map::VecMap}; @@ -504,59 +501,62 @@ impl JMAP { } }; + // Obtain message metadata + let metadata = if let Some(metadata) = self + .get_property::>( + account_id, + Collection::Email, + email_id, + Property::BodyStructure, + ) + .await? + { + metadata.inner + } else { + return Ok(Err(SetError::invalid_properties() + .with_property(Property::EmailId) + .with_description("Email not found."))); + }; + // Add recipients to envelope if missing if rcpt_to.is_empty() { - if let Some(metadata) = self - .get_property::>( - account_id, - Collection::Email, - email_id, - Property::BodyStructure, - ) - .await? - { - let mut envelope_values = Vec::new(); - for header in &metadata.inner.contents.parts[0].headers { - if matches!( - header.name, - HeaderName::To | HeaderName::Cc | HeaderName::Bcc - ) { - if let HeaderValue::Address(addr) = &header.value { - for address in addr.iter() { - if let Some(address) = address.address().and_then(sanitize_email) { - if !rcpt_to.iter().any(|rcpt| rcpt.address == address) { - envelope_values.push(Value::Object( - Object::with_capacity(1) - .with_property(Property::Email, address.clone()), - )); - rcpt_to.push(RcptTo { - address, - ..Default::default() - }); - } + let mut envelope_values = Vec::new(); + for header in &metadata.contents.parts[0].headers { + if matches!( + header.name, + HeaderName::To | HeaderName::Cc | HeaderName::Bcc + ) { + if let HeaderValue::Address(addr) = &header.value { + for address in addr.iter() { + if let Some(address) = address.address().and_then(sanitize_email) { + if !rcpt_to.iter().any(|rcpt| rcpt.address == address) { + envelope_values.push(Value::Object( + Object::with_capacity(1) + .with_property(Property::Email, address.clone()), + )); + rcpt_to.push(RcptTo { + address, + ..Default::default() + }); } } } } } + } - if !rcpt_to.is_empty() { - submission - .properties - .get_mut_or_insert_with(Property::Envelope, || { - Value::Object(Object::with_capacity(1)) - }) - .as_obj_mut() - .unwrap() - .set(Property::RcptTo, Value::List(envelope_values)); - } else { - return Ok(Err(SetError::new(SetErrorType::NoRecipients) - .with_description("No recipients found in email."))); - } + if !rcpt_to.is_empty() { + submission + .properties + .get_mut_or_insert_with(Property::Envelope, || { + Value::Object(Object::with_capacity(1)) + }) + .as_obj_mut() + .unwrap() + .set(Property::RcptTo, Value::List(envelope_values)); } else { - return Ok(Err(SetError::invalid_properties() - .with_property(Property::EmailId) - .with_description("Email not found."))); + return Ok(Err(SetError::new(SetErrorType::NoRecipients) + .with_description("No recipients found in email."))); } } @@ -573,15 +573,7 @@ impl JMAP { ); // Obtain raw message - let message = if let Some(message) = self - .get_blob( - &BlobKind::LinkedMaildir { - account_id, - document_id: email_id, - }, - 0..u32::MAX, - ) - .await? + let message = if let Some(message) = self.get_blob(&metadata.blob_hash, 0..u32::MAX).await? { if message.len() > self.config.mail_max_size { return Ok(Err(SetError::new(SetErrorType::InvalidEmail) diff --git a/crates/jmap/src/thread/get.rs b/crates/jmap/src/thread/get.rs index 31a41958..5b8cc1b1 100644 --- a/crates/jmap/src/thread/get.rs +++ b/crates/jmap/src/thread/get.rs @@ -27,10 +27,7 @@ use jmap_proto::{ object::Object, types::{collection::Collection, id::Id, property::Property}, }; -use store::query::{ - sort::{Pagination, StoreSort}, - Comparator, ResultSet, -}; +use store::query::{sort::Pagination, Comparator, ResultSet}; use crate::JMAP; diff --git a/crates/jmap/src/vacation/set.rs b/crates/jmap/src/vacation/set.rs index ca7bb374..20ed9f17 100644 --- a/crates/jmap/src/vacation/set.rs +++ b/crates/jmap/src/vacation/set.rs @@ -32,6 +32,7 @@ use jmap_proto::{ object::{index::ObjectIndexBuilder, Object}, response::references::EvalObjectReferences, types::{ + blob::BlobId, collection::Collection, id::Id, property::Property, @@ -41,12 +42,12 @@ use jmap_proto::{ use mail_builder::MessageBuilder; use mail_parser::decoders::html::html_to_text; use store::{ - write::{assert::HashedValue, log::ChangeLogBuilder, BatchBuilder, F_CLEAR, F_VALUE}, - BlobKind, + write::{assert::HashedValue, log::ChangeLogBuilder, BatchBuilder, BlobOp, F_CLEAR, F_VALUE}, + BlobClass, }; use crate::{ - sieve::set::{ScriptSize, SCHEMA}, + sieve::set::{ObjectBlobId, SCHEMA}, NamedKey, JMAP, }; @@ -226,62 +227,76 @@ impl JMAP { }) .with_changes(changes); - // Create sieve script only if there are changes - let (update_quota, script_blob) = if build_script { - let script_blob = self.build_script(&mut obj)?; - let script_size = obj.changes().unwrap().script_size(); - - ( - if let Some(current) = obj.current() { - let current_script_size = current.inner.script_size(); - match script_size.cmp(¤t_script_size) { - std::cmp::Ordering::Greater => script_size - current_script_size, - std::cmp::Ordering::Less => -current_script_size + script_size, - std::cmp::Ordering::Equal => 0, - } - } else { - script_size - }, - Some(script_blob), - ) - } else { - (0, None) - }; - - // Write changes + // Update id let document_id = if let Some(document_id) = document_id { batch .update_document(document_id) - .value(Property::EmailIds, (), F_VALUE | F_CLEAR) - .custom(obj); + .value(Property::EmailIds, (), F_VALUE | F_CLEAR); change_log.log_insert(Collection::SieveScript, document_id); document_id } else { let document_id = self .assign_document_id(account_id, Collection::SieveScript) .await?; - batch.create_document(document_id).custom(obj); + batch.create_document(document_id); change_log.log_update(Collection::SieveScript, document_id); document_id }; - if !batch.is_empty() { - if update_quota != 0 { - batch.add(NamedKey::Quota::<&[u8]>(account_id), update_quota); - } - self.write_batch(batch).await?; - } - // Write blob - if let Some(script_blob) = script_blob { - self.put_blob( - &BlobKind::Linked { - account_id, - collection: Collection::SieveScript.into(), - document_id, - }, - &script_blob, - ) - .await?; + // Create sieve script only if there are changes + if build_script { + // Upload new blob + let hash = self + .put_blob(account_id, &self.build_script(&mut obj)?, false) + .await? + .hash; + let blob_id = obj.changes_mut().unwrap().blob_id_mut().unwrap(); + blob_id.hash = hash; + blob_id.class = BlobClass::Linked { + account_id, + collection: Collection::SieveScript.into(), + document_id, + }; + + // Link blob + batch.blob(blob_id.hash.clone(), BlobOp::Link, 0); + + let script_size = blob_id.section.as_ref().unwrap().size as i64; + + if let Some(current) = obj.current() { + let current_blob_id = current.inner.blob_id().ok_or_else(|| { + tracing::warn!( + event = "error", + context = "vacation_response_set", + account_id = account_id, + document_id = document_id, + "Sieve object does not contain a blobId." + ); + MethodError::ServerPartialFail + })?; + + // Unlink previous blob + batch.blob(current_blob_id.hash.clone(), BlobOp::Link, F_CLEAR); + + // Update quota + let current_script_size = current_blob_id.section.as_ref().unwrap().size as i64; + let quota = match script_size.cmp(¤t_script_size) { + std::cmp::Ordering::Greater => script_size - current_script_size, + std::cmp::Ordering::Less => -current_script_size + script_size, + std::cmp::Ordering::Equal => 0, + }; + if quota != 0 { + batch.add(NamedKey::Quota::<&[u8]>(account_id), quota); + } + } else { + batch.add(NamedKey::Quota::<&[u8]>(account_id), script_size); + } + }; + + // Write changes + batch.custom(obj); + if !batch.is_empty() { + self.write_batch(batch).await?; } // Deactivate other sieve scripts @@ -414,7 +429,10 @@ impl JMAP { match self.sieve_compiler.compile(&script) { Ok(compiled_script) => { // Update blob length - obj.set(Property::Size, Value::UnsignedInt(script.len() as u64)); + obj.set( + Property::BlobId, + BlobId::default().with_section_size(script.len()).into(), + ); // Serialize script script.extend(bincode::serialize(&compiled_script).unwrap_or_default()); diff --git a/crates/managesieve/src/core/client.rs b/crates/managesieve/src/core/client.rs index 76794e43..04471d6a 100644 --- a/crates/managesieve/src/core/client.rs +++ b/crates/managesieve/src/core/client.rs @@ -24,7 +24,7 @@ use imap::core::IMAP; use imap_proto::receiver::{self, Request}; use jmap_proto::types::{collection::Collection, property::Property}; -use store::query::{filter::StoreQuery, Filter}; +use store::query::Filter; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use super::{Command, IsTls, ResponseCode, ResponseType, Session, State, StatusResponse}; diff --git a/crates/managesieve/src/op/getscript.rs b/crates/managesieve/src/op/getscript.rs index ff14b047..b0c65f31 100644 --- a/crates/managesieve/src/op/getscript.rs +++ b/crates/managesieve/src/op/getscript.rs @@ -22,11 +22,11 @@ */ use imap_proto::receiver::Request; +use jmap::sieve::set::ObjectBlobId; use jmap_proto::{ object::Object, types::{collection::Collection, property::Property, value::Value}, }; -use store::BlobKind; use tokio::io::{AsyncRead, AsyncWrite}; use crate::core::{Command, ResponseCode, Session, StatusResponse}; @@ -41,7 +41,7 @@ impl Session { .ok_or_else(|| StatusResponse::no("Expected script name as a parameter."))?; let account_id = self.state.access_token().primary_id(); let document_id = self.get_script_id(account_id, &name).await?; - let script_size = self + let (blob_section, blob_hash) = self .jmap .get_property::>( account_id, @@ -53,30 +53,23 @@ impl Session { .ok_or_else(|| { StatusResponse::no("Script not found").with_code(ResponseCode::NonExistent) })? - .remove(&Property::Size) - .try_unwrap_uint() + .blob_id() + .and_then(|id| (id.section.as_ref()?.clone(), id.hash.clone()).into()) .ok_or_else(|| { - StatusResponse::no("Filed to retrieve blob size").with_code(ResponseCode::TryLater) - })? as u32; + StatusResponse::no("Filed to retrieve blobId").with_code(ResponseCode::TryLater) + })?; let script = self .jmap - .get_blob( - &BlobKind::Linked { - account_id, - collection: Collection::SieveScript.into(), - document_id, - }, - 0..script_size, - ) + .get_blob_section(&blob_hash, &blob_section) .await? .ok_or_else(|| { StatusResponse::no("Script blob not found").with_code(ResponseCode::NonExistent) })?; - debug_assert_eq!(script.len() as u32, script_size); + debug_assert_eq!(script.len(), blob_section.size); let mut response = Vec::with_capacity(script.len() + 30); response.push(b'{'); - response.extend_from_slice(script_size.to_string().as_bytes()); + response.extend_from_slice(blob_section.size.to_string().as_bytes()); response.extend_from_slice(b"}\r\n"); response.extend(script); diff --git a/crates/managesieve/src/op/putscript.rs b/crates/managesieve/src/op/putscript.rs index b271aab5..87af5b4a 100644 --- a/crates/managesieve/src/op/putscript.rs +++ b/crates/managesieve/src/op/putscript.rs @@ -22,7 +22,10 @@ */ use imap_proto::receiver::Request; -use jmap::sieve::set::SCHEMA; +use jmap::{ + sieve::set::{ObjectBlobId, SCHEMA}, + NamedKey, +}; use jmap_proto::{ object::{index::ObjectIndexBuilder, Object}, types::{blob::BlobId, collection::Collection, property::Property, value::Value}, @@ -30,7 +33,8 @@ use jmap_proto::{ use sieve::compiler::ErrorType; use store::{ query::Filter, - write::{assert::HashedValue, BatchBuilder}, + write::{assert::HashedValue, BatchBuilder, BlobOp, F_CLEAR}, + BlobClass, }; use tokio::io::{AsyncRead, AsyncWrite}; @@ -45,17 +49,17 @@ impl Session { .ok_or_else(|| StatusResponse::no("Expected script name as a parameter."))? .trim() .to_string(); - let mut script = tokens + let mut script_bytes = tokens .next() .ok_or_else(|| StatusResponse::no("Expected script as a parameter."))? .unwrap_bytes(); - let script_len = script.len() as u64; + let script_size = script_bytes.len() as i64; // Check quota let access_token = self.state.access_token(); let account_id = access_token.primary_id(); if access_token.quota > 0 - && script.len() as i64 + self.jmap.get_used_quota(account_id).await? + && script_bytes.len() as i64 + self.jmap.get_used_quota(account_id).await? > access_token.quota as i64 { return Err(StatusResponse::no("Quota exceeded.").with_code(ResponseCode::Quota)); @@ -74,9 +78,9 @@ impl Session { } // Compile script - match self.jmap.sieve_compiler.compile(&script) { + match self.jmap.sieve_compiler.compile(&script_bytes) { Ok(compiled_script) => { - script.extend(bincode::serialize(&compiled_script).unwrap_or_default()); + script_bytes.extend(bincode::serialize(&compiled_script).unwrap_or_default()); } Err(err) => { return Err(if let ErrorType::ScriptTooLong = &err.error_type() { @@ -89,14 +93,6 @@ impl Session { // Validate name if let Some(document_id) = self.validate_name(account_id, &name).await? { - // Update blob - self.jmap - .put_blob( - &BlobId::linked(account_id, Collection::SieveScript, document_id).kind, - &script, - ) - .await?; - // Obtain script values let script = self .jmap @@ -110,6 +106,24 @@ impl Session { .ok_or_else(|| { StatusResponse::no("Script not found").with_code(ResponseCode::NonExistent) })?; + let prev_blob_id = script.inner.blob_id().ok_or_else(|| { + StatusResponse::no("Internal error while obtaining blobId") + .with_code(ResponseCode::TryLater) + })?; + + // Write script blob + let blob_id = BlobId::new( + self.jmap + .put_blob(account_id, &script_bytes, false) + .await? + .hash, + BlobClass::Linked { + account_id, + collection: Collection::SieveScript.into(), + document_id, + }, + ) + .with_section_size(script_size as usize); // Write record let mut batch = BatchBuilder::new(); @@ -117,14 +131,28 @@ impl Session { .with_account_id(account_id) .with_collection(Collection::SieveScript) .update_document(document_id) - .custom( - ObjectIndexBuilder::new(SCHEMA) - .with_current(script) - .with_changes( - Object::with_capacity(1) - .with_property(Property::Size, Value::UnsignedInt(script_len)), - ), - ); + .blob(prev_blob_id.hash.clone(), BlobOp::Link, F_CLEAR) + .blob(blob_id.hash.clone(), BlobOp::Link, 0); + + // Update quota + let prev_script_size = prev_blob_id.section.as_ref().unwrap().size as i64; + let update_quota = match script_size.cmp(&prev_script_size) { + std::cmp::Ordering::Greater => script_size - prev_script_size, + std::cmp::Ordering::Less => -prev_script_size + script_size, + std::cmp::Ordering::Equal => 0, + }; + if update_quota != 0 { + batch.add(NamedKey::Quota::<&[u8]>(account_id), update_quota); + } + + batch.custom( + ObjectIndexBuilder::new(SCHEMA) + .with_current(script) + .with_changes( + Object::with_capacity(1) + .with_property(Property::BlobId, Value::BlobId(blob_id)), + ), + ); self.jmap.write_batch(batch).await?; } else { // Obtain document id @@ -133,13 +161,19 @@ impl Session { .assign_document_id(account_id, Collection::SieveScript) .await?; - // Store blob - self.jmap - .put_blob( - &BlobId::linked(account_id, Collection::SieveScript, document_id).kind, - &script, - ) - .await?; + // Write script blob + let blob_id = BlobId::new( + self.jmap + .put_blob(account_id, &script_bytes, false) + .await? + .hash, + BlobClass::Linked { + account_id, + collection: Collection::SieveScript.into(), + document_id, + }, + ) + .with_section_size(script_size as usize); // Write record let mut changelog = self.jmap.begin_changes(account_id).await?; @@ -149,12 +183,14 @@ impl Session { .with_account_id(account_id) .with_collection(Collection::SieveScript) .create_document(document_id) + .add(NamedKey::Quota::<&[u8]>(account_id), script_size) + .blob(blob_id.hash.clone(), BlobOp::Link, 0) .custom( ObjectIndexBuilder::new(SCHEMA).with_changes( Object::with_capacity(3) .with_property(Property::Name, name) .with_property(Property::IsActive, Value::Bool(false)) - .with_property(Property::Size, Value::UnsignedInt(script_len)), + .with_property(Property::BlobId, Value::BlobId(blob_id)), ), ) .custom(changelog); diff --git a/crates/store/src/backend/foundationdb/id_assign.rs b/crates/store/src/backend/foundationdb/id_assign.rs index c22571f1..8f8961ae 100644 --- a/crates/store/src/backend/foundationdb/id_assign.rs +++ b/crates/store/src/backend/foundationdb/id_assign.rs @@ -21,7 +21,7 @@ * for more details. */ -use crate::{write::key::DeserializeBigEndian, Deserialize, Key, Serialize}; +use crate::{write::key::DeserializeBigEndian, Deserialize, Key, Serialize, U32_LEN}; use ahash::AHashSet; use foundationdb::{options::StreamingMode, FdbError, KeySelector, RangeOption}; use futures::StreamExt; @@ -30,7 +30,7 @@ use std::time::Instant; use crate::{ write::{key::KeySerializer, now}, - BitmapKey, IndexKey, StoreId, SUBSPACE_VALUES, + BitmapKey, IndexKey, SUBSPACE_VALUES, }; use super::{ @@ -39,9 +39,8 @@ use super::{ FdbStore, }; -#[async_trait::async_trait] -impl StoreId for FdbStore { - async fn assign_document_id( +impl FdbStore { + pub(crate) async fn assign_document_id( &self, account_id: u32, collection: impl Into + Sync + Send, @@ -91,8 +90,7 @@ impl StoreId for FdbStore { while let Some(values) = values.next().await { for value in values? { let key = value.key(); - let document_id = - key.deserialize_be_u32(key.len() - std::mem::size_of::())?; + let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; if u64::deserialize(value.value())? <= expired_timestamp { // Found an expired id, reuse it expired_ids.push(document_id); @@ -135,7 +133,7 @@ impl StoreId for FdbStore { let key = value.key(); if let Some(next_id) = next_available_index( value.value(), - key.deserialize_be_u32(key.len() - std::mem::size_of::())?, + key.deserialize_be_u32(key.len() - U32_LEN)?, &reserved_ids, ) { document_id = next_id; @@ -185,9 +183,9 @@ impl StoreId for FdbStore { } } - async fn assign_change_id(&self, account_id: u32) -> crate::Result { + pub(crate) async fn assign_change_id(&self, account_id: u32) -> crate::Result { let start = Instant::now(); - let counter = KeySerializer::new(std::mem::size_of::() + 2) + let counter = KeySerializer::new(U32_LEN + 2) .write(SUBSPACE_VALUES) .write(account_id) .finalize(); diff --git a/crates/store/src/backend/foundationdb/main.rs b/crates/store/src/backend/foundationdb/main.rs index 90d59c2d..af7e02e6 100644 --- a/crates/store/src/backend/foundationdb/main.rs +++ b/crates/store/src/backend/foundationdb/main.rs @@ -24,17 +24,13 @@ use foundationdb::Database; use utils::config::Config; -use crate::{blob::BlobStore, StoreInit}; - use super::FdbStore; -#[async_trait::async_trait] -impl StoreInit for FdbStore { - async fn open(config: &Config) -> crate::Result { +impl FdbStore { + pub async fn open(_: &Config) -> crate::Result { Ok(Self { guard: unsafe { foundationdb::boot() }, db: Database::default()?, - blob: BlobStore::new(config).await?, }) } } diff --git a/crates/store/src/backend/foundationdb/mod.rs b/crates/store/src/backend/foundationdb/mod.rs index 103ee646..aa051600 100644 --- a/crates/store/src/backend/foundationdb/mod.rs +++ b/crates/store/src/backend/foundationdb/mod.rs @@ -23,7 +23,7 @@ use foundationdb::{api::NetworkAutoStop, Database, FdbError}; -use crate::{blob::BlobStore, Error}; +use crate::Error; pub mod bitmap; pub mod id_assign; @@ -36,7 +36,6 @@ pub mod write; pub struct FdbStore { db: Database, guard: NetworkAutoStop, - blob: BlobStore, } impl From for Error { diff --git a/crates/store/src/backend/foundationdb/purge.rs b/crates/store/src/backend/foundationdb/purge.rs index 58ff3a1f..c833aa5c 100644 --- a/crates/store/src/backend/foundationdb/purge.rs +++ b/crates/store/src/backend/foundationdb/purge.rs @@ -28,17 +28,16 @@ use foundationdb::{ use futures::StreamExt; use crate::{ - write::key::KeySerializer, StorePurge, SUBSPACE_BITMAPS, SUBSPACE_INDEXES, SUBSPACE_LOGS, - SUBSPACE_QUOTAS, SUBSPACE_VALUES, + write::key::KeySerializer, SUBSPACE_BITMAPS, SUBSPACE_INDEXES, SUBSPACE_LOGS, SUBSPACE_VALUES, + U32_LEN, }; use super::{bitmap::DenseBitmap, FdbStore}; const MAX_COMMIT_ATTEMPTS: u8 = 25; -#[async_trait::async_trait] -impl StorePurge for FdbStore { - async fn purge_bitmaps(&self) -> crate::Result<()> { +impl FdbStore { + pub(crate) async fn purge_bitmaps(&self) -> crate::Result<()> { // Obtain all empty bitmaps let trx = self.db.create_trx()?; let mut iter = trx.get_ranges( @@ -92,19 +91,19 @@ impl StorePurge for FdbStore { Ok(()) } - async fn purge_account(&self, account_id: u32) -> crate::Result<()> { + pub(crate) async fn purge_account(&self, account_id: u32) -> crate::Result<()> { for subspace in [ SUBSPACE_BITMAPS, SUBSPACE_VALUES, SUBSPACE_LOGS, SUBSPACE_INDEXES, ] { - let from_key = KeySerializer::new(std::mem::size_of::() + 2) + let from_key = KeySerializer::new(U32_LEN + 2) .write(subspace) .write(account_id) .write(0u8) .finalize(); - let to_key = KeySerializer::new(std::mem::size_of::() + 2) + let to_key = KeySerializer::new(U32_LEN + 2) .write(subspace) .write(account_id) .write(u8::MAX) @@ -117,18 +116,6 @@ impl StorePurge for FdbStore { } } - // Delete quota key - let trx = self.db.create_trx()?; - trx.clear( - &KeySerializer::new(5) - .write(SUBSPACE_QUOTAS) - .write(account_id) - .finalize(), - ); - if let Err(err) = trx.commit().await { - return Err(FdbError::from(err).into()); - } - Ok(()) } } diff --git a/crates/store/src/backend/foundationdb/read.rs b/crates/store/src/backend/foundationdb/read.rs index 033baf22..ab54cd59 100644 --- a/crates/store/src/backend/foundationdb/read.rs +++ b/crates/store/src/backend/foundationdb/read.rs @@ -34,15 +34,14 @@ use crate::{ key::{DeserializeBigEndian, KeySerializer}, BitmapClass, ValueClass, }, - BitmapKey, Deserialize, IndexKey, IndexKeyPrefix, Key, LogKey, StoreRead, ValueKey, - SUBSPACE_INDEXES, SUBSPACE_QUOTAS, + BitmapKey, Deserialize, IndexKey, IndexKeyPrefix, IterateParams, Key, ValueKey, SUBSPACE_BLOBS, + SUBSPACE_INDEXES, U32_LEN, }; use super::{bitmap::DeserializeBlock, FdbStore}; -#[async_trait::async_trait] -impl StoreRead for FdbStore { - async fn get_value(&self, key: impl Key) -> crate::Result> +impl FdbStore { + pub(crate) async fn get_value(&self, key: impl Key) -> crate::Result> where U: Deserialize, { @@ -56,7 +55,7 @@ impl StoreRead for FdbStore { } } - async fn get_bitmap( + pub(crate) async fn get_bitmap( &self, mut key: BitmapKey, ) -> crate::Result> { @@ -83,7 +82,7 @@ impl StoreRead for FdbStore { if key.len() == key_len { bm.deserialize_block( value.value(), - key.deserialize_be_u32(key.len() - std::mem::size_of::())?, + key.deserialize_be_u32(key.len() - U32_LEN)?, ); } } @@ -91,7 +90,7 @@ impl StoreRead for FdbStore { Ok(if !bm.is_empty() { Some(bm) } else { None }) } - async fn range_to_bitmap( + pub(crate) async fn range_to_bitmap( &self, account_id: u32, collection: u8, @@ -99,20 +98,20 @@ impl StoreRead for FdbStore { value: Vec, op: query::Operator, ) -> crate::Result> { - let k1 = KeySerializer::new( - std::mem::size_of::>() + value.len() + 1 + std::mem::size_of::(), - ) - .write(SUBSPACE_INDEXES) - .write(account_id) - .write(collection) - .write(field); - let k2 = KeySerializer::new( - std::mem::size_of::>() + value.len() + 1 + std::mem::size_of::(), - ) - .write(SUBSPACE_INDEXES) - .write(account_id) - .write(collection) - .write(field + matches!(op, Operator::GreaterThan | Operator::GreaterEqualThan) as u8); + let k1 = + KeySerializer::new(std::mem::size_of::>() + value.len() + 1 + U32_LEN) + .write(SUBSPACE_INDEXES) + .write(account_id) + .write(collection) + .write(field); + let k2 = + KeySerializer::new(std::mem::size_of::>() + value.len() + 1 + U32_LEN) + .write(SUBSPACE_INDEXES) + .write(account_id) + .write(collection) + .write( + field + matches!(op, Operator::GreaterThan | Operator::GreaterEqualThan) as u8, + ); let (begin, end) = match op { Operator::LowerThan => ( @@ -158,7 +157,7 @@ impl StoreRead for FdbStore { while let Some(values) = range_stream.next().await { for value in values? { let key = value.key(); - bm.insert(key.deserialize_be_u32(key.len() - std::mem::size_of::())?); + bm.insert(key.deserialize_be_u32(key.len() - U32_LEN)?); } } } else { @@ -166,7 +165,7 @@ impl StoreRead for FdbStore { for value in values? { let key = value.key(); if key.len() == key_len { - bm.insert(key.deserialize_be_u32(key.len() - std::mem::size_of::())?); + bm.insert(key.deserialize_be_u32(key.len() - U32_LEN)?); } } } @@ -175,7 +174,7 @@ impl StoreRead for FdbStore { Ok(Some(bm)) } - async fn sort_index( + pub(crate) async fn sort_index( &self, account_id: u32, collection: impl Into + Sync + Send, @@ -214,7 +213,7 @@ impl StoreRead for FdbStore { while let Some(values) = sorted_iter.next().await { for value in values? { let key = value.key(); - let id_pos = key.len() - std::mem::size_of::(); + let id_pos = key.len() - U32_LEN; debug_assert!(key.starts_with(&from_key)); if !cb( key.get(prefix_len..id_pos).ok_or_else(|| { @@ -230,28 +229,25 @@ impl StoreRead for FdbStore { Ok(()) } - async fn iterate( + pub(crate) async fn iterate( &self, - begin: impl Key, - end: impl Key, - first: bool, - ascending: bool, + params: IterateParams, mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> crate::Result + Sync + Send, ) -> crate::Result<()> { - let begin = begin.serialize(true); - let end = end.serialize(true); + let begin = params.begin.serialize(true); + let end = params.end.serialize(true); let trx = self.db.create_trx()?; let mut iter = trx.get_ranges( RangeOption { begin: KeySelector::first_greater_or_equal(&begin), end: KeySelector::first_greater_than(&end), - mode: if first { + mode: if params.first { options::StreamingMode::Small } else { options::StreamingMode::Iterator }, - reverse: !ascending, + reverse: !params.ascending, ..Default::default() }, true, @@ -262,7 +258,7 @@ impl StoreRead for FdbStore { let key = value.key().get(1..).unwrap_or_default(); let value = value.value(); - if !cb(key, value)? || first { + if !cb(key, value)? || params.first { return Ok(()); } } @@ -271,51 +267,7 @@ impl StoreRead for FdbStore { Ok(()) } - async fn get_last_change_id( - &self, - account_id: u32, - collection: impl Into + Sync + Send, - ) -> crate::Result> { - let collection = collection.into(); - let from_key = LogKey { - account_id, - collection, - change_id: 0, - } - .serialize(true); - let to_key = LogKey { - account_id, - collection, - change_id: u64::MAX, - } - .serialize(true); - - let trx = self.db.create_trx()?; - let mut iter = trx.get_ranges( - RangeOption { - begin: KeySelector::first_greater_or_equal(&from_key), - end: KeySelector::first_greater_or_equal(&to_key), - mode: options::StreamingMode::Small, - reverse: true, - ..Default::default() - }, - true, - ); - - while let Some(values) = iter.next().await { - if let Some(value) = (values?).into_iter().next() { - let key = value.key(); - - return key - .deserialize_be_u64(key.len() - std::mem::size_of::()) - .map(Some); - } - } - - Ok(None) - } - - async fn get_counter( + pub(crate) async fn get_counter( &self, key: impl Into> + Sync + Send, ) -> crate::Result { @@ -330,11 +282,8 @@ impl StoreRead for FdbStore { } #[cfg(feature = "test_mode")] - async fn assert_is_empty(&self) { - use crate::{StorePurge, SUBSPACE_BITMAPS, SUBSPACE_LOGS, SUBSPACE_VALUES}; - - // Purge bitmaps - self.purge_bitmaps().await.unwrap(); + pub(crate) async fn assert_is_empty(&self) { + use crate::{SUBSPACE_ACLS, SUBSPACE_BITMAPS, SUBSPACE_LOGS, SUBSPACE_VALUES}; let conn = self.db.create_trx().unwrap(); @@ -399,12 +348,11 @@ impl StoreRead for FdbStore { ); } } - SUBSPACE_QUOTAS => { - let v = i64::from_le_bytes(value[..].try_into().unwrap()); - if v != 0 { - let k = u32::from_be_bytes(key[1..].try_into().unwrap()); - panic!("Table quotas is not empty: {k:?} = {v:?} (key {key:?})"); - } + SUBSPACE_BLOBS | SUBSPACE_ACLS => { + panic!( + "Subspace {:?} is not empty: {key:?} {value:?}", + char::from(subspace) + ); } SUBSPACE_LOGS => { delete_keys.push(key.to_vec()); diff --git a/crates/store/src/backend/foundationdb/write.rs b/crates/store/src/backend/foundationdb/write.rs index 1c29fc86..92c1a447 100644 --- a/crates/store/src/backend/foundationdb/write.rs +++ b/crates/store/src/backend/foundationdb/write.rs @@ -28,7 +28,7 @@ use foundationdb::{options::MutationType, FdbError}; use crate::{ write::{Batch, Operation, ValueOp}, - BitmapKey, IndexKey, Key, LogKey, StoreWrite, ValueKey, + BitmapKey, BlobKey, IndexKey, Key, LogKey, ValueKey, }; use super::{bitmap::DenseBitmap, FdbStore}; @@ -54,9 +54,8 @@ pub static ref BITMAPS: std::sync::Arc crate::Result<()> { +impl FdbStore { + pub(crate) async fn write(&self, batch: Batch) -> crate::Result<()> { let start = Instant::now(); let mut retry_count = 0; let mut set_bitmaps = AHashMap::new(); @@ -151,6 +150,22 @@ impl StoreWrite for FdbStore { .set(document_id); } } + Operation::Blob { hash, op, set } => { + let key = BlobKey { + account_id, + collection, + document_id, + hash, + op: *op, + } + .serialize(true); + + if *set { + trx.set(&key, &[]); + } else { + trx.clear(&key); + } + } Operation::Log { collection, change_id, @@ -271,7 +286,7 @@ impl StoreWrite for FdbStore { } #[cfg(feature = "test_mode")] - async fn destroy(&self) { + pub(crate) async fn destroy(&self) { let trx = self.db.create_trx().unwrap(); trx.clear_range(&[0u8], &[u8::MAX]); trx.commit().await.unwrap(); diff --git a/crates/store/src/backend/fs/mod.rs b/crates/store/src/backend/fs/mod.rs new file mode 100644 index 00000000..61e671b6 --- /dev/null +++ b/crates/store/src/backend/fs/mod.rs @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2023, Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{io::SeekFrom, ops::Range, path::PathBuf}; + +use tokio::{ + fs::{self, File}, + io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}, +}; +use utils::{codec::base32_custom::Base32Writer, config::Config}; + +use crate::BlobStore; + +pub struct FsStore { + path: PathBuf, + hash_levels: usize, +} + +impl FsStore { + pub async fn open(config: &Config) -> crate::Result { + let path = config.property_require::("store.blob.local.path")?; + if path.exists() { + Ok(FsStore { + path, + hash_levels: std::cmp::min( + config.property_or_static("store.blob.local.depth", "2")?, + 5, + ), + }) + } else { + Err(crate::Error::InternalError(format!( + "Blob store path {:?} does not exist", + path + ))) + } + } +} + +#[async_trait::async_trait] +impl BlobStore for FsStore { + async fn get_blob(&self, key: &[u8], range: Range) -> crate::Result>> { + let blob_path = self.build_path(key); + let blob_size = match fs::metadata(&blob_path).await { + Ok(m) => m.len(), + Err(_) => return Ok(None), + }; + let mut blob = File::open(&blob_path).await?; + + Ok(Some(if range.start != 0 || range.end != u32::MAX { + let from_offset = if range.start < blob_size as u32 { + range.start + } else { + 0 + }; + let mut buf = + vec![0; (std::cmp::min(range.end, blob_size as u32) - from_offset) as usize]; + + if from_offset > 0 { + blob.seek(SeekFrom::Start(from_offset as u64)).await?; + } + blob.read_exact(&mut buf).await?; + buf + } else { + let mut buf = Vec::with_capacity(blob_size as usize); + blob.read_to_end(&mut buf).await?; + buf + })) + } + + async fn put_blob(&self, key: &[u8], data: &[u8]) -> crate::Result<()> { + let blob_path = self.build_path(key); + + if fs::metadata(&blob_path) + .await + .map_or(true, |m| m.len() as usize != data.len()) + { + fs::create_dir_all(blob_path.parent().unwrap()).await?; + let mut blob_file = File::create(&blob_path).await?; + blob_file.write_all(data).await?; + blob_file.flush().await?; + } + + Ok(()) + } + + async fn delete_blob(&self, key: &[u8]) -> crate::Result { + let blob_path = self.build_path(key); + if blob_path.exists() { + fs::remove_file(&blob_path).await?; + Ok(true) + } else { + Ok(false) + } + } +} + +impl FsStore { + fn build_path(&self, key: &[u8]) -> PathBuf { + let mut path = self.path.clone(); + + for byte in key.iter().take(self.hash_levels) { + path.push(format!("{:x}", byte)); + } + path.push(Base32Writer::from_bytes(key).finalize()); + path + } +} diff --git a/crates/store/src/backend/mod.rs b/crates/store/src/backend/mod.rs index 6bc01dd7..bac9a2b8 100644 --- a/crates/store/src/backend/mod.rs +++ b/crates/store/src/backend/mod.rs @@ -23,10 +23,18 @@ #[cfg(feature = "foundation")] pub mod foundationdb; +pub mod fs; #[cfg(feature = "rocks")] pub mod rocksdb; +pub mod s3; #[cfg(feature = "sqlite")] pub mod sqlite; pub(crate) const MAX_TOKEN_LENGTH: usize = (u8::MAX >> 2) as usize; pub(crate) const MAX_TOKEN_MASK: usize = MAX_TOKEN_LENGTH - 1; + +impl From for crate::Error { + fn from(err: std::io::Error) -> Self { + Self::InternalError(format!("IO error: {}", err)) + } +} diff --git a/crates/store/src/backend/rocksdb/bitmap.rs b/crates/store/src/backend/rocksdb/bitmap.rs index 194522b4..c88aafcf 100644 --- a/crates/store/src/backend/rocksdb/bitmap.rs +++ b/crates/store/src/backend/rocksdb/bitmap.rs @@ -87,7 +87,7 @@ macro_rules! impl_bit { ($single:ident, $many:ident, $flag:ident) => { #[inline(always)] pub fn $single(document: u32) -> Vec { - let mut buf = Vec::with_capacity(std::mem::size_of::() + 2); + let mut buf = Vec::with_capacity(U32_LEN + 2); buf.push(IS_BITLIST); buf.push($flag); buf.push_leb128(document); @@ -102,7 +102,7 @@ macro_rules! impl_bit { debug_assert!(documents.size_hint().0 > 0); let mut buf = Vec::with_capacity( - ((std::mem::size_of::() + 1) + ((U32_LEN + 1) * documents .size_hint() .1 @@ -145,7 +145,7 @@ where .size_hint() .1 .unwrap_or_else(|| documents.size_hint().0); - let buf_len = (std::mem::size_of::() * total_docs) + (total_docs / 0x7F) + 2; + let buf_len = (U32_LEN * total_docs) + (total_docs / 0x7F) + 2; let mut set_buf = Vec::with_capacity(buf_len); let mut clear_buf = Vec::with_capacity(buf_len); diff --git a/crates/store/src/backend/rocksdb/log.rs b/crates/store/src/backend/rocksdb/log.rs index d6c950d6..3b439261 100644 --- a/crates/store/src/backend/rocksdb/log.rs +++ b/crates/store/src/backend/rocksdb/log.rs @@ -31,7 +31,7 @@ use crate::{ use super::CF_LOGS; -const CHANGE_ID_POS: usize = std::mem::size_of::() + std::mem::size_of::(); +const CHANGE_ID_POS: usize = U32_LEN + std::mem::size_of::(); impl Store { pub fn get_last_change_id( diff --git a/crates/store/src/backend/rocksdb/mod.rs b/crates/store/src/backend/rocksdb/mod.rs index 618f8574..7d6b8a9f 100644 --- a/crates/store/src/backend/rocksdb/mod.rs +++ b/crates/store/src/backend/rocksdb/mod.rs @@ -37,10 +37,9 @@ pub const CF_LOGS: &str = "l"; pub const CF_BLOBS: &str = "o"; pub const CF_INDEXES: &str = "i"; -pub const COLLECTION_PREFIX_LEN: usize = std::mem::size_of::() + std::mem::size_of::(); +pub const COLLECTION_PREFIX_LEN: usize = U32_LEN + std::mem::size_of::(); pub const FIELD_PREFIX_LEN: usize = COLLECTION_PREFIX_LEN + std::mem::size_of::(); -pub const ACCOUNT_KEY_LEN: usize = - std::mem::size_of::() + std::mem::size_of::() + std::mem::size_of::(); +pub const ACCOUNT_KEY_LEN: usize = U32_LEN + std::mem::size_of::() + U32_LEN; impl> Serialize for IndexKey { fn serialize(self) -> Vec { diff --git a/crates/store/src/backend/rocksdb/read.rs b/crates/store/src/backend/rocksdb/read.rs index b417a2ee..19fa51e1 100644 --- a/crates/store/src/backend/rocksdb/read.rs +++ b/crates/store/src/backend/rocksdb/read.rs @@ -192,7 +192,7 @@ impl Store { if !key.starts_with(match_prefix) { break; } - let doc_id_pos = key.len() - std::mem::size_of::(); + let doc_id_pos = key.len() - U32_LEN; let value = key.get(FIELD_PREFIX_LEN..doc_id_pos).ok_or_else(|| { Error::InternalError("Invalid key found in 'indexes' column family.".to_string()) })?; diff --git a/crates/store/src/backend/s3/mod.rs b/crates/store/src/backend/s3/mod.rs new file mode 100644 index 00000000..a1151c4a --- /dev/null +++ b/crates/store/src/backend/s3/mod.rs @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2023, Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ops::Range, time::Duration}; + +use s3::{ + creds::{error::CredentialsError, Credentials}, + error::S3Error, + Bucket, Region, +}; +use utils::{codec::base32_custom::Base32Writer, config::Config}; + +use crate::BlobStore; + +pub struct S3Store { + bucket: Bucket, +} + +impl S3Store { + pub async fn open(config: &Config) -> crate::Result { + // Obtain region and endpoint from config + let region = config.value_require("store.blob.s3.region")?; + let region = if let Some(endpoint) = config.value("store.blob.s3.endpoint") { + Region::Custom { + region: region.to_string(), + endpoint: endpoint.to_string(), + } + } else { + region.parse().unwrap() + }; + let credentials = Credentials::new( + config.value("store.blob.s3.access-key"), + config.value("store.blob.s3.secret-key"), + config.value("store.blob.s3.security-token"), + config.value("store.blob.s3.session-token"), + config.value("store.blob.s3.profile"), + )?; + let timeout = config.property_or_static::("store.blob.s3.timeout", "30s")?; + + Ok(S3Store { + bucket: Bucket::new( + config.value_require("store.blob.s3.bucket")?, + region, + credentials, + )? + .with_path_style() + .with_request_timeout(timeout), + }) + } +} + +#[async_trait::async_trait] +impl BlobStore for S3Store { + async fn get_blob(&self, key: &[u8], range: Range) -> crate::Result>> { + let path = Base32Writer::from_bytes(key).finalize(); + let response = if range.start != 0 || range.end != u32::MAX { + self.bucket + .get_object_range( + path, + range.start as u64, + Some(range.end.saturating_sub(1) as u64), + ) + .await + } else { + self.bucket.get_object(path).await + }; + match response { + Ok(response) if (200..300).contains(&response.status_code()) => { + Ok(Some(response.to_vec())) + } + Ok(response) if response.status_code() == 404 => Ok(None), + Ok(response) => Err(crate::Error::InternalError(format!( + "S3 error code {}: {}", + response.status_code(), + String::from_utf8_lossy(response.as_slice()) + ))), + Err(err) => Err(err.into()), + } + } + + async fn put_blob(&self, key: &[u8], data: &[u8]) -> crate::Result<()> { + match self + .bucket + .put_object(Base32Writer::from_bytes(key).finalize(), data) + .await + { + Ok(response) if (200..300).contains(&response.status_code()) => Ok(()), + Ok(response) => Err(crate::Error::InternalError(format!( + "S3 error code {}: {}", + response.status_code(), + String::from_utf8_lossy(response.as_slice()) + ))), + Err(e) => Err(e.into()), + } + } + + async fn delete_blob(&self, key: &[u8]) -> crate::Result { + self.bucket + .delete_object(Base32Writer::from_bytes(key).finalize()) + .await + .map(|response| (200..300).contains(&response.status_code())) + .map_err(|e| e.into()) + } +} + +impl From for crate::Error { + fn from(err: S3Error) -> Self { + Self::InternalError(format!("S3 error: {}", err)) + } +} + +impl From for crate::Error { + fn from(err: CredentialsError) -> Self { + Self::InternalError(format!("S3 Credentials error: {}", err)) + } +} diff --git a/crates/store/src/backend/sqlite/id_assign.rs b/crates/store/src/backend/sqlite/id_assign.rs index b68ca006..a38378f9 100644 --- a/crates/store/src/backend/sqlite/id_assign.rs +++ b/crates/store/src/backend/sqlite/id_assign.rs @@ -23,7 +23,7 @@ use roaring::RoaringBitmap; -use crate::{BitmapKey, StoreId, StoreRead}; +use crate::{write::key::DeserializeBigEndian, BitmapKey, IterateParams, LogKey, U64_LEN}; use super::SqliteStore; @@ -93,9 +93,8 @@ impl IdAssigner { } } -#[async_trait::async_trait] -impl StoreId for SqliteStore { - async fn assign_change_id(&self, account_id: u32) -> crate::Result { +impl SqliteStore { + pub(crate) async fn assign_change_id(&self, account_id: u32) -> crate::Result { let collection = u8::MAX; let key = IdCacheKey::new(account_id, collection); for _ in 0..2 { @@ -108,7 +107,7 @@ impl StoreId for SqliteStore { unreachable!() } - async fn assign_document_id( + pub(crate) async fn assign_document_id( &self, account_id: u32, collection: impl Into + Sync + Send, @@ -123,10 +122,8 @@ impl StoreId for SqliteStore { unreachable!() } -} -impl SqliteStore { - async fn build_id_assigner(&self, key: IdCacheKey) -> crate::Result<()> { + pub(crate) async fn build_id_assigner(&self, key: IdCacheKey) -> crate::Result<()> { // Obtain used ids let used_ids = self .get_bitmap(BitmapKey::document_ids(key.account_id, key.collection)) @@ -146,6 +143,41 @@ impl SqliteStore { Ok(()) } + + async fn get_last_change_id( + &self, + account_id: u32, + collection: impl Into + Sync + Send, + ) -> crate::Result> { + let collection = collection.into(); + + let from_key = LogKey { + account_id, + collection, + change_id: u64::MAX, + }; + let to_key = LogKey { + account_id, + collection, + change_id: 0, + }; + + let mut last_change_id = None; + + self.iterate( + IterateParams::new(from_key, to_key) + .descending() + .no_values() + .only_first(), + |key, _| { + last_change_id = key.deserialize_be_u64(key.len() - U64_LEN)?.into(); + Ok(false) + }, + ) + .await?; + + Ok(last_change_id) + } } #[cfg(test)] diff --git a/crates/store/src/backend/sqlite/main.rs b/crates/store/src/backend/sqlite/main.rs index dd8fe7ff..8b9d21a5 100644 --- a/crates/store/src/backend/sqlite/main.rs +++ b/crates/store/src/backend/sqlite/main.rs @@ -30,15 +30,14 @@ use tokio::sync::oneshot; use utils::{config::Config, UnwrapFailure}; use crate::{ - blob::BlobStore, StoreInit, SUBSPACE_BITMAPS, SUBSPACE_INDEXES, SUBSPACE_LOGS, SUBSPACE_VALUES, + SUBSPACE_ACLS, SUBSPACE_BITMAPS, SUBSPACE_BLOBS, SUBSPACE_COUNTERS, SUBSPACE_INDEXES, + SUBSPACE_LOGS, SUBSPACE_VALUES, }; use super::{pool::SqliteConnectionManager, SqliteStore}; -#[async_trait::async_trait] -impl StoreInit for SqliteStore { - async fn open(config: &Config) -> crate::Result { - let blob = BlobStore::new(config).await?; +impl SqliteStore { + pub async fn open(config: &Config) -> crate::Result { let db = Self { conn_pool: Pool::builder() .max_size(config.property_or_static("store.db.pool.max-connections", "10")?) @@ -71,18 +70,15 @@ impl StoreInit for SqliteStore { id_assigner: Arc::new(Mutex::new(LruCache::new( config.property_or_static("store.db.cache.size", "1000")?, ))), - blob, }; db.create_tables()?; Ok(db) } -} -impl SqliteStore { pub(super) fn create_tables(&self) -> crate::Result<()> { let conn = self.conn_pool.get()?; - for table in [SUBSPACE_VALUES, SUBSPACE_LOGS] { + for table in [SUBSPACE_VALUES, SUBSPACE_LOGS, SUBSPACE_ACLS] { let table = char::from(table); conn.execute( &format!( @@ -95,21 +91,26 @@ impl SqliteStore { )?; } + for table in [SUBSPACE_INDEXES, SUBSPACE_BLOBS] { + let table = char::from(table); + conn.execute( + &format!( + "CREATE TABLE IF NOT EXISTS {table} ( + k BLOB PRIMARY KEY + )" + ), + [], + )?; + } + conn.execute( &format!( "CREATE TABLE IF NOT EXISTS {} ( - k BLOB PRIMARY KEY - )", - char::from(SUBSPACE_INDEXES) - ), - [], - )?; - - conn.execute( - "CREATE TABLE IF NOT EXISTS q ( k BLOB PRIMARY KEY, v INTEGER NOT NULL DEFAULT 0 )", + char::from(SUBSPACE_COUNTERS) + ), [], )?; diff --git a/crates/store/src/backend/sqlite/mod.rs b/crates/store/src/backend/sqlite/mod.rs index 79b14566..216ca473 100644 --- a/crates/store/src/backend/sqlite/mod.rs +++ b/crates/store/src/backend/sqlite/mod.rs @@ -27,11 +27,7 @@ use lru_cache::LruCache; use parking_lot::Mutex; use r2d2::Pool; -use crate::{ - blob::BlobStore, - query::{filter::StoreQuery, log::StoreLog, sort::StoreSort}, - Store, -}; +use crate::U64_LEN; use self::{ id_assign::{IdAssigner, IdCacheKey}, @@ -46,7 +42,7 @@ pub mod read; pub mod write; const WORD_SIZE_BITS: u32 = (WORD_SIZE * 8) as u32; -const WORD_SIZE: usize = std::mem::size_of::(); +const WORD_SIZE: usize = U64_LEN; const WORDS_PER_BLOCK: u32 = 16; pub const BITS_PER_BLOCK: u32 = WORD_SIZE_BITS * WORDS_PER_BLOCK; const BITS_MASK: u32 = BITS_PER_BLOCK - 1; @@ -73,10 +69,4 @@ pub struct SqliteStore { pub(crate) conn_pool: Pool, pub(crate) id_assigner: Arc>>, pub(crate) worker_pool: rayon::ThreadPool, - pub(crate) blob: BlobStore, } - -impl Store for SqliteStore {} -impl StoreQuery for SqliteStore {} -impl StoreSort for SqliteStore {} -impl StoreLog for SqliteStore {} diff --git a/crates/store/src/backend/sqlite/purge.rs b/crates/store/src/backend/sqlite/purge.rs index 631875b9..61e686ae 100644 --- a/crates/store/src/backend/sqlite/purge.rs +++ b/crates/store/src/backend/sqlite/purge.rs @@ -22,15 +22,14 @@ */ use crate::{ - write::key::KeySerializer, StorePurge, SUBSPACE_BITMAPS, SUBSPACE_INDEXES, SUBSPACE_LOGS, - SUBSPACE_VALUES, + write::key::KeySerializer, SUBSPACE_BITMAPS, SUBSPACE_INDEXES, SUBSPACE_LOGS, SUBSPACE_VALUES, + U32_LEN, }; use super::SqliteStore; -#[async_trait::async_trait] -impl StorePurge for SqliteStore { - async fn purge_bitmaps(&self) -> crate::Result<()> { +impl SqliteStore { + pub(crate) async fn purge_bitmaps(&self) -> crate::Result<()> { let conn = self.conn_pool.get()?; self.spawn_worker(move || { //Todo @@ -60,15 +59,11 @@ impl StorePurge for SqliteStore { .await } - async fn purge_account(&self, account_id: u32) -> crate::Result<()> { + pub(crate) async fn purge_account(&self, account_id: u32) -> crate::Result<()> { let conn = self.conn_pool.get()?; self.spawn_worker(move || { - let from_key = KeySerializer::new(std::mem::size_of::()) - .write(account_id) - .finalize(); - let to_key = KeySerializer::new(std::mem::size_of::()) - .write(account_id + 1) - .finalize(); + let from_key = KeySerializer::new(U32_LEN).write(account_id).finalize(); + let to_key = KeySerializer::new(U32_LEN).write(account_id + 1).finalize(); for (table, i) in [ (SUBSPACE_BITMAPS, 'z'), @@ -84,8 +79,6 @@ impl StorePurge for SqliteStore { ))? .execute([&from_key, &to_key])?; } - conn.prepare_cached("DELETE FROM q WHERE k = ?")? - .execute([account_id as i64])?; Ok(()) }) diff --git a/crates/store/src/backend/sqlite/read.rs b/crates/store/src/backend/sqlite/read.rs index 1a610f58..a9bfd3cf 100644 --- a/crates/store/src/backend/sqlite/read.rs +++ b/crates/store/src/backend/sqlite/read.rs @@ -30,14 +30,13 @@ use crate::{ key::{DeserializeBigEndian, KeySerializer}, BitmapClass, ValueClass, }, - BitmapKey, Deserialize, IndexKey, IndexKeyPrefix, Key, LogKey, StoreRead, ValueKey, + BitmapKey, Deserialize, IndexKey, IndexKeyPrefix, IterateParams, Key, ValueKey, U32_LEN, }; use super::{SqliteStore, BITS_PER_BLOCK, WORDS_PER_BLOCK, WORD_SIZE_BITS}; -#[async_trait::async_trait] -impl StoreRead for SqliteStore { - async fn get_value(&self, key: impl Key) -> crate::Result> +impl SqliteStore { + pub(crate) async fn get_value(&self, key: impl Key) -> crate::Result> where U: Deserialize + 'static, { @@ -56,7 +55,7 @@ impl StoreRead for SqliteStore { .await } - async fn get_bitmap( + pub(crate) async fn get_bitmap( &self, mut key: BitmapKey, ) -> crate::Result> { @@ -75,7 +74,7 @@ impl StoreRead for SqliteStore { while let Some(row) = rows.next()? { let key = row.get_ref(0)?.as_bytes()?; if key.len() == key_len { - let block_num = key.deserialize_be_u32(key.len() - std::mem::size_of::())?; + let block_num = key.deserialize_be_u32(key.len() - U32_LEN)?; for word_num in 0..WORDS_PER_BLOCK { match row.get::<_, i64>((word_num + 1) as usize)? as u64 { @@ -106,7 +105,7 @@ impl StoreRead for SqliteStore { }).await } - async fn range_to_bitmap( + pub(crate) async fn range_to_bitmap( &self, account_id: u32, collection: u8, @@ -117,19 +116,13 @@ impl StoreRead for SqliteStore { let conn = self.conn_pool.get()?; self.spawn_worker(move || { let k1 = KeySerializer::new( - std::mem::size_of::>() - + value.len() - + 1 - + std::mem::size_of::(), + std::mem::size_of::>() + value.len() + 1 + U32_LEN, ) .write(account_id) .write(collection) .write(field); let k2 = KeySerializer::new( - std::mem::size_of::>() - + value.len() - + 1 - + std::mem::size_of::(), + std::mem::size_of::>() + value.len() + 1 + U32_LEN, ) .write(account_id) .write(collection) @@ -170,14 +163,14 @@ impl StoreRead for SqliteStore { if op != Operator::Equal { while let Some(row) = rows.next()? { let key = row.get_ref(0)?.as_bytes()?; - bm.insert(key.deserialize_be_u32(key.len() - std::mem::size_of::())?); + bm.insert(key.deserialize_be_u32(key.len() - U32_LEN)?); } } else { let key_len = begin.len(); while let Some(row) = rows.next()? { let key = row.get_ref(0)?.as_bytes()?; if key.len() == key_len { - bm.insert(key.deserialize_be_u32(key.len() - std::mem::size_of::())?); + bm.insert(key.deserialize_be_u32(key.len() - U32_LEN)?); } } } @@ -187,7 +180,7 @@ impl StoreRead for SqliteStore { .await } - async fn sort_index( + pub(crate) async fn sort_index( &self, account_id: u32, collection: impl Into + Sync + Send, @@ -223,7 +216,7 @@ impl StoreRead for SqliteStore { while let Some(row) = rows.next()? { let key = row.get_ref(0)?.as_bytes()?; - let id_pos = key.len() - std::mem::size_of::(); + let id_pos = key.len() - U32_LEN; debug_assert!(key.starts_with(&begin)); if !cb( key.get(prefix_len..id_pos).ok_or_else(|| { @@ -240,47 +233,53 @@ impl StoreRead for SqliteStore { .await } - async fn iterate( + pub(crate) async fn iterate( &self, - begin: impl Key, - end: impl Key, - first: bool, - ascending: bool, + params: IterateParams, mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> crate::Result + Sync + Send, ) -> crate::Result<()> { let conn = self.conn_pool.get()?; self.spawn_worker(move || { - let table = char::from(begin.subspace()); - let begin = begin.serialize(false); - let end = end.serialize(false); + let table = char::from(params.begin.subspace()); + let begin = params.begin.serialize(false); + let end = params.end.serialize(false); + let keys = if params.values { "k, v" } else { "k" }; - let mut query = conn.prepare_cached(&match (first, ascending) { + let mut query = conn.prepare_cached(&match (params.first, params.ascending) { (true, true) => { format!( - "SELECT k, v FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC LIMIT 1" + "SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC LIMIT 1" ) } (true, false) => { format!( - "SELECT k, v FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC LIMIT 1" + "SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC LIMIT 1" ) } (false, true) => { - format!("SELECT k, v FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC") + format!("SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC") } (false, false) => { - format!("SELECT k, v FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC") + format!("SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC") } })?; let mut rows = query.query([&begin, &end])?; - while let Some(row) = rows.next()? { - let key = row.get_ref(0)?.as_bytes()?; - let value = row.get_ref(1)?.as_bytes()?; + if params.values { + while let Some(row) = rows.next()? { + let key = row.get_ref(0)?.as_bytes()?; + let value = row.get_ref(1)?.as_bytes()?; - if !cb(key, value)? { - break; + if !cb(key, value)? { + break; + } + } + } else { + while let Some(row) = rows.next()? { + if !cb(row.get_ref(0)?.as_bytes()?, b"")? { + break; + } } } @@ -289,36 +288,7 @@ impl StoreRead for SqliteStore { .await } - async fn get_last_change_id( - &self, - account_id: u32, - collection: impl Into + Sync + Send, - ) -> crate::Result> { - let conn = self.conn_pool.get()?; - let collection = collection.into(); - - self.spawn_worker(move || { - let key = LogKey { - account_id, - collection, - change_id: u64::MAX, - } - .serialize(false); - - conn.prepare_cached("SELECT k FROM l WHERE k < ? ORDER BY k DESC LIMIT 1")? - .query_row([&key], |row| { - let key = row.get_ref(0)?.as_bytes()?; - - key.deserialize_be_u64(key.len() - std::mem::size_of::()) - .map_err(|err| rusqlite::Error::ToSqlConversionFailure(err.into())) - }) - .optional() - .map_err(Into::into) - }) - .await - } - - async fn get_counter( + pub(crate) async fn get_counter( &self, key: impl Into> + Sync + Send, ) -> crate::Result { @@ -326,7 +296,7 @@ impl StoreRead for SqliteStore { let conn = self.conn_pool.get()?; self.spawn_worker(move || { match conn - .prepare_cached("SELECT v FROM q WHERE k = ?")? + .prepare_cached("SELECT v FROM c WHERE k = ?")? .query_row([&key], |row| row.get::<_, i64>(0)) { Ok(value) => Ok(value), @@ -338,45 +308,64 @@ impl StoreRead for SqliteStore { } #[cfg(feature = "test_mode")] - async fn assert_is_empty(&self) { - use crate::StorePurge; - + pub(crate) async fn assert_is_empty(&self) { let conn = self.conn_pool.get().unwrap(); - self.purge_bitmaps().await.unwrap(); - self.spawn_worker(move || { + // Values - let mut query = conn.prepare_cached("SELECT k, v FROM v").unwrap(); - let mut rows = query.query([]).unwrap(); let mut has_errors = false; + for table in [crate::SUBSPACE_VALUES, crate::SUBSPACE_ACLS, crate::SUBSPACE_COUNTERS] { + let table = char::from(table); + let mut query = conn.prepare_cached(&format!("SELECT k, v FROM {table}")).unwrap(); + let mut rows = query.query([]).unwrap(); - while let Some(row) = rows.next().unwrap() { - let key = row.get_ref(0).unwrap().as_bytes().unwrap(); - let value = row.get_ref(1).unwrap().as_bytes().unwrap(); + while let Some(row) = rows.next().unwrap() { + let key = row.get_ref(0).unwrap().as_bytes().unwrap(); + if table != 'c' { + let value = row.get_ref(1).unwrap().as_bytes().unwrap(); - if key[0..4] != u32::MAX.to_be_bytes() { - eprintln!("Table values is not empty: {key:?} {value:?}"); - has_errors = true; + if key[0..4] != u32::MAX.to_be_bytes() { + eprintln!("Table {table:?} is not empty: {key:?} {value:?}"); + has_errors = true; + } + } else { + let value = row.get::<_, i64>(1).unwrap(); + if value != 0 { + eprintln!( + "Table counter is not empty, account {:?}, quota: {}", + key, value, + ); + has_errors = true; + } + } } } // Indexes - let mut query = conn.prepare_cached("SELECT k FROM i").unwrap(); - let mut rows = query.query([]).unwrap(); + for table in [crate::SUBSPACE_INDEXES, crate::SUBSPACE_BLOBS] { + let table = char::from(table); + let mut query = conn.prepare_cached(&format!("SELECT k FROM {table}")).unwrap(); + let mut rows = query.query([]).unwrap(); - while let Some(row) = rows.next().unwrap() { - let key = row.get_ref(0).unwrap().as_bytes().unwrap(); + while let Some(row) = rows.next().unwrap() { + let key = row.get_ref(0).unwrap().as_bytes().unwrap(); - eprintln!( - "Table index is not empty, account {}, collection {}, document {}, property {}, value {:?}: {:?}", - u32::from_be_bytes(key[0..4].try_into().unwrap()), - key[4], - u32::from_be_bytes(key[key.len()-4..].try_into().unwrap()), - key[5], - String::from_utf8_lossy(&key[6..key.len()-4]), - key - ); - has_errors = true; + if table == 'i' { + eprintln!( + "Table index is not empty, account {}, collection {}, document {}, property {}, value {:?}: {:?}", + u32::from_be_bytes(key[0..4].try_into().unwrap()), + key[4], + u32::from_be_bytes(key[key.len()-4..].try_into().unwrap()), + key[5], + String::from_utf8_lossy(&key[6..key.len()-4]), + key + ); + + } else { + eprintln!("Table {table:?} is not empty: {key:?}"); + } + has_errors = true; + } } // Bitmaps @@ -402,22 +391,6 @@ impl StoreRead for SqliteStore { } } - // Quotas - let mut query = conn.prepare_cached("SELECT k, v FROM q").unwrap(); - let mut rows = query.query([]).unwrap(); - - while let Some(row) = rows.next().unwrap() { - let key = row.get_ref(0).unwrap().as_bytes().unwrap(); - let value = row.get::<_, i64>(1).unwrap(); - if value != 0 { - eprintln!( - "Table counter is not empty, account {:?}, quota: {}", - key, value, - ); - has_errors = true; - } - } - // Delete logs conn.execute("DELETE FROM l", []).unwrap(); diff --git a/crates/store/src/backend/sqlite/write.rs b/crates/store/src/backend/sqlite/write.rs index ba2b4b99..39ea01ef 100644 --- a/crates/store/src/backend/sqlite/write.rs +++ b/crates/store/src/backend/sqlite/write.rs @@ -24,8 +24,8 @@ use rusqlite::{params, OptionalExtension, TransactionBehavior}; use crate::{ - write::{Batch, Operation, ValueOp}, - BitmapKey, IndexKey, Key, LogKey, StoreWrite, ValueKey, + write::{Batch, Operation, ValueClass, ValueOp}, + BitmapKey, BlobKey, IndexKey, Key, LogKey, ValueKey, }; use super::{SqliteStore, BITS_MASK, BITS_PER_BLOCK}; @@ -85,9 +85,8 @@ const CLEAR_QUERIES: &[&str] = &[ "UPDATE b SET p = p & ? WHERE z = ?", ]; -#[async_trait::async_trait] -impl StoreWrite for SqliteStore { - async fn write(&self, batch: Batch) -> crate::Result<()> { +impl SqliteStore { + pub(crate) async fn write(&self, batch: Batch) -> crate::Result<()> { let mut conn = self.conn_pool.get()?; self.spawn_worker(move || { let mut account_id = u32::MAX; @@ -135,12 +134,12 @@ impl StoreWrite for SqliteStore { if *by >= 0 { trx.prepare_cached(concat!( - "INSERT INTO q (k, v) VALUES (?, ?) ", + "INSERT INTO c (k, v) VALUES (?, ?) ", "ON CONFLICT(k) DO UPDATE SET v = v + excluded.v" ))? .execute(params![&key, *by])?; } else { - trx.prepare_cached("UPDATE q SET v = v + ? WHERE k = ?")? + trx.prepare_cached("UPDATE c SET v = v + ? WHERE k = ?")? .execute(params![*by, &key])?; } } @@ -154,11 +153,19 @@ impl StoreWrite for SqliteStore { .serialize(false); if let ValueOp::Set(value) = op { - trx.prepare_cached("INSERT OR REPLACE INTO v (k, v) VALUES (?, ?)")? - .execute([&key, value])?; + trx.prepare_cached(if !matches!(class, ValueClass::Acl(_)) { + "INSERT OR REPLACE INTO v (k, v) VALUES (?, ?)" + } else { + "INSERT OR REPLACE INTO a (k, v) VALUES (?, ?)" + })? + .execute([&key, value])?; } else { - trx.prepare_cached("DELETE FROM v WHERE k = ?")? - .execute([&key])?; + trx.prepare_cached(if !matches!(class, ValueClass::Acl(_)) { + "DELETE FROM v WHERE k = ?" + } else { + "DELETE FROM a WHERE k = ?" + })? + .execute([&key])?; } } Operation::Index { field, key, set } => { @@ -200,7 +207,24 @@ impl StoreWrite for SqliteStore { .execute(params![bitmap_value_clear, &key])?; }; } + Operation::Blob { hash, op, set } => { + let key = BlobKey { + account_id, + collection, + document_id, + hash, + op: *op, + } + .serialize(false); + if *set { + trx.prepare_cached("INSERT OR IGNORE INTO o (k) VALUES (?)")? + .execute([&key])?; + } else { + trx.prepare_cached("DELETE FROM o WHERE k = ?")? + .execute([&key])?; + } + } Operation::Log { collection, change_id, @@ -248,9 +272,10 @@ impl StoreWrite for SqliteStore { } #[cfg(feature = "test_mode")] - async fn destroy(&self) { + pub(crate) async fn destroy(&self) { use crate::{ - SUBSPACE_BITMAPS, SUBSPACE_INDEXES, SUBSPACE_LOGS, SUBSPACE_QUOTAS, SUBSPACE_VALUES, + SUBSPACE_ACLS, SUBSPACE_BITMAPS, SUBSPACE_BLOBS, SUBSPACE_COUNTERS, SUBSPACE_INDEXES, + SUBSPACE_LOGS, SUBSPACE_VALUES, }; let conn = self.conn_pool.get().unwrap(); @@ -259,7 +284,9 @@ impl StoreWrite for SqliteStore { SUBSPACE_LOGS, SUBSPACE_BITMAPS, SUBSPACE_INDEXES, - SUBSPACE_QUOTAS, + SUBSPACE_BLOBS, + SUBSPACE_ACLS, + SUBSPACE_COUNTERS, ] { conn.execute(&format!("DROP TABLE {}", char::from(table)), []) .unwrap(); diff --git a/crates/store/src/blob/mod.rs b/crates/store/src/blob/mod.rs deleted file mode 100644 index de8704ac..00000000 --- a/crates/store/src/blob/mod.rs +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of the Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -pub mod read; -pub mod write; - -use std::{path::PathBuf, time::Duration}; - -use s3::{ - creds::{error::CredentialsError, Credentials}, - error::S3Error, - Bucket, Region, -}; -use utils::config::Config; - -use crate::BlobKind; - -pub enum BlobStore { - Local(BlobPaths), - Remote(Bucket), -} - -pub struct BlobPaths { - path_email: PathBuf, - path_temporary: PathBuf, - path_other: PathBuf, -} - -impl BlobStore { - pub async fn new(config: &Config) -> crate::Result { - match config.value_require("store.blob.type")? { - "s3" | "minio" | "gcs" => { - // Obtain region and endpoint from config - let region = config.value_require("store.blob.s3.region")?; - let region = if let Some(endpoint) = config.value("store.blob.s3.endpoint") { - Region::Custom { - region: region.to_string(), - endpoint: endpoint.to_string(), - } - } else { - region.parse().unwrap() - }; - let credentials = Credentials::new( - config.value("store.blob.s3.access-key"), - config.value("store.blob.s3.secret-key"), - config.value("store.blob.s3.security-token"), - config.value("store.blob.s3.session-token"), - config.value("store.blob.s3.profile"), - )?; - let timeout = - config.property_or_static::("store.blob.s3.timeout", "30s")?; - - Ok(BlobStore::Remote( - Bucket::new( - config.value_require("store.blob.s3.bucket")?, - region, - credentials, - )? - .with_path_style() - .with_request_timeout(timeout), - )) - } - "local" => { - let path = config.property_require::("store.blob.local.path")?; - let mut path_email = path.clone(); - path_email.push("emails"); - let mut path_temporary = path.clone(); - path_temporary.push("tmp"); - let mut path_other = path; - path_other.push("blobs"); - - Ok(BlobStore::Local(BlobPaths { - path_email, - path_temporary, - path_other, - })) - } - unknown => Err(crate::Error::InternalError(format!( - "Unknown blob store type: {unknown}", - ))), - } - } -} - -impl From for crate::Error { - fn from(err: std::io::Error) -> Self { - Self::InternalError(format!("IO error: {}", err)) - } -} - -impl From for crate::Error { - fn from(err: S3Error) -> Self { - Self::InternalError(format!("S3 error: {}", err)) - } -} - -impl From for crate::Error { - fn from(err: CredentialsError) -> Self { - Self::InternalError(format!("S3 Credentials error: {}", err)) - } -} - -fn get_local_path(base_path: &BlobPaths, kind: &BlobKind) -> PathBuf { - match kind { - BlobKind::LinkedMaildir { - account_id, - document_id, - } => { - let mut path = base_path.path_email.to_path_buf(); - path.push(format!("{:x}", account_id)); - path.push("Maildir"); - path.push("cur"); - path.push(format!("{:x}", document_id)); - path - } - BlobKind::Linked { - account_id, - collection, - document_id, - } => { - let mut path = base_path.path_other.to_path_buf(); - path.push(format!("{:x}", account_id)); - path.push(format!("{:x}", collection)); - path.push(format!("{:x}", document_id)); - path - } - BlobKind::Temporary { - account_id, - timestamp, - seq, - } => { - let mut path = base_path.path_temporary.to_path_buf(); - path.push(format!("{:x}", account_id)); - path.push(format!("{:x}_{:x}", timestamp, seq)); - path - } - } -} - -fn get_s3_path(kind: &BlobKind) -> String { - match kind { - BlobKind::LinkedMaildir { - account_id, - document_id, - } => format!("/{:x}/{:x}", account_id, document_id), - BlobKind::Linked { - account_id, - collection, - document_id, - } => format!("/{:x}/{:x}/{:x}", account_id, collection, document_id), - BlobKind::Temporary { - account_id, - timestamp, - seq, - } => format!("/tmp/{:x}/{:x}_{:x}", account_id, timestamp, seq), - } -} diff --git a/crates/store/src/blob/read.rs b/crates/store/src/blob/read.rs deleted file mode 100644 index 0b1fad97..00000000 --- a/crates/store/src/blob/read.rs +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of the Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::{io::SeekFrom, ops::Range}; - -use tokio::{ - fs::{self, File}, - io::{AsyncReadExt, AsyncSeekExt}, -}; - -use crate::{backend::sqlite::SqliteStore, BlobKind}; - -use super::{get_local_path, get_s3_path, BlobStore}; - -impl SqliteStore { - pub async fn get_blob( - &self, - kind: &BlobKind, - range: Range, - ) -> crate::Result>> { - match &self.blob { - BlobStore::Local(base_path) => { - let blob_path = get_local_path(base_path, kind); - let blob_size = match fs::metadata(&blob_path).await { - Ok(m) => m.len(), - Err(_) => return Ok(None), - }; - let mut blob = File::open(&blob_path).await?; - - Ok(Some(if range.start != 0 || range.end != u32::MAX { - let from_offset = if range.start < blob_size as u32 { - range.start - } else { - 0 - }; - let mut buf = vec![ - 0; - (std::cmp::min(range.end, blob_size as u32) - from_offset) - as usize - ]; - - if from_offset > 0 { - blob.seek(SeekFrom::Start(from_offset as u64)).await?; - } - blob.read_exact(&mut buf).await?; - buf - } else { - let mut buf = Vec::with_capacity(blob_size as usize); - blob.read_to_end(&mut buf).await?; - buf - })) - } - BlobStore::Remote(bucket) => { - let path = get_s3_path(kind); - let response = if range.start != 0 || range.end != u32::MAX { - bucket - .get_object_range( - path, - range.start as u64, - Some(range.end.saturating_sub(1) as u64), - ) - .await - } else { - bucket.get_object(path).await - }; - match response { - Ok(response) if (200..300).contains(&response.status_code()) => { - Ok(Some(response.to_vec())) - } - Ok(response) if response.status_code() == 404 => Ok(None), - Ok(response) => Err(crate::Error::InternalError(format!( - "S3 error code {}: {}", - response.status_code(), - String::from_utf8_lossy(response.as_slice()) - ))), - Err(err) => Err(err.into()), - } - } - } - } -} diff --git a/crates/store/src/blob/write.rs b/crates/store/src/blob/write.rs deleted file mode 100644 index 37bec0fc..00000000 --- a/crates/store/src/blob/write.rs +++ /dev/null @@ -1,332 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of the Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::ops::Range; - -use tokio::{ - fs::{self, File}, - io::AsyncWriteExt, -}; - -use crate::{backend::sqlite::SqliteStore, write::now, BlobKind}; - -use super::{get_local_path, get_s3_path, BlobStore}; - -impl SqliteStore { - pub async fn put_blob(&self, kind: &BlobKind, data: &[u8]) -> crate::Result<()> { - match &self.blob { - BlobStore::Local(base_path) => { - let blob_path = get_local_path(base_path, kind); - - fs::create_dir_all(blob_path.parent().unwrap()).await?; - let mut blob_file = File::create(&blob_path).await?; - blob_file.write_all(data).await?; - blob_file.flush().await?; - - Ok(()) - } - BlobStore::Remote(bucket) => { - let path = get_s3_path(kind); - match bucket.put_object(path, data).await { - Ok(response) if (200..300).contains(&response.status_code()) => Ok(()), - Ok(response) => Err(crate::Error::InternalError(format!( - "S3 error code {}: {}", - response.status_code(), - String::from_utf8_lossy(response.as_slice()) - ))), - Err(e) => Err(e.into()), - } - } - } - } - - pub async fn copy_blob( - &self, - src: &BlobKind, - dest: &BlobKind, - range: Option>, - ) -> crate::Result { - if let Some(range) = range { - if let Some(bytes) = self.get_blob(src, range).await? { - self.put_blob(dest, &bytes).await?; - Ok(true) - } else { - Ok(false) - } - } else { - match &self.blob { - BlobStore::Local(base_path) => { - let dest_path = get_local_path(base_path, dest); - let src_path = get_local_path(base_path, src); - - if fs::metadata(&src_path).await.is_ok() { - fs::create_dir_all(dest_path.parent().unwrap()).await?; - fs::copy(src_path, dest_path).await?; - Ok(true) - } else { - Ok(false) - } - } - BlobStore::Remote(bucket) => { - let src_path = get_s3_path(src); - let dest_path = get_s3_path(dest); - - bucket - .copy_object_internal(src_path, dest_path) - .await - .map(|code| (200..300).contains(&code)) - .map_err(|e| e.into()) - } - } - } - } - - pub async fn delete_blob(&self, kind: &BlobKind) -> crate::Result { - match &self.blob { - BlobStore::Local(base_path) => { - let blob_path = get_local_path(base_path, kind); - - if blob_path.exists() { - fs::remove_file(&blob_path).await?; - Ok(true) - } else { - Ok(false) - } - } - BlobStore::Remote(bucket) => { - let path = get_s3_path(kind); - bucket - .delete_object(path) - .await - .map(|response| (200..300).contains(&response.status_code())) - .map_err(|e| e.into()) - } - } - } - - pub async fn delete_account_blobs(&self, account_id: u32) -> crate::Result<()> { - match &self.blob { - BlobStore::Local(base_path) => { - for path in [ - &base_path.path_email, - &base_path.path_other, - &base_path.path_temporary, - ] { - let mut path = path.to_path_buf(); - path.push(format!("{:x}", account_id)); - if fs::metadata(&path).await.is_ok() { - fs::remove_dir_all(path).await?; - } - } - - Ok(()) - } - BlobStore::Remote(bucket) => { - for prefix in [ - format!("/{:x}/", account_id), - format!("/tmp/{:x}/", account_id), - ] { - let prefix_base = prefix.strip_prefix('/').unwrap(); - for object in bucket - .list(prefix.clone(), None) - .await? - .into_iter() - .flat_map(|result| result.contents) - { - if object.key.starts_with(&prefix) || object.key.starts_with(prefix_base) { - let result = bucket.delete_object(object.key).await?; - if !(200..300).contains(&result.status_code()) { - return Err(crate::Error::InternalError(format!( - "Failed to delete bucket item, code {}: {}", - result.status_code(), - String::from_utf8_lossy(result.as_slice()) - ))); - } - } else { - tracing::debug!("Unexpected S3 object while deleting: {}", object.key); - } - } - } - Ok(()) - } - } - } - - pub async fn purge_tmp_blobs(&self, ttl: u64) -> crate::Result<()> { - let now = now(); - match &self.blob { - BlobStore::Local(base_path) => { - if fs::metadata(&base_path.path_temporary).await.is_ok() { - let mut dir = fs::read_dir(&base_path.path_temporary).await?; - while let Some(item) = dir.next_entry().await? { - if item.metadata().await?.is_dir() { - let mut dir = fs::read_dir(item.path()).await?; - while let Some(item) = dir.next_entry().await? { - if item.metadata().await?.is_file() { - if let Some(timestamp) = - item.file_name().to_str().and_then(parse_timestamp) - { - if now.saturating_sub(timestamp) > ttl { - fs::remove_file(item.path()).await?; - } - } else { - tracing::debug!( - "Found invalid temporary filename while purging: {}", - item.file_name().to_string_lossy() - ); - } - } - } - } - } - } - - Ok(()) - } - BlobStore::Remote(bucket) => { - for object in bucket - .list("/tmp/".to_string(), None) - .await? - .into_iter() - .flat_map(|result| result.contents) - { - if object.key.starts_with("/tmp/") || object.key.starts_with("tmp/") { - if let Some(timestamp) = object - .key - .rsplit_once('/') - .and_then(|(_, name)| parse_timestamp(name)) - { - if now.saturating_sub(timestamp) > ttl { - let result = bucket.delete_object(object.key).await?; - if !(200..300).contains(&result.status_code()) { - return Err(crate::Error::InternalError(format!( - "Failed to delete bucket item, code {}: {}", - result.status_code(), - String::from_utf8_lossy(result.as_slice()) - ))); - } - } - } else { - tracing::debug!( - "Found invalid temporary filename while purging: {}", - object.key - ); - } - } else { - tracing::debug!("Unexpected S3 object while purging: {}", object.key); - } - } - Ok(()) - } - } - } - - pub async fn get_tmp_blob_usage( - &self, - account_id: u32, - ttl: u64, - ) -> crate::Result<(usize, usize)> { - let now = now(); - let mut total_bytes = 0; - let mut total_files = 0; - - match &self.blob { - BlobStore::Local(base_path) => { - let mut path = base_path.path_temporary.to_path_buf(); - path.push(format!("{:x}", account_id)); - - if fs::metadata(&path).await.is_ok() { - let mut dir = fs::read_dir(path).await?; - while let Some(item) = dir.next_entry().await? { - match item.metadata().await { - Ok(metadata) if metadata.is_file() => { - if let Some(timestamp) = - item.file_name().to_str().and_then(parse_timestamp) - { - if now.saturating_sub(timestamp) > ttl { - let _ = fs::remove_file(item.path()).await; - } else { - total_bytes += metadata.len() as usize; - total_files += 1; - } - } else { - tracing::debug!( - "Found invalid temporary filename while purging: {}", - item.file_name().to_string_lossy() - ); - } - } - _ => (), - } - } - } - } - BlobStore::Remote(bucket) => { - let prefix = format!("/tmp/{:x}/", account_id); - let prefix_base = prefix.strip_prefix('/').unwrap(); - for object in bucket - .list(prefix.clone(), None) - .await? - .into_iter() - .flat_map(|result| result.contents) - { - if object.key.starts_with(prefix_base) || object.key.starts_with(&prefix) { - if let Some(timestamp) = object - .key - .rsplit_once('/') - .and_then(|(_, name)| parse_timestamp(name)) - { - if now.saturating_sub(timestamp) > ttl { - let result = bucket.delete_object(object.key).await?; - if !(200..300).contains(&result.status_code()) { - return Err(crate::Error::InternalError(format!( - "Failed to delete bucket item, code {}: {}", - result.status_code(), - String::from_utf8_lossy(result.as_slice()) - ))); - } - } else { - total_bytes += object.size as usize; - total_files += 1; - } - } else { - tracing::debug!( - "Found invalid temporary filename while purging: {}", - object.key - ); - } - } else { - tracing::debug!("Unexpected S3 object while purging: {}", object.key); - } - } - } - } - - Ok((total_files, total_bytes)) - } -} - -fn parse_timestamp(name: &str) -> Option { - name.split_once('_') - .and_then(|(timestamp, _)| u64::from_str_radix(timestamp, 16).ok()) -} diff --git a/crates/store/src/dispatch.rs b/crates/store/src/dispatch.rs new file mode 100644 index 00000000..c3fe937a --- /dev/null +++ b/crates/store/src/dispatch.rs @@ -0,0 +1,265 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::ops::BitAndAssign; + +use roaring::RoaringBitmap; + +use crate::{ + query, + write::{Batch, BitmapClass, ValueClass}, + BitmapKey, Deserialize, IterateParams, Key, Store, ValueKey, +}; + +impl Store { + pub async fn assign_change_id(&self, account_id: u32) -> crate::Result { + match self { + Self::SQLite(store) => store.assign_change_id(account_id).await, + Self::FoundationDb(store) => store.assign_change_id(account_id).await, + } + } + + pub async fn assign_document_id( + &self, + account_id: u32, + collection: impl Into + Sync + Send, + ) -> crate::Result { + match self { + Self::SQLite(store) => store.assign_document_id(account_id, collection).await, + Self::FoundationDb(store) => store.assign_document_id(account_id, collection).await, + } + } + + pub async fn get_value(&self, key: impl Key) -> crate::Result> + where + U: Deserialize + 'static, + { + match self { + Self::SQLite(store) => store.get_value(key).await, + Self::FoundationDb(store) => store.get_value(key).await, + } + } + + pub async fn get_values(&self, key: Vec) -> crate::Result>> + where + U: Deserialize + 'static, + { + let mut results = Vec::with_capacity(key.len()); + + for key in key { + results.push(self.get_value(key).await?); + } + + Ok(results) + } + + pub async fn get_bitmap( + &self, + key: BitmapKey, + ) -> crate::Result> { + match self { + Self::SQLite(store) => store.get_bitmap(key).await, + Self::FoundationDb(store) => store.get_bitmap(key).await, + } + } + + pub async fn get_bitmaps_intersection( + &self, + keys: Vec>, + ) -> crate::Result> { + let mut result: Option = None; + for key in keys { + if let Some(bitmap) = self.get_bitmap(key).await? { + if let Some(result) = &mut result { + result.bitand_assign(&bitmap); + if result.is_empty() { + break; + } + } else { + result = Some(bitmap); + } + } else { + return Ok(None); + } + } + Ok(result) + } + + pub async fn range_to_bitmap( + &self, + account_id: u32, + collection: u8, + field: u8, + value: Vec, + op: query::Operator, + ) -> crate::Result> { + match self { + Self::SQLite(store) => { + store + .range_to_bitmap(account_id, collection, field, value, op) + .await + } + Self::FoundationDb(store) => { + store + .range_to_bitmap(account_id, collection, field, value, op) + .await + } + } + } + + pub async fn sort_index( + &self, + account_id: u32, + collection: impl Into + Sync + Send, + field: impl Into + Sync + Send, + ascending: bool, + cb: impl for<'x> FnMut(&'x [u8], u32) -> crate::Result + Sync + Send, + ) -> crate::Result<()> { + match self { + Self::SQLite(store) => { + store + .sort_index(account_id, collection, field, ascending, cb) + .await + } + Self::FoundationDb(store) => { + store + .sort_index(account_id, collection, field, ascending, cb) + .await + } + } + } + + pub(crate) async fn iterate( + &self, + params: IterateParams, + cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> crate::Result + Sync + Send, + ) -> crate::Result<()> { + match self { + Self::SQLite(store) => store.iterate(params, cb).await, + Self::FoundationDb(store) => store.iterate(params, cb).await, + } + } + + pub async fn get_counter( + &self, + key: impl Into> + Sync + Send, + ) -> crate::Result { + match self { + Self::SQLite(store) => store.get_counter(key).await, + Self::FoundationDb(store) => store.get_counter(key).await, + } + } + + pub async fn write(&self, batch: Batch) -> crate::Result<()> { + match self { + Self::SQLite(store) => store.write(batch).await, + Self::FoundationDb(store) => store.write(batch).await, + } + } + + pub async fn purge_bitmaps(&self) -> crate::Result<()> { + match self { + Self::SQLite(store) => store.purge_bitmaps().await, + Self::FoundationDb(store) => store.purge_bitmaps().await, + } + } + pub async fn purge_account(&self, account_id: u32) -> crate::Result<()> { + match self { + Self::SQLite(store) => store.purge_account(account_id).await, + Self::FoundationDb(store) => store.purge_account(account_id).await, + } + } + + #[cfg(feature = "test_mode")] + pub async fn destroy(&self) { + match self { + Self::SQLite(store) => store.destroy().await, + Self::FoundationDb(store) => store.destroy().await, + } + } + + #[cfg(feature = "test_mode")] + pub async fn blob_hash_expire_all(&self) { + use crate::{ + write::{key::DeserializeBigEndian, BatchBuilder, BlobOp, F_CLEAR}, + BlobHash, BlobKey, BLOB_HASH_LEN, U32_LEN, U64_LEN, + }; + + // Delete all temporary hashes + let from_key = BlobKey { + account_id: 0, + collection: 0, + document_id: 0, + op: BlobOp::Reserve { until: 0, size: 0 }, + hash: BlobHash::default(), + }; + let to_key = BlobKey { + account_id: u32::MAX, + collection: 0, + document_id: 0, + op: BlobOp::Reserve { until: 0, size: 0 }, + hash: BlobHash::default(), + }; + let mut batch = BatchBuilder::new(); + let mut last_account_id = u32::MAX; + self.iterate( + IterateParams::new(from_key, to_key).ascending().no_values(), + |key, _| { + let account_id = key.deserialize_be_u32(1)?; + if account_id != last_account_id { + last_account_id = account_id; + batch.with_account_id(account_id); + } + + batch.blob( + BlobHash::try_from_hash_slice( + key.get(1 + U32_LEN..1 + U32_LEN + BLOB_HASH_LEN).unwrap(), + ) + .unwrap(), + BlobOp::Reserve { + until: key.deserialize_be_u64(key.len() - (U64_LEN + U32_LEN))?, + size: key.deserialize_be_u32(key.len() - U32_LEN)? as usize, + }, + F_CLEAR, + ); + + Ok(true) + }, + ) + .await + .unwrap(); + self.write(batch.build()).await.unwrap(); + } + + #[cfg(feature = "test_mode")] + pub async fn assert_is_empty(&self, blob_store: std::sync::Arc) { + self.blob_hash_expire_all().await; + self.blob_hash_purge(blob_store).await.unwrap(); + self.purge_bitmaps().await.unwrap(); + + match self { + Self::SQLite(store) => store.assert_is_empty().await, + Self::FoundationDb(store) => store.assert_is_empty().await, + } + } +} diff --git a/crates/store/src/fts/bloom.rs b/crates/store/src/fts/bloom.rs index 31e36427..6145a637 100644 --- a/crates/store/src/fts/bloom.rs +++ b/crates/store/src/fts/bloom.rs @@ -232,7 +232,7 @@ impl From> for BloomHashGroup { impl Serialize for BloomFilter { fn serialize(self) -> Vec { - let mut buf = Vec::with_capacity(std::mem::size_of::() + self.b.serialized_size()); + let mut buf = Vec::with_capacity(U64_LEN + self.b.serialized_size()); buf.push_leb128(self.m); let _ = self.b.serialize_into(&mut buf); buf diff --git a/crates/store/src/fts/query.rs b/crates/store/src/fts/query.rs index 8c0a2a62..37938e3f 100644 --- a/crates/store/src/fts/query.rs +++ b/crates/store/src/fts/query.rs @@ -26,9 +26,7 @@ use std::ops::BitOrAssign; use nlp::language::{stemmer::Stemmer, Language}; use roaring::RoaringBitmap; -use crate::{ - fts::builder::MAX_TOKEN_LENGTH, BitmapKey, StoreRead, ValueKey, HASH_EXACT, HASH_STEMMED, -}; +use crate::{fts::builder::MAX_TOKEN_LENGTH, BitmapKey, ValueKey, HASH_EXACT, HASH_STEMMED}; use super::term_index::TermIndex; diff --git a/crates/store/src/fts/term_index.rs b/crates/store/src/fts/term_index.rs index b91f74db..2b876578 100644 --- a/crates/store/src/fts/term_index.rs +++ b/crates/store/src/fts/term_index.rs @@ -41,7 +41,7 @@ pub enum Error { pub type TermId = u32; pub type Result = std::result::Result; -const LENGTH_SIZE: usize = std::mem::size_of::(); +const LENGTH_SIZE: usize = U32_LEN; #[derive(Debug, PartialEq, Eq)] pub struct Term { @@ -290,9 +290,8 @@ impl Serialize for TermIndexBuilder { } // Serialize tokens - let mut bytes = Vec::with_capacity( - terms_len + ((self.items.len() / self.terms.len()) * std::mem::size_of::() * 2), - ); + let mut bytes = + Vec::with_capacity(terms_len + ((self.items.len() / self.terms.len()) * U64_LEN * 2)); bytes.push_leb128(self.terms.len()); for terms in terms { bytes.extend_from_slice(terms.as_bytes()); diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index d0e5de6d..bb66b46c 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -21,22 +21,21 @@ * for more details. */ -use std::{fmt::Display, ops::BitAndAssign}; +use std::{fmt::Display, ops::Range, sync::Arc}; pub mod backend; -pub mod blob; //pub mod fts; +pub mod dispatch; pub mod query; pub mod write; pub use ahash; +use backend::{foundationdb::FdbStore, sqlite::SqliteStore}; pub use blake3; pub use parking_lot; -use query::{filter::StoreQuery, log::StoreLog, sort::StoreSort}; pub use rand; pub use roaring; -use roaring::RoaringBitmap; -use write::{Batch, BitmapClass, ValueClass}; +use write::{BitmapClass, BlobOp, ValueClass}; #[cfg(feature = "rocks")] pub struct Store { @@ -88,6 +87,15 @@ pub struct ValueKey> { pub class: T, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct BlobKey> { + pub account_id: u32, + pub collection: u8, + pub document_id: u32, + pub hash: T, + pub op: BlobOp, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct LogKey { pub account_id: u32, @@ -95,41 +103,33 @@ pub struct LogKey { pub change_id: u64, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum BlobKind { +const BLOB_HASH_LEN: usize = 32; +const U64_LEN: usize = std::mem::size_of::(); +const U32_LEN: usize = std::mem::size_of::(); + +#[derive(Clone, Debug, Default, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub struct BlobHash([u8; BLOB_HASH_LEN]); + +pub type Result = std::result::Result; + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum BlobClass { + Reserved { + account_id: u32, + }, Linked { account_id: u32, collection: u8, document_id: u32, }, - LinkedMaildir { - account_id: u32, - document_id: u32, - }, - Temporary { - account_id: u32, - timestamp: u64, - seq: u32, - }, } -impl BlobKind { - pub fn is_document( - &self, - account_id: u32, - collection: impl Into, - document_id: u32, - ) -> bool { - matches!(self, BlobKind::Linked { - account_id: a, - collection: c, - document_id: d, - } if *a == account_id && *c == collection.into() && *d == document_id) +impl Default for BlobClass { + fn default() -> Self { + BlobClass::Reserved { account_id: 0 } } } -pub type Result = std::result::Result; - #[derive(Debug)] pub enum Error { InternalError(String), @@ -157,138 +157,26 @@ pub const SUBSPACE_BITMAPS: u8 = b'b'; pub const SUBSPACE_VALUES: u8 = b'v'; pub const SUBSPACE_LOGS: u8 = b'l'; pub const SUBSPACE_INDEXES: u8 = b'i'; -pub const SUBSPACE_QUOTAS: u8 = b'q'; +pub const SUBSPACE_BLOBS: u8 = b'o'; +pub const SUBSPACE_ACLS: u8 = b'a'; +pub const SUBSPACE_COUNTERS: u8 = b'c'; -#[async_trait::async_trait] -pub trait StoreInit: Sized { - async fn open(config: &utils::config::Config) -> crate::Result; +pub struct IterateParams { + begin: T, + end: T, + first: bool, + ascending: bool, + values: bool, } #[async_trait::async_trait] -pub trait StorePurge { - async fn purge_bitmaps(&self) -> crate::Result<()>; - async fn purge_account(&self, account_id: u32) -> crate::Result<()>; +pub trait BlobStore: Sync + Send { + async fn get_blob(&self, key: &[u8], range: Range) -> crate::Result>>; + async fn put_blob(&self, key: &[u8], data: &[u8]) -> crate::Result<()>; + async fn delete_blob(&self, key: &[u8]) -> crate::Result; } -#[async_trait::async_trait] -pub trait StoreId { - async fn assign_change_id(&self, account_id: u32) -> crate::Result; - async fn assign_document_id( - &self, - account_id: u32, - collection: impl Into + Sync + Send, - ) -> crate::Result; -} - -#[async_trait::async_trait] -pub trait StoreRead: Sync { - async fn get_value(&self, key: impl Key) -> crate::Result> - where - U: Deserialize + 'static; - - async fn get_values(&self, key: Vec) -> crate::Result>> - where - U: Deserialize + 'static, - { - let mut results = Vec::with_capacity(key.len()); - - for key in key { - results.push(self.get_value(key).await?); - } - - Ok(results) - } - - async fn get_bitmap(&self, key: BitmapKey) - -> crate::Result>; - - async fn get_bitmaps_intersection( - &self, - keys: Vec>, - ) -> crate::Result> { - let mut result: Option = None; - for key in keys { - if let Some(bitmap) = self.get_bitmap(key).await? { - if let Some(result) = &mut result { - result.bitand_assign(&bitmap); - if result.is_empty() { - break; - } - } else { - result = Some(bitmap); - } - } else { - return Ok(None); - } - } - Ok(result) - } - - async fn range_to_bitmap( - &self, - account_id: u32, - collection: u8, - field: u8, - value: Vec, - op: query::Operator, - ) -> crate::Result>; - - async fn sort_index( - &self, - account_id: u32, - collection: impl Into + Sync + Send, - field: impl Into + Sync + Send, - ascending: bool, - cb: impl for<'x> FnMut(&'x [u8], u32) -> crate::Result + Sync + Send, - ) -> crate::Result<()>; - - async fn iterate( - &self, - begin: impl Key, - end: impl Key, - first: bool, - ascending: bool, - cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> crate::Result + Sync + Send, - ) -> crate::Result<()>; - - async fn get_last_change_id( - &self, - account_id: u32, - collection: impl Into + Sync + Send, - ) -> crate::Result>; - - async fn get_counter( - &self, - key: impl Into> + Sync + Send, - ) -> crate::Result; - - #[cfg(feature = "test_mode")] - async fn assert_is_empty(&self); -} - -#[async_trait::async_trait] -pub trait StoreWrite { - async fn write(&self, batch: Batch) -> crate::Result<()>; - /*async fn set_value( - &self, - key: impl Key, - value: impl Serialize + Sync + Send + 'static, - ) -> crate::Result<()>;*/ - #[cfg(feature = "test_mode")] - async fn destroy(&self); -} - -pub trait Store: - StoreInit - + StoreRead - + StoreWrite - + StoreId - + StorePurge - + StoreQuery - + StoreSort - + StoreLog - + Sync - + Send - + 'static -{ +pub enum Store { + SQLite(Arc), + FoundationDb(Arc), } diff --git a/crates/store/src/query/acl.rs b/crates/store/src/query/acl.rs new file mode 100644 index 00000000..56264fef --- /dev/null +++ b/crates/store/src/query/acl.rs @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use crate::{ + write::{key::DeserializeBigEndian, BatchBuilder, Operation, ValueClass, ValueOp}, + Deserialize, Error, IterateParams, Store, ValueKey, U32_LEN, +}; + +pub enum AclQuery { + SharedWith { + grant_account_id: u32, + to_account_id: u32, + to_collection: u8, + }, + HasAccess { + grant_account_id: u32, + }, +} + +#[derive(Debug)] +pub struct AclItem { + pub to_account_id: u32, + pub to_collection: u8, + pub to_document_id: u32, + pub permissions: u64, +} + +impl Store { + pub async fn acl_query(&self, query: AclQuery) -> crate::Result> { + let mut results = Vec::new(); + let (from_key, to_key) = match query { + AclQuery::SharedWith { + grant_account_id, + to_account_id, + to_collection, + } => { + let from_key = ValueKey { + account_id: to_account_id, + collection: to_collection, + document_id: 0, + class: ValueClass::Acl(grant_account_id), + }; + let mut to_key = from_key.clone(); + to_key.document_id = u32::MAX; + + (from_key, to_key) + } + AclQuery::HasAccess { grant_account_id } => ( + ValueKey { + account_id: 0, + collection: 0, + document_id: 0, + class: ValueClass::Acl(grant_account_id), + }, + ValueKey { + account_id: u32::MAX, + collection: u8::MAX, + document_id: u32::MAX, + class: ValueClass::Acl(grant_account_id), + }, + ), + }; + + self.iterate( + IterateParams::new(from_key, to_key).ascending(), + |key, value| { + results.push(AclItem::deserialize(key)?.with_permissions(u64::deserialize(value)?)); + + Ok(true) + }, + ) + .await?; + + Ok(results) + } + + pub async fn acl_revoke_all(&self, account_id: u32) -> crate::Result<()> { + let from_key = ValueKey { + account_id: 0, + collection: 0, + document_id: 0, + class: ValueClass::Acl(0), + }; + let to_key = ValueKey { + account_id: u32::MAX, + collection: u8::MAX, + document_id: u32::MAX, + class: ValueClass::Acl(u32::MAX), + }; + + let mut delete_keys = Vec::new(); + self.iterate( + IterateParams::new(from_key, to_key).ascending().no_values(), + |key, _| { + if account_id == key.deserialize_be_u32(U32_LEN)? { + delete_keys.push(( + ValueClass::Acl(key.deserialize_be_u32(0)?), + AclItem::deserialize(key)?, + )); + } + + Ok(true) + }, + ) + .await?; + + // Remove permissions + let mut batch = BatchBuilder::new(); + batch.with_account_id(account_id); + let mut last_collection = u8::MAX; + for (pos, (class, acl_item)) in delete_keys.into_iter().enumerate() { + if pos > 0 && pos & 511 == 0 { + self.write(batch.build()).await?; + batch = BatchBuilder::new(); + batch.with_account_id(account_id); + last_collection = u8::MAX; + } + if acl_item.to_collection != last_collection { + batch.with_collection(acl_item.to_collection); + last_collection = acl_item.to_collection; + } + batch.update_document(acl_item.to_document_id); + batch.ops.push(Operation::Value { + class, + op: ValueOp::Clear, + }) + } + if !batch.is_empty() { + self.write(batch.build()).await?; + } + + Ok(()) + } +} + +impl Deserialize for AclItem { + fn deserialize(bytes: &[u8]) -> crate::Result { + Ok(AclItem { + to_account_id: bytes.deserialize_be_u32(U32_LEN)?, + to_collection: *bytes + .get(U32_LEN * 2) + .ok_or_else(|| Error::InternalError(format!("Corrupted acl key {bytes:?}")))?, + to_document_id: bytes.deserialize_be_u32((U32_LEN * 2) + 1)?, + permissions: 0, + }) + } +} + +impl AclItem { + fn with_permissions(mut self, permissions: u64) -> Self { + self.permissions = permissions; + self + } +} diff --git a/crates/store/src/query/filter.rs b/crates/store/src/query/filter.rs index c768b242..7dfa0043 100644 --- a/crates/store/src/query/filter.rs +++ b/crates/store/src/query/filter.rs @@ -27,7 +27,7 @@ use ahash::HashSet; use nlp::tokenizers::space::SpaceTokenizer; use roaring::RoaringBitmap; -use crate::{backend::MAX_TOKEN_LENGTH, BitmapKey, StoreRead}; +use crate::{backend::MAX_TOKEN_LENGTH, BitmapKey, Store}; use super::{Filter, ResultSet}; @@ -36,9 +36,8 @@ struct State { bm: Option, } -#[async_trait::async_trait] -pub trait StoreQuery: StoreRead { - async fn filter( +impl Store { + pub async fn filter( &self, account_id: u32, collection: impl Into + Sync + Send, diff --git a/crates/store/src/query/log.rs b/crates/store/src/query/log.rs index d097f50c..20e8c896 100644 --- a/crates/store/src/query/log.rs +++ b/crates/store/src/query/log.rs @@ -23,7 +23,7 @@ use utils::codec::leb128::Leb128Iterator; -use crate::{write::key::DeserializeBigEndian, Error, LogKey, StoreRead}; +use crate::{write::key::DeserializeBigEndian, Error, IterateParams, LogKey, Store, U64_LEN}; #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum Change { @@ -58,9 +58,8 @@ impl Default for Changes { } } -#[async_trait::async_trait] -pub trait StoreLog: StoreRead { - async fn changes( +impl Store { + pub async fn changes( &self, account_id: u32, collection: impl Into + Sync + Send, @@ -88,22 +87,25 @@ pub trait StoreLog: StoreRead { let mut changelog = Changes::default(); - self.iterate(from_key, to_key, false, true, |key, value| { - let change_id = key.deserialize_be_u64(key.len() - std::mem::size_of::())?; - if is_inclusive || change_id != from_change_id { - if changelog.changes.is_empty() { - changelog.from_change_id = change_id; + self.iterate( + IterateParams::new(from_key, to_key).ascending(), + |key, value| { + let change_id = key.deserialize_be_u64(key.len() - U64_LEN)?; + if is_inclusive || change_id != from_change_id { + if changelog.changes.is_empty() { + changelog.from_change_id = change_id; + } + changelog.to_change_id = change_id; + changelog.deserialize(value).ok_or_else(|| { + Error::InternalError(format!( + "Failed to deserialize changelog for [{}/{:?}]: [{:?}]", + account_id, collection, query + )) + })?; } - changelog.to_change_id = change_id; - changelog.deserialize(value).ok_or_else(|| { - Error::InternalError(format!( - "Failed to deserialize changelog for [{}/{:?}]: [{:?}]", - account_id, collection, query - )) - })?; - } - Ok(true) - }) + Ok(true) + }, + ) .await?; if changelog.changes.is_empty() { @@ -117,6 +119,41 @@ pub trait StoreLog: StoreRead { Ok(changelog) } + + pub async fn get_last_change_id( + &self, + account_id: u32, + collection: impl Into + Sync + Send, + ) -> crate::Result> { + let collection = collection.into(); + + let from_key = LogKey { + account_id, + collection, + change_id: u64::MAX, + }; + let to_key = LogKey { + account_id, + collection, + change_id: 0, + }; + + let mut last_change_id = None; + + self.iterate( + IterateParams::new(from_key, to_key) + .descending() + .no_values() + .only_first(), + |key, _| { + last_change_id = key.deserialize_be_u64(key.len() - U64_LEN)?.into(); + Ok(false) + }, + ) + .await?; + + Ok(last_change_id) + } } impl Changes { diff --git a/crates/store/src/query/mod.rs b/crates/store/src/query/mod.rs index b02196bf..d5a9814e 100644 --- a/crates/store/src/query/mod.rs +++ b/crates/store/src/query/mod.rs @@ -21,6 +21,7 @@ * for more details. */ +pub mod acl; pub mod filter; pub mod log; pub mod sort; @@ -29,7 +30,7 @@ use roaring::RoaringBitmap; use crate::{ write::{BitmapClass, TagValue}, - BitmapKey, Serialize, + BitmapKey, IterateParams, Key, Serialize, }; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -285,6 +286,38 @@ impl BitmapKey { } } +impl IterateParams { + pub fn new(begin: T, end: T) -> Self { + IterateParams { + begin, + end, + first: false, + ascending: true, + values: true, + } + } + + pub fn ascending(mut self) -> Self { + self.ascending = true; + self + } + + pub fn descending(mut self) -> Self { + self.ascending = false; + self + } + + pub fn only_first(mut self) -> Self { + self.first = true; + self + } + + pub fn no_values(mut self) -> Self { + self.values = false; + self + } +} + /* #[derive(Debug)] pub struct RawValue { diff --git a/crates/store/src/query/sort.rs b/crates/store/src/query/sort.rs index 1e7228c9..4c7e03ba 100644 --- a/crates/store/src/query/sort.rs +++ b/crates/store/src/query/sort.rs @@ -25,7 +25,7 @@ use std::cmp::Ordering; use ahash::{AHashMap, AHashSet}; -use crate::{write::ValueClass, StoreRead, ValueKey}; +use crate::{write::ValueClass, Store, ValueKey}; use super::{Comparator, ResultSet, SortedResultSet}; @@ -42,9 +42,8 @@ pub struct Pagination { prefix_unique: bool, } -#[async_trait::async_trait] -pub trait StoreSort: StoreRead { - async fn sort( +impl Store { + pub async fn sort( &self, result_set: ResultSet, mut comparators: Vec, diff --git a/crates/store/src/write/assert.rs b/crates/store/src/write/assert.rs index a4f2755e..9ca97014 100644 --- a/crates/store/src/write/assert.rs +++ b/crates/store/src/write/assert.rs @@ -21,7 +21,7 @@ * for more details. */ -use crate::Deserialize; +use crate::{Deserialize, U32_LEN, U64_LEN}; #[derive(Debug, Clone)] pub struct HashedValue { @@ -80,12 +80,8 @@ impl ToAssertValue for &HashedValue { impl AssertValue { pub fn matches(&self, bytes: &[u8]) -> bool { match self { - AssertValue::U32(v) => { - bytes.len() == std::mem::size_of::() && u32::deserialize(bytes).unwrap() == *v - } - AssertValue::U64(v) => { - bytes.len() == std::mem::size_of::() && u64::deserialize(bytes).unwrap() == *v - } + AssertValue::U32(v) => bytes.len() == U32_LEN && u32::deserialize(bytes).unwrap() == *v, + AssertValue::U64(v) => bytes.len() == U64_LEN && u64::deserialize(bytes).unwrap() == *v, AssertValue::Hash(v) => xxhash_rust::xxh3::xxh3_64(bytes) == *v, AssertValue::None => false, } diff --git a/crates/store/src/write/batch.rs b/crates/store/src/write/batch.rs index deec9361..0e0241c7 100644 --- a/crates/store/src/write/batch.rs +++ b/crates/store/src/write/batch.rs @@ -21,9 +21,12 @@ * for more details. */ +use crate::BlobHash; + use super::{ - assert::ToAssertValue, Batch, BatchBuilder, BitmapClass, HasFlag, IntoOperations, Operation, - Serialize, TagValue, ToBitmaps, ValueClass, ValueOp, F_BITMAP, F_CLEAR, F_INDEX, F_VALUE, + assert::ToAssertValue, Batch, BatchBuilder, BitmapClass, BlobOp, HasFlag, IntoOperations, + Operation, Serialize, TagValue, ToBitmaps, ValueClass, ValueOp, F_BITMAP, F_CLEAR, F_INDEX, + F_VALUE, }; impl BatchBuilder { @@ -140,6 +143,15 @@ impl BatchBuilder { self } + pub fn blob(&mut self, hash: BlobHash, op: BlobOp, options: u32) -> &mut Self { + self.ops.push(Operation::Blob { + hash, + op, + set: !options.has_flag(F_CLEAR), + }); + self + } + pub fn add(&mut self, class: impl Into, value: i64) -> &mut Self { self.ops.push(Operation::Value { class: class.into(), diff --git a/crates/store/src/write/blob.rs b/crates/store/src/write/blob.rs new file mode 100644 index 00000000..bc0e9e0e --- /dev/null +++ b/crates/store/src/write/blob.rs @@ -0,0 +1,379 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::sync::Arc; + +use ahash::AHashSet; + +use crate::{ + write::{BatchBuilder, F_CLEAR}, + BlobClass, BlobHash, BlobKey, BlobStore, IterateParams, Store, BLOB_HASH_LEN, U32_LEN, U64_LEN, +}; + +use super::{key::DeserializeBigEndian, now, BlobOp}; + +#[derive(Debug)] +pub struct BlobQuota { + pub bytes: usize, + pub count: usize, +} + +impl Store { + pub async fn blob_hash_exists( + &self, + hash: impl AsRef + Sync + Send, + ) -> crate::Result { + let from_key = BlobKey { + account_id: u32::MAX, + collection: 0, + document_id: 0, + op: BlobOp::Link, + hash: hash.as_ref().clone(), + }; + let to_key = BlobKey { + account_id: u32::MAX, + collection: 1, + document_id: 0, + op: BlobOp::Link, + hash: hash.as_ref().clone(), + }; + + let mut exists = false; + + self.iterate( + IterateParams::new(from_key, to_key) + .ascending() + .no_values() + .only_first(), + |_, _| { + exists = true; + Ok(false) + }, + ) + .await?; + + Ok(exists) + } + + pub async fn blob_hash_quota(&self, account_id: u32) -> crate::Result { + let from_key = BlobKey { + account_id, + collection: 0, + document_id: 0, + op: BlobOp::Reserve { until: 0, size: 0 }, + hash: BlobHash::default(), + }; + let to_key = BlobKey { + account_id: account_id + 1, + collection: 0, + document_id: 0, + op: BlobOp::Reserve { until: 0, size: 0 }, + hash: BlobHash::default(), + }; + + let now = now(); + let mut quota = BlobQuota { bytes: 0, count: 0 }; + + self.iterate( + IterateParams::new(from_key, to_key).ascending().no_values(), + |key, _| { + let until = key.deserialize_be_u64(key.len() - (U64_LEN + U32_LEN))?; + if until > now { + let bytes = key.deserialize_be_u32(key.len() - U32_LEN)? as usize; + if bytes > 0 { + quota.bytes += bytes; + quota.count += 1; + } + } + Ok(true) + }, + ) + .await?; + + Ok(quota) + } + + pub async fn blob_hash_can_read( + &self, + hash: impl AsRef + Sync + Send, + class: impl AsRef + Sync + Send, + ) -> crate::Result { + let (from_key, to_key) = match class.as_ref() { + BlobClass::Reserved { account_id } => ( + BlobKey { + account_id: *account_id, + collection: 0, + document_id: 0, + op: BlobOp::Reserve { until: 0, size: 0 }, + hash: hash.as_ref().clone(), + }, + BlobKey { + account_id: *account_id, + collection: 0, + document_id: 0, + op: BlobOp::Reserve { + until: u64::MAX, + size: 0, + }, + hash: hash.as_ref().clone(), + }, + ), + BlobClass::Linked { + account_id, + collection, + document_id, + } => ( + BlobKey { + account_id: *account_id, + collection: *collection, + document_id: *document_id, + op: BlobOp::Link, + hash: hash.as_ref().clone(), + }, + BlobKey { + account_id: *account_id, + collection: *collection, + document_id: *document_id + 1, + op: BlobOp::Link, + hash: hash.as_ref().clone(), + }, + ), + }; + + let mut has_access = false; + + self.iterate( + IterateParams::new(from_key, to_key) + .ascending() + .no_values() + .only_first(), + |_, _| { + has_access = true; + Ok(false) + }, + ) + .await?; + + Ok(has_access) + } + + pub async fn blob_hash_purge(&self, blob_store: Arc) -> crate::Result<()> { + // Remove expired temporary blobs + let from_key = BlobKey { + account_id: 0, + collection: 0, + document_id: 0, + op: BlobOp::Reserve { until: 0, size: 0 }, + hash: BlobHash::default(), + }; + let to_key = BlobKey { + account_id: u32::MAX, + collection: 0, + document_id: 0, + op: BlobOp::Reserve { until: 0, size: 0 }, + hash: BlobHash::default(), + }; + let mut delete_keys = Vec::new(); + let mut active_hashes = AHashSet::new(); + let now = now(); + self.iterate( + IterateParams::new(from_key, to_key).ascending().no_values(), + |key, _| { + let hash = BlobHash::try_from_hash_slice( + key.get(1 + U32_LEN..1 + U32_LEN + BLOB_HASH_LEN) + .ok_or_else(|| { + crate::Error::InternalError(format!( + "Invalid key {key:?} in blob hash tables" + )) + })?, + ) + .unwrap(); + let until = key.deserialize_be_u64(key.len() - (U64_LEN + U32_LEN))?; + if until < now { + let account_id = key.deserialize_be_u32(1)?; + let size = key.deserialize_be_u32(key.len() - U32_LEN)? as usize; + delete_keys.push(BlobKey { + account_id, + collection: 0, + document_id: 0, + hash, + op: BlobOp::Reserve { until, size }, + }); + } else { + active_hashes.insert(hash); + } + Ok(true) + }, + ) + .await?; + + // Validate linked blobs + let from_key = BlobKey { + account_id: 0, + collection: 0, + document_id: 0, + op: BlobOp::Link, + hash: BlobHash::default(), + }; + let to_key = BlobKey { + account_id: u32::MAX, + collection: u8::MAX, + document_id: u32::MAX, + op: BlobOp::Link, + hash: BlobHash::new_max(), + }; + let mut last_hash = BlobHash::default(); + self.iterate( + IterateParams::new(from_key, to_key).ascending().no_values(), + |key, _| { + let hash = BlobHash::try_from_hash_slice( + key.get(1..1 + BLOB_HASH_LEN).ok_or_else(|| { + crate::Error::InternalError(format!( + "Invalid key {key:?} in blob hash tables" + )) + })?, + ) + .unwrap(); + let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; + + if document_id != u32::MAX { + if last_hash != hash { + last_hash = hash; + } + } else if last_hash != hash && !active_hashes.contains(&hash) { + // Unlinked or expired blob, delete. + delete_keys.push(BlobKey { + account_id: 0, + collection: 0, + document_id: 0, + hash, + op: BlobOp::Commit, + }); + } + + Ok(true) + }, + ) + .await?; + + // Delete expired or unlinked blobs + for key in &delete_keys { + if matches!(key.op, BlobOp::Commit) { + blob_store.delete_blob(key.hash.as_ref()).await?; + } + } + + // Delete hashes + let mut batch = BatchBuilder::new(); + let mut last_account_id = u32::MAX; + for (pos, key) in delete_keys.into_iter().enumerate() { + if pos > 0 && pos & 511 == 0 { + last_account_id = u32::MAX; + self.write(batch.build()).await?; + batch = BatchBuilder::new(); + } + if matches!(key.op, BlobOp::Reserve { .. }) && key.account_id != last_account_id { + batch.with_account_id(key.account_id); + last_account_id = key.account_id; + } + batch.blob(key.hash, key.op, F_CLEAR); + } + if !batch.is_empty() { + self.write(batch.build()).await?; + } + + Ok(()) + } + + pub async fn blob_hash_unlink_account(&self, account_id: u32) -> crate::Result<()> { + // Validate linked blobs + let from_key = BlobKey { + account_id: 0, + collection: 0, + document_id: 0, + op: BlobOp::Link, + hash: BlobHash::default(), + }; + let to_key = BlobKey { + account_id: u32::MAX, + collection: u8::MAX, + document_id: u32::MAX, + op: BlobOp::Link, + hash: BlobHash::new_max(), + }; + let mut delete_keys = Vec::new(); + self.iterate( + IterateParams::new(from_key, to_key).ascending().no_values(), + |key, _| { + let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; + + if document_id != u32::MAX + && key.deserialize_be_u32(1 + BLOB_HASH_LEN)? == account_id + { + delete_keys.push(BlobKey { + account_id, + collection: key[1 + BLOB_HASH_LEN + U32_LEN], + document_id, + hash: BlobHash::try_from_hash_slice( + key.get(1..1 + BLOB_HASH_LEN).ok_or_else(|| { + crate::Error::InternalError(format!( + "Invalid key {key:?} in blob hash tables" + )) + })?, + ) + .unwrap(), + op: BlobOp::Link, + }); + } + + Ok(true) + }, + ) + .await?; + + // Unlink blobs + let mut batch = BatchBuilder::new(); + batch.with_account_id(account_id); + let mut last_collection = u8::MAX; + for (pos, key) in delete_keys.into_iter().enumerate() { + if pos > 0 && pos & 511 == 0 { + self.write(batch.build()).await?; + batch = BatchBuilder::new(); + batch.with_account_id(account_id); + last_collection = u8::MAX; + } + if key.collection != last_collection { + batch.with_collection(key.collection); + last_collection = key.collection; + } + batch + .update_document(key.document_id) + .blob(key.hash, key.op, F_CLEAR); + } + if !batch.is_empty() { + self.write(batch.build()).await?; + } + + Ok(()) + } +} diff --git a/crates/store/src/write/key.rs b/crates/store/src/write/key.rs index ea559308..31eb1b41 100644 --- a/crates/store/src/write/key.rs +++ b/crates/store/src/write/key.rs @@ -25,11 +25,12 @@ use std::{convert::TryInto, hash::Hasher}; use utils::codec::leb128::Leb128_; use crate::{ - backend::MAX_TOKEN_MASK, BitmapKey, Deserialize, Error, IndexKey, IndexKeyPrefix, Key, LogKey, - ValueKey, SUBSPACE_BITMAPS, SUBSPACE_INDEXES, SUBSPACE_LOGS, SUBSPACE_VALUES, + backend::MAX_TOKEN_MASK, BitmapKey, BlobHash, BlobKey, IndexKey, IndexKeyPrefix, Key, LogKey, + ValueKey, BLOB_HASH_LEN, SUBSPACE_ACLS, SUBSPACE_BITMAPS, SUBSPACE_INDEXES, SUBSPACE_LOGS, + SUBSPACE_VALUES, U32_LEN, U64_LEN, }; -use super::{BitmapClass, TagValue, ValueClass}; +use super::{BitmapClass, BlobOp, TagValue, ValueClass}; pub struct KeySerializer { buf: Vec, @@ -110,7 +111,7 @@ impl KeySerialize for u64 { impl DeserializeBigEndian for &[u8] { fn deserialize_be_u32(&self, index: usize) -> crate::Result { - self.get(index..index + std::mem::size_of::()) + self.get(index..index + U32_LEN) .ok_or_else(|| { crate::Error::InternalError( "Index out of range while deserializing u32.".to_string(), @@ -127,7 +128,7 @@ impl DeserializeBigEndian for &[u8] { } fn deserialize_be_u64(&self, index: usize) -> crate::Result { - self.get(index..index + std::mem::size_of::()) + self.get(index..index + U64_LEN) .ok_or_else(|| { crate::Error::InternalError( "Index out of range while deserializing u64.".to_string(), @@ -206,31 +207,40 @@ impl Key for LogKey { impl + Sync + Send> Key for ValueKey { fn subspace(&self) -> u8 { - SUBSPACE_VALUES + if !matches!(self.class.as_ref(), ValueClass::Acl(_)) { + SUBSPACE_VALUES + } else { + SUBSPACE_ACLS + } } fn serialize(&self, include_subspace: bool) -> Vec { - let ks = { - if include_subspace { - KeySerializer::new(self.len() + 2).write(crate::SUBSPACE_VALUES) - } else { - KeySerializer::new(self.len() + 1) - } - }; - match self.class.as_ref() { - ValueClass::Property(field) => ks - .write(self.account_id) - .write(self.collection) - .write_leb128(self.document_id) - .write(*field), - ValueClass::Acl(grant_account_id) => ks - .write(*grant_account_id) - .write(u8::MAX) - .write(self.account_id) - .write(self.collection) - .write(self.document_id), - ValueClass::Named(name) => ks.write(u32::MAX).write(name.as_slice()), + ValueClass::Property(field) => if include_subspace { + KeySerializer::new(U32_LEN * 2 + 3).write(crate::SUBSPACE_VALUES) + } else { + KeySerializer::new(U32_LEN * 2 + 2) + } + .write(self.account_id) + .write(self.collection) + .write_leb128(self.document_id) + .write(*field), + ValueClass::Acl(grant_account_id) => if include_subspace { + KeySerializer::new(U32_LEN * 3 + 2).write(crate::SUBSPACE_ACLS) + } else { + KeySerializer::new(U32_LEN * 3 + 1) + } + .write(*grant_account_id) + .write(self.account_id) + .write(self.collection) + .write(self.document_id), + ValueClass::Named(name) => if include_subspace { + KeySerializer::new(U32_LEN + name.len() + 1).write(crate::SUBSPACE_VALUES) + } else { + KeySerializer::new(U32_LEN + name.len()) + } + .write(u32::MAX) + .write(name.as_slice()), } .finalize() } @@ -302,6 +312,44 @@ impl + Sync + Send> Key for BitmapKey { } } +impl + Sync + Send> Key for BlobKey { + fn serialize(&self, include_subspace: bool) -> Vec { + let ks = { + if include_subspace { + KeySerializer::new(BLOB_HASH_LEN + (U64_LEN * 3) + 1).write(crate::SUBSPACE_BLOBS) + } else { + KeySerializer::new(BLOB_HASH_LEN + (U64_LEN * 3)) + } + }; + + match self.op { + BlobOp::Reserve { until, size } => ks + .write(1u8) + .write(self.account_id) + .write::<&[u8]>(self.hash.as_ref().as_ref()) + .write(until) + .write(size as u32), + BlobOp::Commit => ks + .write(0u8) + .write::<&[u8]>(self.hash.as_ref().as_ref()) + .write(u32::MAX) + .write(0u8) + .write(u32::MAX), + BlobOp::Link => ks + .write(0u8) + .write::<&[u8]>(self.hash.as_ref().as_ref()) + .write(self.account_id) + .write(self.collection) + .write(self.document_id), + } + .finalize() + } + + fn subspace(&self) -> u8 { + crate::SUBSPACE_BLOBS + } +} + const AHASHER: ahash::RandomState = ahash::RandomState::with_seeds( 0xaf1f2242106c64b3, 0x60ca4cfb4b3ed0ce, @@ -359,7 +407,7 @@ impl> BitmapKey { + match self.class.as_ref() { BitmapClass::DocumentIds => 0, BitmapClass::Tag { value, .. } => match value { - TagValue::Id(_) => std::mem::size_of::(), + TagValue::Id(_) => U32_LEN, TagValue::Text(v) => v.len(), TagValue::Static(_) => 1, }, @@ -374,28 +422,8 @@ impl> ValueKey { std::mem::size_of::>() + match self.class.as_ref() { ValueClass::Property(_) => 1, - ValueClass::Acl(_) => std::mem::size_of::(), + ValueClass::Acl(_) => U32_LEN, ValueClass::Named(v) => v.len(), } } } - -pub struct AclKey { - pub to_account_id: u32, - pub to_collection: u8, - //pub grant_account_id: u32, - //pub to_document_id: u32, -} - -impl Deserialize for AclKey { - fn deserialize(bytes: &[u8]) -> crate::Result { - Ok(AclKey { - to_account_id: bytes.deserialize_be_u32(std::mem::size_of::() + 1)?, - to_collection: *bytes - .get((std::mem::size_of::() * 2) + 1) - .ok_or_else(|| Error::InternalError(format!("Corrupted acl key {bytes:?}")))?, - //grant_account_id: bytes.deserialize_be_u32(0)?, - //to_document_id: bytes.deserialize_be_u32((std::mem::size_of::() * 2) + 2)?, - }) - } -} diff --git a/crates/store/src/write/mod.rs b/crates/store/src/write/mod.rs index 90713a61..82f73221 100644 --- a/crates/store/src/write/mod.rs +++ b/crates/store/src/write/mod.rs @@ -21,17 +21,20 @@ * for more details. */ -use std::{collections::HashSet, slice::Iter, time::SystemTime}; +use std::{collections::HashSet, hash::Hash, slice::Iter, time::SystemTime}; use nlp::tokenizers::space::SpaceTokenizer; use utils::codec::leb128::{Leb128Iterator, Leb128Vec}; -use crate::{backend::MAX_TOKEN_LENGTH, Deserialize, Serialize}; +use crate::{ + backend::MAX_TOKEN_LENGTH, BlobClass, BlobHash, Deserialize, Serialize, BLOB_HASH_LEN, +}; use self::assert::AssertValue; pub mod assert; pub mod batch; +pub mod blob; pub mod key; pub mod log; @@ -77,6 +80,11 @@ pub enum Operation { class: BitmapClass, set: bool, }, + Blob { + hash: BlobHash, + op: BlobOp, + set: bool, + }, Log { change_id: u64, collection: u8, @@ -113,6 +121,13 @@ pub enum ValueOp { Clear, } +#[derive(Debug, PartialEq, Clone, Copy, Eq, Hash)] +pub enum BlobOp { + Reserve { until: u64, size: usize }, + Commit, + Link, +} + impl From for TagValue { fn from(value: u32) -> Self { TagValue::Id(value) @@ -455,3 +470,65 @@ impl BitmapClass { } } } + +impl BlobHash { + pub fn new_max() -> Self { + BlobHash([u8::MAX; BLOB_HASH_LEN]) + } + + pub fn try_from_hash_slice(value: &[u8]) -> Result { + value.try_into().map(BlobHash) + } +} + +impl From<&[u8]> for BlobHash { + fn from(value: &[u8]) -> Self { + BlobHash(blake3::hash(value).into()) + } +} + +impl From> for BlobHash { + fn from(value: Vec) -> Self { + value.as_slice().into() + } +} + +impl From<&Vec> for BlobHash { + fn from(value: &Vec) -> Self { + value.as_slice().into() + } +} + +impl AsRef for BlobHash { + fn as_ref(&self) -> &BlobHash { + self + } +} + +impl AsRef<[u8]> for BlobHash { + fn as_ref(&self) -> &[u8] { + self.0.as_ref() + } +} + +impl AsMut<[u8]> for BlobHash { + fn as_mut(&mut self) -> &mut [u8] { + self.0.as_mut() + } +} + +impl AsRef for BlobClass { + fn as_ref(&self) -> &BlobClass { + self + } +} + +impl BlobClass { + pub fn account_id(&self) -> u32 { + match self { + BlobClass::Reserved { account_id } | BlobClass::Linked { account_id, .. } => { + *account_id + } + } + } +} diff --git a/crates/utils/src/codec/base32_custom.rs b/crates/utils/src/codec/base32_custom.rs index 5a03580b..1c35a580 100644 --- a/crates/utils/src/codec/base32_custom.rs +++ b/crates/utils/src/codec/base32_custom.rs @@ -21,7 +21,7 @@ * for more details. */ -use std::slice::Iter; +use std::{io::Write, slice::Iter}; use super::leb128::{Leb128Iterator, Leb128Writer}; @@ -49,6 +49,13 @@ pub struct Base32Writer { } impl Base32Writer { + pub fn from_bytes(bytes: impl AsRef<[u8]>) -> Self { + let bytes = bytes.as_ref(); + let mut writer = Base32Writer::with_capacity(bytes.len()); + writer.write_all(bytes).unwrap(); + writer + } + pub fn with_capacity(capacity: usize) -> Self { Base32Writer { result: String::with_capacity((capacity + 3) / 4 * 5), diff --git a/tests/src/imap/mod.rs b/tests/src/imap/mod.rs index 5e3e1977..d3989889 100644 --- a/tests/src/imap/mod.rs +++ b/tests/src/imap/mod.rs @@ -38,7 +38,6 @@ pub mod thread; use std::{path::PathBuf, sync::Arc, time::Duration}; use ::managesieve::core::ManageSieveSessionManager; -use ::store::StoreWrite; use directory::config::ConfigDirectory; use imap::core::{ImapSessionManager, IMAP}; use imap_proto::ResponseType; diff --git a/tests/src/jmap/auth_acl.rs b/tests/src/jmap/auth_acl.rs index a996fc05..e9598beb 100644 --- a/tests/src/jmap/auth_acl.rs +++ b/tests/src/jmap/auth_acl.rs @@ -39,7 +39,7 @@ use jmap_client::{ }; use jmap_proto::types::id::Id; use std::fmt::Debug; -use store::{ahash::AHashMap, StoreRead}; +use store::ahash::AHashMap; use crate::{ directory::sql::{ @@ -777,7 +777,10 @@ pub async fn test(server: Arc, admin_client: &mut Client) { admin_client.set_default_account_id(&id.to_string()); destroy_all_mailboxes(admin_client).await; } - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } pub fn assert_forbidden(result: Result) { @@ -786,7 +789,7 @@ pub fn assert_forbidden(result: Result) { Err(jmap_client::Error::Method(MethodError { p_type: MethodErrorType::Forbidden })) | Err(jmap_client::Error::Set(SetError { - type_: SetErrorType::Forbidden, + type_: SetErrorType::BlobNotFound | SetErrorType::Forbidden, .. })) ) { diff --git a/tests/src/jmap/auth_limits.rs b/tests/src/jmap/auth_limits.rs index e320bb96..8bede987 100644 --- a/tests/src/jmap/auth_limits.rs +++ b/tests/src/jmap/auth_limits.rs @@ -30,7 +30,6 @@ use jmap_client::{ mailbox::{self}, }; use jmap_proto::types::id::Id; -use store::StoreRead; use crate::{ directory::sql::{create_test_user_with_email, link_test_address}, @@ -203,5 +202,8 @@ pub async fn test(server: Arc, admin_client: &mut Client) { // Destroy test accounts admin_client.set_default_account_id(&account_id); destroy_all_mailboxes(admin_client).await; - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } diff --git a/tests/src/jmap/auth_oauth.rs b/tests/src/jmap/auth_oauth.rs index 64650c0b..2353bbf0 100644 --- a/tests/src/jmap/auth_oauth.rs +++ b/tests/src/jmap/auth_oauth.rs @@ -38,7 +38,7 @@ use jmap_client::{ use jmap_proto::types::id::Id; use reqwest::{header, redirect::Policy}; use serde::de::DeserializeOwned; -use store::{ahash::AHashMap, StoreRead}; +use store::ahash::AHashMap; use crate::{directory::sql::create_test_user_with_email, jmap::mailbox::destroy_all_mailboxes}; @@ -307,7 +307,10 @@ pub async fn test(server: Arc, admin_client: &mut Client) { // Destroy test accounts admin_client.set_default_account_id(john_id); destroy_all_mailboxes(admin_client).await; - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } async fn post_bytes(url: &str, params: &AHashMap) -> Bytes { diff --git a/tests/src/jmap/blob.rs b/tests/src/jmap/blob.rs index 52ce11fc..d6693893 100644 --- a/tests/src/jmap/blob.rs +++ b/tests/src/jmap/blob.rs @@ -27,7 +27,6 @@ use jmap::{mailbox::INBOX_ID, JMAP}; use jmap_client::client::Client; use jmap_proto::types::id::Id; use serde_json::Value; -use store::StoreRead; use crate::{ directory::sql::create_test_user_with_email, @@ -40,11 +39,7 @@ pub async fn test(server: Arc, admin_client: &mut Client) { create_test_user_with_email(directory, "jdoe@example.com", "12345", "John Doe").await; let account_id = Id::from(server.get_account_id("jdoe@example.com").await.unwrap()); - server - .store - .delete_account_blobs(account_id.document_id()) - .await - .unwrap(); + server.store.blob_hash_expire_all().await; // Blob/set simple test let response = jmap_json_request( @@ -194,11 +189,7 @@ pub async fn test(server: Arc, admin_client: &mut Client) { ); } - server - .store - .delete_account_blobs(account_id.document_id()) - .await - .unwrap(); + server.store.blob_hash_expire_all().await; // Blob/upload Complex Example let response = jmap_json_request( @@ -292,11 +283,7 @@ pub async fn test(server: Arc, admin_client: &mut Client) { "Pointer {pointer:?} Response: {response:?}", ); } - server - .store - .delete_account_blobs(account_id.document_id()) - .await - .unwrap(); + server.store.blob_hash_expire_all().await; // Blob/get Example with Range and Encoding Errors let response = jmap_json_request( @@ -435,11 +422,7 @@ pub async fn test(server: Arc, admin_client: &mut Client) { "Pointer {pointer:?} Response: {response:?}", ); } - server - .store - .delete_account_blobs(account_id.document_id()) - .await - .unwrap(); + server.store.blob_hash_expire_all().await; // Blob/lookup admin_client.set_default_account_id(account_id.to_string()); @@ -499,12 +482,15 @@ pub async fn test(server: Arc, admin_client: &mut Client) { .map(|arr| arr.len()) .unwrap_or_default(), 1, - "Pointer {pointer:?} Response: {response:?}", + "Pointer {pointer:?} Response: {response:#?}", ); } // Remove test data admin_client.set_default_account_id(account_id.to_string()); destroy_all_mailboxes(admin_client).await; - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } diff --git a/tests/src/jmap/delivery.rs b/tests/src/jmap/delivery.rs index a5022053..9ec1e8a6 100644 --- a/tests/src/jmap/delivery.rs +++ b/tests/src/jmap/delivery.rs @@ -26,7 +26,7 @@ use std::{sync::Arc, time::Duration}; use jmap::JMAP; use jmap_client::client::Client; use jmap_proto::types::{collection::Collection, id::Id}; -use store::StoreRead; + use tokio::{ io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines, ReadHalf, WriteHalf}, net::TcpStream, @@ -248,7 +248,10 @@ pub async fn test(server: Arc, client: &mut Client) { client.set_default_account_id(account_id); destroy_all_mailboxes(client).await; } - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } pub struct SmtpConnection { diff --git a/tests/src/jmap/email_changes.rs b/tests/src/jmap/email_changes.rs index cbc404de..d7947b90 100644 --- a/tests/src/jmap/email_changes.rs +++ b/tests/src/jmap/email_changes.rs @@ -32,7 +32,6 @@ use jmap_proto::{ use store::{ ahash::AHashSet, write::{log::ChangeLogBuilder, BatchBuilder}, - StoreRead, StoreWrite, }; pub async fn test(server: Arc, client: &mut Client) { @@ -316,7 +315,10 @@ pub async fn test(server: Arc, client: &mut Client) { assert_eq!(created, vec![2, 3, 11, 12]); assert_eq!(changes.updated(), Vec::::new()); assert_eq!(changes.destroyed(), Vec::::new()); - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } #[derive(Debug, Clone, Copy)] diff --git a/tests/src/jmap/email_copy.rs b/tests/src/jmap/email_copy.rs index 681c976b..585a5a8d 100644 --- a/tests/src/jmap/email_copy.rs +++ b/tests/src/jmap/email_copy.rs @@ -26,7 +26,6 @@ use std::sync::Arc; use jmap::JMAP; use jmap_client::{client::Client, mailbox::Role}; use jmap_proto::types::id::Id; -use store::StoreRead; use crate::jmap::mailbox::destroy_all_mailboxes; @@ -117,5 +116,8 @@ pub async fn test(server: Arc, client: &mut Client) { destroy_all_mailboxes(client).await; client.set_default_account_id(Id::new(2).to_string()); destroy_all_mailboxes(client).await; - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } diff --git a/tests/src/jmap/email_get.rs b/tests/src/jmap/email_get.rs index c346e748..366c9034 100644 --- a/tests/src/jmap/email_get.rs +++ b/tests/src/jmap/email_get.rs @@ -30,7 +30,6 @@ use jmap_client::{ }; use jmap_proto::types::id::Id; use mail_parser::HeaderName; -use store::StoreRead; use crate::jmap::{mailbox::destroy_all_mailboxes, replace_blob_ids}; @@ -179,7 +178,10 @@ pub async fn test(server: Arc, client: &mut Client) { destroy_all_mailboxes(client).await; - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } pub fn all_headers() -> Vec { diff --git a/tests/src/jmap/email_parse.rs b/tests/src/jmap/email_parse.rs index 9f8606c8..aacdaa2e 100644 --- a/tests/src/jmap/email_parse.rs +++ b/tests/src/jmap/email_parse.rs @@ -30,7 +30,6 @@ use jmap_client::{ mailbox::Role, }; use jmap_proto::types::id::Id; -use store::StoreRead; use crate::jmap::{email_get::all_headers, mailbox::destroy_all_mailboxes, replace_blob_ids}; @@ -245,5 +244,8 @@ pub async fn test(server: Arc, client: &mut Client) { destroy_all_mailboxes(client).await; - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } diff --git a/tests/src/jmap/email_query.rs b/tests/src/jmap/email_query.rs index 41f14d24..a993030e 100644 --- a/tests/src/jmap/email_query.rs +++ b/tests/src/jmap/email_query.rs @@ -35,8 +35,8 @@ use jmap_client::{ }; use jmap_proto::types::{collection::Collection, id::Id}; use mail_parser::HeaderName; -use store::StoreRead; -use store::{ahash::AHashMap, write::BatchBuilder, StoreWrite}; + +use store::{ahash::AHashMap, write::BatchBuilder}; const MAX_THREADS: usize = 100; const MAX_MESSAGES: usize = 1000; @@ -115,7 +115,10 @@ pub async fn test(server: Arc, client: &mut Client, insert: bool) { .unwrap(); destroy_all_mailboxes(client).await; - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } pub async fn query(client: &mut Client) { diff --git a/tests/src/jmap/email_query_changes.rs b/tests/src/jmap/email_query_changes.rs index 4bee2d5d..a77acf21 100644 --- a/tests/src/jmap/email_query_changes.rs +++ b/tests/src/jmap/email_query_changes.rs @@ -30,11 +30,10 @@ use jmap_client::{ }; use jmap_proto::types::{collection::Collection, id::Id, property::Property, state::State}; use std::sync::Arc; -use store::StoreRead; + use store::{ ahash::{AHashMap, AHashSet}, write::{log::ChangeLogBuilder, BatchBuilder, F_BITMAP, F_CLEAR, F_VALUE}, - StoreWrite, }; use crate::jmap::{ @@ -288,7 +287,10 @@ pub async fn test(server: Arc, client: &mut Client) { } server.store.write(batch.build_batch()).await.unwrap(); - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } #[derive(Debug, Clone)] diff --git a/tests/src/jmap/email_search_snippet.rs b/tests/src/jmap/email_search_snippet.rs index 0b09a4cf..91baacfd 100644 --- a/tests/src/jmap/email_search_snippet.rs +++ b/tests/src/jmap/email_search_snippet.rs @@ -28,7 +28,6 @@ use jmap::{mailbox::INBOX_ID, JMAP}; use jmap_client::{client::Client, core::query, email::query::Filter}; use jmap_proto::types::id::Id; use store::ahash::AHashMap; -use store::StoreRead; pub async fn test(server: Arc, client: &mut Client) { println!("Running SearchSnippet tests..."); @@ -180,5 +179,8 @@ pub async fn test(server: Arc, client: &mut Client) { // Destroy test data destroy_all_mailboxes(client).await; - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } diff --git a/tests/src/jmap/email_set.rs b/tests/src/jmap/email_set.rs index 8d793475..7cf4a078 100644 --- a/tests/src/jmap/email_set.rs +++ b/tests/src/jmap/email_set.rs @@ -33,7 +33,6 @@ use jmap_client::{ Error, Set, }; use jmap_proto::types::id::Id; -use store::StoreRead; use super::{find_values, replace_blob_ids, replace_boundaries, replace_values}; @@ -48,7 +47,10 @@ pub async fn test(server: Arc, client: &mut Client) { destroy_all_mailboxes(client).await; - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } async fn create(client: &mut Client, mailbox_id: &str) { diff --git a/tests/src/jmap/email_submission.rs b/tests/src/jmap/email_submission.rs index 4dc26b0e..d38e8814 100644 --- a/tests/src/jmap/email_submission.rs +++ b/tests/src/jmap/email_submission.rs @@ -37,7 +37,7 @@ use std::{ time::{Duration, Instant}, }; use store::parking_lot::Mutex; -use store::StoreRead; + use tokio::{ io::{AsyncBufReadExt, AsyncWriteExt, BufReader}, net::TcpListener, @@ -471,7 +471,10 @@ pub async fn test(server: Arc, client: &mut Client) { client.email_submission_destroy(&id).await.unwrap(); } destroy_all_mailboxes(client).await; - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } pub fn spawn_mock_smtp_server() -> (mpsc::Receiver, Arc>) { @@ -578,7 +581,7 @@ pub fn spawn_mock_smtp_server() -> (mpsc::Receiver, Arc, admin_client: &mut Client) { @@ -130,7 +130,10 @@ pub async fn test(server: Arc, admin_client: &mut Client) { assert_ping(&mut event_rx).await; destroy_all_mailboxes(admin_client).await; - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } async fn assert_state( diff --git a/tests/src/jmap/mailbox.rs b/tests/src/jmap/mailbox.rs index 49b72069..c6479a6f 100644 --- a/tests/src/jmap/mailbox.rs +++ b/tests/src/jmap/mailbox.rs @@ -36,7 +36,7 @@ use jmap_client::{ use jmap_proto::types::{id::Id, state::State}; use serde::{Deserialize, Serialize}; use store::ahash::AHashMap; -use store::StoreRead; + pub async fn test(server: Arc, client: &mut Client) { println!("Running Mailbox tests..."); @@ -606,7 +606,10 @@ pub async fn test(server: Arc, client: &mut Client) { destroy_all_mailboxes(client).await; client.set_default_account_id(Id::from(1u64)); - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } async fn create_test_mailboxes(client: &mut Client) -> AHashMap { diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 5003d987..fc695c39 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -30,7 +30,6 @@ use jmap_client::client::{Client, Credentials}; use jmap_proto::types::id::Id; use reqwest::header; use smtp::core::{SmtpSessionManager, SMTP}; -use store::StoreWrite; use tokio::sync::{mpsc, watch}; use utils::{config::ServerProtocol, UnwrapFailure}; @@ -253,9 +252,9 @@ pub async fn jmap_tests() { sieve_script::test(params.server.clone(), &mut params.client).await; vacation_response::test(params.server.clone(), &mut params.client).await; email_submission::test(params.server.clone(), &mut params.client).await; - websocket::test(params.server.clone(), &mut params.client).await;*/ + websocket::test(params.server.clone(), &mut params.client).await; quota::test(params.server.clone(), &mut params.client).await; - crypto::test(params.server.clone(), &mut params.client).await; + crypto::test(params.server.clone(), &mut params.client).await;*/ blob::test(params.server.clone(), &mut params.client).await; if delete { diff --git a/tests/src/jmap/push_subscription.rs b/tests/src/jmap/push_subscription.rs index 9a97044a..3fd6c450 100644 --- a/tests/src/jmap/push_subscription.rs +++ b/tests/src/jmap/push_subscription.rs @@ -46,7 +46,7 @@ use jmap_client::{client::Client, mailbox::Role, push_subscription::Keys}; use jmap_proto::types::{id::Id, type_state::DataType}; use reqwest::header::CONTENT_ENCODING; use store::ahash::AHashSet; -use store::StoreRead; + use tokio::{net::TcpStream, sync::mpsc}; use utils::listener::SessionData; @@ -219,7 +219,10 @@ pub async fn test(server: Arc, admin_client: &mut Client) { destroy_all_mailboxes(admin_client).await; - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } #[derive(Clone)] diff --git a/tests/src/jmap/quota.rs b/tests/src/jmap/quota.rs index aec39949..a70910c6 100644 --- a/tests/src/jmap/quota.rs +++ b/tests/src/jmap/quota.rs @@ -37,7 +37,6 @@ use jmap_client::{ email::EmailBodyPart, }; use jmap_proto::types::{collection::Collection, id::Id}; -use store::StoreRead; pub async fn test(server: Arc, admin_client: &mut Client) { println!("Running quota tests..."); @@ -50,24 +49,15 @@ pub async fn test(server: Arc, admin_client: &mut Client) { add_to_group(directory, "robert@example.com", "jdoe@example.com").await; // Delete temporary blobs from previous tests - server - .store - .delete_account_blobs(account_id.document_id()) - .await - .unwrap(); - server - .store - .delete_account_blobs(other_account_id.document_id()) - .await - .unwrap(); + server.store.blob_hash_expire_all().await; // Test temporary blob quota (3 files) DISABLE_UPLOAD_QUOTA.store(false, std::sync::atomic::Ordering::Relaxed); let client = test_account_login("robert@example.com", "aabbcc").await; - for _ in 0..3 { + for i in 0..3 { assert_eq!( client - .upload(None, vec![b'A'; 1024], None) + .upload(None, vec![b'A' + i; 1024], None) .await .unwrap() .size(), @@ -75,24 +65,20 @@ pub async fn test(server: Arc, admin_client: &mut Client) { ); } match client - .upload(None, vec![b'A'; 1024], None) + .upload(None, vec![b'Z'; 1024], None) .await .unwrap_err() { jmap_client::Error::Problem(err) if err.detail().unwrap().contains("quota") => (), other => panic!("Unexpected error: {:?}", other), } - server - .store - .delete_account_blobs(account_id.document_id()) - .await - .unwrap(); + server.store.blob_hash_expire_all().await; // Test temporary blob quota (50000 bytes) - for _ in 0..2 { + for i in 0..2 { assert_eq!( client - .upload(None, vec![b'A'; 25000], None) + .upload(None, vec![b'a' + i; 25000], None) .await .unwrap() .size(), @@ -100,18 +86,14 @@ pub async fn test(server: Arc, admin_client: &mut Client) { ); } match client - .upload(None, vec![b'A'; 1024], None) + .upload(None, vec![b'z'; 1024], None) .await .unwrap_err() { jmap_client::Error::Problem(err) if err.detail().unwrap().contains("quota") => (), other => panic!("Unexpected error: {:?}", other), } - server - .store - .delete_account_blobs(account_id.document_id()) - .await - .unwrap(); + server.store.blob_hash_expire_all().await; // Test JMAP Quotas extension let response = jmap_raw_request( @@ -338,7 +320,10 @@ pub async fn test(server: Arc, admin_client: &mut Client) { admin_client.set_default_account_id(account_id.to_string()); destroy_all_mailboxes(admin_client).await; } - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } fn assert_over_quota(result: Result) { diff --git a/tests/src/jmap/sieve_script.rs b/tests/src/jmap/sieve_script.rs index ae7d3d9e..1f466a25 100644 --- a/tests/src/jmap/sieve_script.rs +++ b/tests/src/jmap/sieve_script.rs @@ -36,7 +36,6 @@ use std::{ sync::Arc, time::{Duration, Instant}, }; -use store::StoreRead; use crate::{ directory::sql::create_test_user_with_email, @@ -487,7 +486,10 @@ pub async fn test(server: Arc, client: &mut Client) { client.sieve_script_destroy(&id).await.unwrap(); } destroy_all_mailboxes(client).await; - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } fn get_script(name: &str) -> Vec { diff --git a/tests/src/jmap/stress_test.rs b/tests/src/jmap/stress_test.rs index efafcfcc..0fbbdc01 100644 --- a/tests/src/jmap/stress_test.rs +++ b/tests/src/jmap/stress_test.rs @@ -33,7 +33,6 @@ use jmap_client::{ }; use jmap_proto::types::{collection::Collection, id::Id, property::Property}; use store::rand::{self, Rng}; -use store::StoreRead; const TEST_USER_ID: u32 = 1; const NUM_PASSES: usize = 1; @@ -256,7 +255,10 @@ async fn email_tests(server: Arc, client: Arc) { destroy_all_mailboxes(&client).await; - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } } @@ -329,7 +331,10 @@ async fn mailbox_tests(server: Arc, client: Arc) { join_all(futures).await; destroy_all_mailboxes(&client).await; - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } async fn create_mailbox(client: &Client, mailbox: &str) -> Vec { diff --git a/tests/src/jmap/thread_get.rs b/tests/src/jmap/thread_get.rs index 46cbbf61..68a6e202 100644 --- a/tests/src/jmap/thread_get.rs +++ b/tests/src/jmap/thread_get.rs @@ -27,7 +27,6 @@ use crate::jmap::mailbox::destroy_all_mailboxes; use jmap::JMAP; use jmap_client::{client::Client, mailbox::Role}; use jmap_proto::types::id::Id; -use store::StoreRead; pub async fn test(server: Arc, client: &mut Client) { println!("Running Email Thread tests..."); @@ -67,5 +66,8 @@ pub async fn test(server: Arc, client: &mut Client) { ); destroy_all_mailboxes(client).await; - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } diff --git a/tests/src/jmap/thread_merge.rs b/tests/src/jmap/thread_merge.rs index d5e1033f..b31b0b27 100644 --- a/tests/src/jmap/thread_merge.rs +++ b/tests/src/jmap/thread_merge.rs @@ -28,7 +28,6 @@ use jmap::JMAP; use jmap_client::{client::Client, email, mailbox::Role}; use jmap_proto::types::id::Id; use store::ahash::{AHashMap, AHashSet}; -use store::StoreRead; pub async fn test(server: Arc, client: &mut Client) { println!("Running Email Merge Threads tests..."); @@ -204,7 +203,10 @@ pub async fn test(server: Arc, client: &mut Client) { } } - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } fn build_message(message: usize, in_reply_to: Option, thread_num: usize) -> String { diff --git a/tests/src/jmap/vacation_response.rs b/tests/src/jmap/vacation_response.rs index 9fa823ba..d9801e63 100644 --- a/tests/src/jmap/vacation_response.rs +++ b/tests/src/jmap/vacation_response.rs @@ -26,7 +26,6 @@ use jmap::JMAP; use jmap_client::client::Client; use jmap_proto::types::id::Id; use std::{sync::Arc, time::Instant}; -use store::StoreRead; use crate::{ directory::sql::create_test_user_with_email, @@ -174,5 +173,8 @@ pub async fn test(server: Arc, client: &mut Client) { // Remove test data client.vacation_response_destroy().await.unwrap(); destroy_all_mailboxes(client).await; - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } diff --git a/tests/src/jmap/websocket.rs b/tests/src/jmap/websocket.rs index b3dc5653..e908afcd 100644 --- a/tests/src/jmap/websocket.rs +++ b/tests/src/jmap/websocket.rs @@ -35,7 +35,7 @@ use jmap_client::{ }; use jmap_proto::types::id::Id; use std::{sync::Arc, time::Duration}; -use store::StoreRead; + use tokio::sync::mpsc; use crate::{ @@ -126,7 +126,10 @@ pub async fn test(server: Arc, admin_client: &mut Client) { admin_client.set_default_account_id(account_id); destroy_all_mailboxes(admin_client).await; - server.store.assert_is_empty().await; + server + .store + .assert_is_empty(server.blob_store.clone()) + .await; } async fn expect_response( diff --git a/tests/src/store/assign_id.rs b/tests/src/store/assign_id.rs index 430845c2..eed81298 100644 --- a/tests/src/store/assign_id.rs +++ b/tests/src/store/assign_id.rs @@ -27,7 +27,7 @@ use store::ahash::AHashSet; use store::{write::BatchBuilder, Store}; -pub async fn test(db: Arc) { +pub async fn test(db: Arc) { println!("Running Store ID assignment tests..."); store::backend::foundationdb::write::ID_ASSIGNMENT_EXPIRY @@ -42,7 +42,7 @@ pub async fn test(db: Arc) { .store(60 * 60, std::sync::atomic::Ordering::Relaxed); } -async fn test_1(db: Arc) { +async fn test_1(db: Arc) { // Test change id assignment let mut handles = Vec::new(); let mut expected_ids = HashSet::new(); @@ -66,7 +66,7 @@ async fn test_1(db: Arc) { db.destroy().await; } -async fn test_2(db: Arc) { +async fn test_2(db: Arc) { // Test document id assignment for wait_for_expiry in [true, false] { let mut handles = Vec::new(); @@ -102,7 +102,7 @@ async fn test_2(db: Arc) { db.destroy().await; } -async fn test_3(db: Arc) { +async fn test_3(db: Arc) { // Create document ids and try reassigning let mut expected_ids = AHashSet::new(); let mut batch = BatchBuilder::new(); @@ -133,7 +133,7 @@ async fn test_3(db: Arc) { db.destroy().await; } -async fn test_4(db: Arc) { +async fn test_4(db: Arc) { // Try reassigning deleted ids let mut expected_ids = AHashSet::new(); let mut batch = BatchBuilder::new(); diff --git a/tests/src/store/blob.rs b/tests/src/store/blob.rs index 808032fb..ed59ca55 100644 --- a/tests/src/store/blob.rs +++ b/tests/src/store/blob.rs @@ -21,7 +21,7 @@ * for more details. */ -use store::{write::now, BlobKind, Store}; +use store::{write::now, BlobHash, Store}; use utils::config::Config; use crate::store::TempDir; @@ -89,7 +89,7 @@ async fn test_blob(store: impl Store) { store.purge_tmp_blobs(0).await.unwrap(); // Store and fetch - let kind = BlobKind::LinkedMaildir { + let kind = BlobHash::LinkedMaildir { account_id: 0, document_id: 0, }; @@ -106,13 +106,13 @@ async fn test_blob(store: impl Store) { assert!(store.get_blob(&kind, 0..u32::MAX).await.unwrap().is_none()); // Copy - let src_kind = BlobKind::LinkedMaildir { + let src_kind = BlobHash::LinkedMaildir { account_id: 0, document_id: 1, }; store.put_blob(&src_kind, DATA).await.unwrap(); for id in 0..4 { - let dest_kind = BlobKind::LinkedMaildir { + let dest_kind = BlobHash::LinkedMaildir { account_id: 1, document_id: id, }; @@ -135,7 +135,7 @@ async fn test_blob(store: impl Store) { let now = now(); let mut tmp_kinds = Vec::new(); for i in 1..=3 { - let tmp_kind = BlobKind::Temporary { + let tmp_kind = BlobHash::Temporary { account_id: 2, timestamp: now - (i * 5), seq: 0, @@ -175,7 +175,7 @@ async fn test_blob(store: impl Store) { for id in 0..4 { assert!(store .get_blob( - &BlobKind::LinkedMaildir { + &BlobHash::LinkedMaildir { account_id: 1, document_id: id, }, diff --git a/tests/src/store/mod.rs b/tests/src/store/mod.rs index 3df084be..1243cc63 100644 --- a/tests/src/store/mod.rs +++ b/tests/src/store/mod.rs @@ -29,7 +29,7 @@ pub mod query; use std::{io::Read, sync::Arc}; use ::store::Store; -use store::StoreWrite; + use utils::config::Config; pub struct TempDir { diff --git a/tests/src/store/query.rs b/tests/src/store/query.rs index a3d9796d..6e0c3d32 100644 --- a/tests/src/store/query.rs +++ b/tests/src/store/query.rs @@ -28,7 +28,7 @@ use std::{ use jmap_proto::types::keyword::Keyword; use nlp::language::Language; -use store::{ahash::AHashMap, query::sort::Pagination, write::ValueClass, StoreWrite}; +use store::{ahash::AHashMap, query::sort::Pagination, write::ValueClass}; use store::{ query::{Comparator, Filter}, @@ -94,7 +94,7 @@ const FIELDS_OPTIONS: [FieldType; 20] = [ ]; #[allow(clippy::mutex_atomic)] -pub async fn test(db: Arc, do_insert: bool) { +pub async fn test(db: Arc, do_insert: bool) { println!("Running Store query tests..."); let pool = rayon::ThreadPoolBuilder::new() @@ -215,7 +215,7 @@ pub async fn test(db: Arc, do_insert: bool) { test_sort(db).await; } -pub async fn test_filter(db: Arc) { +pub async fn test_filter(db: Arc) { /* let mut fields = AHashMap::default(); for (field_num, field) in FIELDS.iter().enumerate() { @@ -361,7 +361,7 @@ pub async fn test_filter(db: Arc) { */ } -pub async fn test_sort(db: Arc) { +pub async fn test_sort(db: Arc) { let mut fields = AHashMap::default(); for (field_num, field) in FIELDS.iter().enumerate() { fields.insert(field.to_string(), field_num as u8);