filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Database schema optimization - part 13 (fixes #1882 fixes #2415)

Browse Source
This commit is contained in:
mdecimus
2025-11-25 12:14:51 +01:00
parent c7fc16d9a2
commit 2b614aa536
62 changed files with 1661 additions and 2264 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
tests/src/store/blob.rs
View File

@@ -4,15 +4,18 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use ahash::AHashMap;
use store::{
BlobStore, SerializeInfallible, Stores,
write::{BatchBuilder, BlobOp, blob::BlobQuota, now},
};
use types::{blob::BlobClass, blob_hash::BlobHash, collection::Collection};
use utils::config::Config;
use crate::store::{CONFIG, TempDir, cleanup::store_destroy};
use ahash::AHashMap;
use common::{Core, Inner, Server, config::storage::Storage};
use email::message::metadata::MessageMetadata;
use http::management::stores::destroy_account_blobs;
use std::sync::Arc;
use store::{
BlobStore, Serialize, SerializeInfallible, Stores,
write::{Archiver, BatchBuilder, BlobLink, BlobOp, ValueClass, blob::BlobQuota, now},
};
use types::{blob::BlobClass, blob_hash::BlobHash, collection::Collection, field::EmailField};
use utils::config::Config;
#[tokio::test]
pub async fn blob_tests() {
@@ -34,6 +37,17 @@ pub async fn blob_tests() {
// Test internal blob store
let blob_store: BlobStore = store.clone().into();
let server = Server {
inner: Arc::new(Inner::default()),
core: Arc::new(Core {
storage: Storage {
data: store.clone(),
blob: blob_store.clone(),
..Default::default()
},
..Default::default()
}),
};
// Blob hash exists
let hash = BlobHash::generate(b"abc".as_slice());
@@ -46,8 +60,8 @@ pub async fn blob_tests() {
BatchBuilder::new()
.with_account_id(0)
.set(
BlobOp::Reserve {
until,
BlobOp::Link {
to: BlobLink::Temporary { until },
hash: hash.clone(),
},
1024u32.serialize(),
@@ -165,26 +179,54 @@ pub async fn blob_tests() {
.enumerate()
{
let hash = BlobHash::generate(blob.as_slice());
let blob_op = if let Some(until) = expiry_times.get(blob) {
BlobOp::Reserve {
until: *until,
hash: hash.clone(),
let mut batch = BatchBuilder::new();
batch
.with_account_id(if document_id > 0 { 0 } else { 1 })
.with_collection(Collection::Email)
.with_document(document_id as u32);
if let Some(until) = expiry_times.get(blob) {
if !blob_value.is_empty() {
batch.set(
BlobOp::Quota {
hash: hash.clone(),
until: *until,
},
blob_value,
);
}
batch.set(
BlobOp::Link {
hash: hash.clone(),
to: BlobLink::Temporary { until: *until },
},
vec![],
);
} else {
BlobOp::Link { hash: hash.clone() }
batch
.set(
BlobOp::Link {
hash: hash.clone(),
to: BlobLink::Document,
},
vec![],
)
.set(
ValueClass::Property(EmailField::Metadata.into()),
Archiver::new(MessageMetadata {
contents: Default::default(),
rcvd_attach: Default::default(),
blob_hash: hash.clone(),
blob_body_offset: Default::default(),
preview: Default::default(),
raw_headers: Default::default(),
})
.serialize()
.unwrap(),
);
};
store
.write(
BatchBuilder::new()
.with_account_id(if document_id > 0 { 0 } else { 1 })
.with_collection(Collection::Email)
.with_document(document_id as u32)
.set(blob_op, blob_value)
.set(BlobOp::Commit { hash: hash.clone() }, vec![])
.build_all(),
)
.await
.unwrap();
batch.set(BlobOp::Commit { hash: hash.clone() }, vec![]);
store.write(batch.build_all()).await.unwrap();
blob_store
.put_blob(hash.as_ref(), blob.as_slice())
.await
@@ -294,6 +336,7 @@ pub async fn blob_tests() {
.with_document(2)
.clear(BlobOp::Link {
hash: BlobHash::generate(b"789".as_slice()),
to: BlobLink::Document,
})
.build_all(),
)
@@ -360,7 +403,7 @@ pub async fn blob_tests() {
}
// Unlink all blobs from accountId 1 and purge
store.blob_hash_unlink_account(1).await.unwrap();
destroy_account_blobs(&server, 1).await.unwrap();
store.purge_blobs(blob_store.clone()).await.unwrap();
// Make sure only accountId 0's blobs are left

70
tests/src/store/cleanup.rs
View File

@@ -10,6 +10,7 @@ use store::{
*,
};
use trc::AddContext;
use types::blob_hash::{BLOB_HASH_LEN, BlobHash};
pub async fn store_destroy(store: &Store) {
store_destroy_sql_indexes(store).await;
@@ -19,7 +20,7 @@ pub async fn store_destroy(store: &Store) {
SUBSPACE_DIRECTORY,
SUBSPACE_TASK_QUEUE,
SUBSPACE_INDEXES,
SUBSPACE_BLOB_RESERVE,
SUBSPACE_BLOB_EXTRA,
SUBSPACE_BLOB_LINK,
SUBSPACE_LOGS,
SUBSPACE_IN_MEMORY_COUNTER,
@@ -102,18 +103,17 @@ pub async fn store_blob_expire_all(store: &Store) {
account_id: 0,
collection: 0,
document_id: 0,
class: ValueClass::Blob(BlobOp::Reserve {
hash: types::blob_hash::BlobHash::default(),
until: 0,
class: ValueClass::Blob(BlobOp::Commit {
hash: BlobHash::default(),
}),
};
let to_key = ValueKey {
account_id: u32::MAX,
collection: 0,
document_id: 0,
class: ValueClass::Blob(BlobOp::Reserve {
hash: types::blob_hash::BlobHash::default(),
until: 0,
collection: u8::MAX,
document_id: u32::MAX,
class: ValueClass::Blob(BlobOp::Link {
hash: BlobHash::new_max(),
to: BlobLink::Document,
}),
};
let mut batch = BatchBuilder::new();
@@ -122,25 +122,31 @@ pub async fn store_blob_expire_all(store: &Store) {
.iterate(
IterateParams::new(from_key, to_key).ascending().no_values(),
|key, _| {
let account_id = key.deserialize_be_u32(0).caused_by(trc::location!())?;
if account_id != last_account_id {
last_account_id = account_id;
batch.with_account_id(account_id);
}
if key.len() == BLOB_HASH_LEN + U32_LEN + U64_LEN {
let account_id = key
.deserialize_be_u32(BLOB_HASH_LEN)
.caused_by(trc::location!())?;
if account_id != last_account_id {
last_account_id = account_id;
batch.with_account_id(account_id);
}
let hash =
BlobHash::try_from_hash_slice(key.get(..BLOB_HASH_LEN).unwrap()).unwrap();
let until = key
.deserialize_be_u64(BLOB_HASH_LEN + U32_LEN)
.caused_by(trc::location!())?;
batch.any_op(Operation::Value {
class: ValueClass::Blob(BlobOp::Reserve {
hash: types::blob_hash::BlobHash::try_from_hash_slice(
key.get(U32_LEN..U32_LEN + types::blob_hash::BLOB_HASH_LEN)
.unwrap(),
)
.unwrap(),
until: key
.deserialize_be_u64(key.len() - U64_LEN)
.caused_by(trc::location!())?,
}),
op: ValueOp::Clear,
});
batch
.clear(ValueClass::Blob(BlobOp::Link {
hash: hash.clone(),
to: BlobLink::Temporary { until },
}))
.clear(ValueClass::Blob(BlobOp::Quota {
hash: hash.clone(),
until,
}))
.clear(ValueClass::Blob(BlobOp::Undelete { hash, until }));
}
Ok(true)
},
@@ -211,7 +217,7 @@ pub async fn store_lookup_expire_all(store: &Store) {
}
#[allow(unused_variables)]
pub async fn store_assert_is_empty(store: &Store, blob_store: BlobStore) {
pub async fn store_assert_is_empty(store: &Store, blob_store: BlobStore, include_directory: bool) {
store_blob_expire_all(store).await;
store_lookup_expire_all(store).await;
store.purge_blobs(blob_store).await.unwrap();
@@ -222,7 +228,7 @@ pub async fn store_assert_is_empty(store: &Store, blob_store: BlobStore) {
for (subspace, with_values) in [
(SUBSPACE_ACL, true),
//(SUBSPACE_DIRECTORY, true),
(SUBSPACE_DIRECTORY, true),
(SUBSPACE_TASK_QUEUE, true),
(SUBSPACE_IN_MEMORY_VALUE, true),
(SUBSPACE_IN_MEMORY_COUNTER, false),
@@ -232,7 +238,7 @@ pub async fn store_assert_is_empty(store: &Store, blob_store: BlobStore) {
(SUBSPACE_QUEUE_EVENT, true),
(SUBSPACE_REPORT_OUT, true),
(SUBSPACE_REPORT_IN, true),
(SUBSPACE_BLOB_RESERVE, true),
(SUBSPACE_BLOB_EXTRA, true),
(SUBSPACE_BLOB_LINK, true),
(SUBSPACE_BLOBS, true),
(SUBSPACE_COUNTER, false),
@@ -242,7 +248,9 @@ pub async fn store_assert_is_empty(store: &Store, blob_store: BlobStore) {
(SUBSPACE_TELEMETRY_METRIC, true),
(SUBSPACE_SEARCH_INDEX, true),
] {
if subspace == SUBSPACE_SEARCH_INDEX && store.is_pg_or_mysql() {
if (subspace == SUBSPACE_SEARCH_INDEX && store.is_pg_or_mysql())
|| (subspace == SUBSPACE_DIRECTORY && !include_directory)
{
continue;
}

78
tests/src/store/import_export.rs
View File

@@ -4,13 +4,16 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::store::TempDir;
use crate::store::{
TempDir,
cleanup::{store_assert_is_empty, store_destroy},
};
use ahash::AHashSet;
use common::{Core, manager::backup::BackupParams};
use common::{Core, DATABASE_SCHEMA_VERSION, manager::backup::BackupParams};
use store::{
rand,
write::{
AnyKey, BatchBuilder, BlobOp, DirectoryClass, InMemoryClass, Operation, QueueClass,
AnyClass, AnyKey, BatchBuilder, BlobLink, BlobOp, DirectoryClass, Operation, QueueClass,
QueueEvent, ValueClass,
},
*,
@@ -29,11 +32,18 @@ pub async fn test(db: Store) {
core.storage.lookup = db.clone().into();
// Make sure the store is empty
db.assert_is_empty(db.clone().into()).await;
store_assert_is_empty(&db, db.clone().into(), true).await;
// Create blobs
println!("Creating blobs...");
let mut batch = BatchBuilder::new();
batch.set(
ValueClass::Any(AnyClass {
subspace: SUBSPACE_PROPERTY,
key: vec![0u8],
}),
DATABASE_SCHEMA_VERSION.serialize(),
);
let mut blob_hashes = Vec::new();
for blob_size in [16, 128, 1024, 2056, 102400] {
let data = random_bytes(blob_size);
@@ -82,13 +92,6 @@ pub async fn test(db: Store) {
batch.set(ValueClass::Property(idx as u8), random_bytes(value_size));
}
for value_size in [1, 4, 7, 8, 9, 16] {
batch.set(
ValueClass::FtsIndex(BitmapHash::new(random_bytes(value_size))),
random_bytes(value_size * 2),
);
}
for grant_account_id in 0u32..10u32 {
if account_id != grant_account_id {
batch.set(
@@ -100,50 +103,17 @@ pub async fn test(db: Store) {
for hash in &blob_hashes {
batch.set(
ValueClass::Blob(BlobOp::Link { hash: hash.clone() }),
ValueClass::Blob(BlobOp::Link {
hash: hash.clone(),
to: BlobLink::Document,
}),
vec![],
);
}
batch.log_item_insert(SyncCollection::from(collection), None);
/*batch.any_op(Operation::ChangeId {
change_id: document_id as u64 + account_id as u64 + collection as u64,
});
batch.any_op(Operation::Log {
set: MaybeDynamicValue::Static(vec![
account_id as u8,
collection,
document_id as u8,
]),
});*/
for field in 0..5 {
batch.any_op(Operation::Bitmap {
class: BitmapClass::Tag {
field,
value: TagValue::Id(rand::random()),
},
set: true,
});
batch.any_op(Operation::Bitmap {
class: BitmapClass::Tag {
field,
value: TagValue::Text(random_bytes(field as usize + 2)),
},
set: true,
});
batch.any_op(Operation::Bitmap {
class: BitmapClass::Text {
field,
token: BitmapHash::new(random_bytes(field as usize + 2)),
},
set: true,
});
batch.any_op(Operation::Index {
field,
key: random_bytes(field as usize + 2),
@@ -172,14 +142,14 @@ pub async fn test(db: Store) {
})),
random_bytes(idx),
);
batch.set(
/*batch.set(
ValueClass::InMemory(InMemoryClass::Key(random_bytes(idx))),
random_bytes(idx),
);
batch.add(
ValueClass::InMemory(InMemoryClass::Counter(random_bytes(idx))),
rand::random(),
);
);*/
batch.set(
ValueClass::Config(random_bytes(idx + 10)),
random_bytes(idx + 10),
@@ -246,8 +216,8 @@ pub async fn test(db: Store) {
// Destroy store
println!("Destroying store...");
db.destroy().await;
db.assert_is_empty(db.clone().into()).await;
store_destroy(&db).await;
store_assert_is_empty(&db, db.clone().into(), true).await;
// Import store
println!("Importing store...");
@@ -259,7 +229,7 @@ pub async fn test(db: Store) {
println!(" GREAT SUCCESS!");
// Destroy store
db.destroy().await;
store_destroy(&db).await;
temp_dir.delete();
}
@@ -286,7 +256,7 @@ impl Snapshot {
(SUBSPACE_DIRECTORY, true),
(SUBSPACE_TASK_QUEUE, true),
(SUBSPACE_INDEXES, false),
(SUBSPACE_BLOB_RESERVE, true),
(SUBSPACE_BLOB_EXTRA, true),
(SUBSPACE_BLOB_LINK, true),
(SUBSPACE_BLOBS, true),
(SUBSPACE_LOGS, true),

8
tests/src/store/lookup.rs
View File

@@ -68,7 +68,7 @@ pub async fn lookup_tests() {
store.purge_in_memory_store().await.unwrap();
if let InMemoryStore::Store(store) = &store {
store_assert_is_empty(store, store.clone().into()).await;
store_assert_is_empty(store, store.clone().into(), false).await;
}
// Test counter
@@ -126,7 +126,7 @@ pub async fn lookup_tests() {
tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
store.purge_in_memory_store().await.unwrap();
if let InMemoryStore::Store(store) = &store {
store_assert_is_empty(store, store.clone().into()).await;
store_assert_is_empty(store, store.clone().into(), false).await;
}
// Test locking
@@ -152,7 +152,7 @@ pub async fn lookup_tests() {
}
store.purge_in_memory_store().await.unwrap();
if let InMemoryStore::Store(store) = &store {
store_assert_is_empty(store, store.clone().into()).await;
store_assert_is_empty(store, store.clone().into(), false).await;
}
// Test prefix delete
@@ -284,7 +284,7 @@ pub async fn lookup_tests() {
);
if let InMemoryStore::Store(store) = &store {
store_assert_is_empty(store, store.clone().into()).await;
store_assert_is_empty(store, store.clone().into(), false).await;
}
}
}

4
tests/src/store/mod.rs
View File

@@ -5,8 +5,8 @@
*/
pub mod blob;
//pub mod import_export;
pub mod cleanup;
pub mod import_export;
pub mod lookup;
pub mod ops;
pub mod query;
@@ -45,7 +45,7 @@ pub async fn store_tests() {
store_destroy(&store).await;
}
//import_export::test(store.clone()).await;
import_export::test(store.clone()).await;
ops::test(store.clone()).await;
if insert {

2
tests/src/store/ops.rs
View File

@@ -473,6 +473,6 @@ pub async fn test(db: Store) {
db.write(batch.build_all()).await.unwrap();
// Make sure everything is deleted
store_assert_is_empty(&db, db.clone().into()).await;
store_assert_is_empty(&db, db.clone().into(), false).await;
}
}