filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Storage layer refactoring: faster id generation, automatic batching and virtual thread ids

Browse Source
This commit is contained in:
mdecimus
2025-04-02 17:37:14 +02:00
parent 76f085ab7c
commit fac2975a5a
152 changed files with 3489 additions and 4039 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
tests/src/store/assign_id.rs
View File

@@ -1,108 +0,0 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use std::collections::HashSet;
use store::{Store, write::BatchBuilder};
pub async fn test(db: Store) {
println!("Running Store ID assignment tests...");
test_0(db).await;
}
async fn test_0(db: Store) {
// Test document id assignment
println!("Creating 1000 documentIds concurrently...");
let mut handles = Vec::new();
let mut assigned_ids = HashSet::new();
// Create 1000 ids concurrently
for _ in 0..1000 {
handles.push({
let db = db.clone();
tokio::spawn(async move {
db.write(
BatchBuilder::new()
.with_account_id(0)
.with_collection(u8::MAX)
.create_document()
.build_batch(),
)
.await
.unwrap()
.last_document_id()
.unwrap()
})
});
}
for handle in handles {
let assigned_id = handle.await.unwrap();
assert!(
assigned_ids.insert(assigned_id),
"already assigned or invalid: {assigned_id}"
);
}
assert_eq!(assigned_ids.len(), 1000);
// Create 1000 ids concurrently
println!("Deleting 1000 documentIds concurrently...");
let mut handles = Vec::new();
for document_id in assigned_ids {
let db = db.clone();
handles.push({
tokio::spawn(async move {
db.write(
BatchBuilder::new()
.with_account_id(0)
.with_collection(u8::MAX)
.delete_document(document_id)
.build_batch(),
)
.await
.unwrap();
})
});
}
for handle in handles {
handle.await.unwrap();
}
// Reuse 1000 ids concurrently
println!("Reusing 1000 freed documentIds concurrently...");
let mut handles = Vec::new();
let mut assigned_ids = HashSet::new();
for _ in 0..1000 {
handles.push({
let db = db.clone();
tokio::spawn(async move {
db.write(
BatchBuilder::new()
.with_account_id(0)
.with_collection(u8::MAX)
.create_document()
.build_batch(),
)
.await
.unwrap()
.last_document_id()
.unwrap()
})
});
}
for handle in handles {
let assigned_id = handle.await.unwrap();
assert!(
assigned_ids.insert(assigned_id),
"freed id already assigned or invalid: {assigned_id}"
);
}
assert_eq!(assigned_ids.len(), 1000);
db.destroy().await;
}

8
tests/src/store/blob.rs
View File

@@ -51,7 +51,7 @@ pub async fn blob_tests() {
},
1024u32.serialize(),
)
.build_batch(),
.build_all(),
)
.await
.unwrap();
@@ -67,7 +67,7 @@ pub async fn blob_tests() {
.write(
BatchBuilder::new()
.set(BlobOp::Commit { hash: hash.clone() }, Vec::new())
.build_batch(),
.build_all(),
)
.await
.unwrap();
@@ -180,7 +180,7 @@ pub async fn blob_tests() {
.update_document(document_id as u32)
.set(blob_op, blob_value)
.set(BlobOp::Commit { hash: hash.clone() }, vec![])
.build_batch(),
.build_all(),
)
.await
.unwrap();
@@ -294,7 +294,7 @@ pub async fn blob_tests() {
.clear(BlobOp::Link {
hash: BlobHash::generate(b"789".as_slice()),
})
.build_batch(),
.build_all(),
)
.await
.unwrap();

44
tests/src/store/import_export.rs
View File

@@ -11,7 +11,7 @@ use store::{
rand,
write::{
AnyKey, BatchBuilder, BitmapClass, BitmapHash, BlobOp, DirectoryClass, InMemoryClass,
MaybeDynamicId, MaybeDynamicValue, Operation, QueueClass, QueueEvent, TagValue, ValueClass,
Operation, QueueClass, QueueEvent, TagValue, ValueClass,
},
*,
};
@@ -44,7 +44,7 @@ pub async fn test(db: Store) {
.unwrap();
batch.set(ValueClass::Blob(BlobOp::Commit { hash }), vec![]);
}
db.write(batch.build()).await.unwrap();
db.write(batch.build_all()).await.unwrap();
// Create account data
println!("Creating account data...");
@@ -57,7 +57,7 @@ pub async fn test(db: Store) {
batch.with_collection(collection);
for document_id in [0, 10, 20, 30, 40] {
batch.create_document_with_id(document_id);
batch.create_document(document_id);
if collection == u8::from(Collection::Mailbox) {
batch
@@ -98,28 +98,30 @@ pub async fn test(db: Store) {
);
}
batch.ops.push(Operation::ChangeId {
batch.log_insert(None);
/*batch.any_op(Operation::ChangeId {
change_id: document_id as u64 + account_id as u64 + collection as u64,
});
batch.ops.push(Operation::Log {
batch.any_op(Operation::Log {
set: MaybeDynamicValue::Static(vec![
account_id as u8,
collection,
document_id as u8,
]),
});
});*/
for field in 0..5 {
batch.ops.push(Operation::Bitmap {
batch.any_op(Operation::Bitmap {
class: BitmapClass::Tag {
field,
value: TagValue::Id(MaybeDynamicId::Static(rand::random())),
value: TagValue::Id(rand::random()),
},
set: true,
});
batch.ops.push(Operation::Bitmap {
batch.any_op(Operation::Bitmap {
class: BitmapClass::Tag {
field,
value: TagValue::Text(random_bytes(field as usize + 2)),
@@ -127,7 +129,7 @@ pub async fn test(db: Store) {
set: true,
});
batch.ops.push(Operation::Bitmap {
batch.any_op(Operation::Bitmap {
class: BitmapClass::Text {
field,
token: BitmapHash::new(random_bytes(field as usize + 2)),
@@ -135,7 +137,7 @@ pub async fn test(db: Store) {
set: true,
});
batch.ops.push(Operation::Index {
batch.any_op(Operation::Index {
field,
key: random_bytes(field as usize + 2),
set: true,
@@ -144,7 +146,7 @@ pub async fn test(db: Store) {
}
}
db.write(batch.build()).await.unwrap();
db.write(batch.build_all()).await.unwrap();
}
// Create queue, config and lookup data
@@ -175,7 +177,7 @@ pub async fn test(db: Store) {
random_bytes(idx + 10),
);
}
db.write(batch.build()).await.unwrap();
db.write(batch.build_all()).await.unwrap();
// Create directory data
println!("Creating directory data...");
@@ -186,7 +188,7 @@ pub async fn test(db: Store) {
for account_id in [1, 2, 3, 4, 5] {
batch
.create_document_with_id(account_id)
.create_document(account_id)
.add(
ValueClass::Directory(DirectoryClass::UsedQuota(account_id)),
rand::random(),
@@ -204,27 +206,25 @@ pub async fn test(db: Store) {
random_bytes(4),
)
.set(
ValueClass::Directory(DirectoryClass::Principal(MaybeDynamicId::Static(
account_id,
))),
ValueClass::Directory(DirectoryClass::Principal(account_id)),
random_bytes(30),
)
.set(
ValueClass::Directory(DirectoryClass::MemberOf {
principal_id: MaybeDynamicId::Static(account_id),
member_of: MaybeDynamicId::Static(rand::random()),
principal_id: account_id,
member_of: rand::random(),
}),
random_bytes(15),
)
.set(
ValueClass::Directory(DirectoryClass::Members {
principal_id: MaybeDynamicId::Static(account_id),
has_member: MaybeDynamicId::Static(rand::random()),
principal_id: account_id,
has_member: rand::random(),
}),
random_bytes(15),
);
}
db.write(batch.build()).await.unwrap();
db.write(batch.build_all()).await.unwrap();
// Obtain store hash
println!("Calculating store hash...");

2
tests/src/store/mod.rs
View File

@@ -4,7 +4,6 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
pub mod assign_id;
pub mod blob;
pub mod import_export;
pub mod lookup;
@@ -97,7 +96,6 @@ pub async fn store_tests() {
}
import_export::test(store.clone()).await;
assign_id::test(store.clone()).await;
ops::test(store.clone()).await;
query::test(store.clone(), FtsStore::Store(store.clone()), insert).await;

106
tests/src/store/ops.rs
View File

@@ -6,10 +6,9 @@
use std::collections::HashSet;
use jmap_proto::types::{collection::Collection, property::Property};
use store::{
BitmapKey, Store, ValueKey,
write::{BatchBuilder, BitmapClass, DirectoryClass, MaybeDynamicId, TagValue, ValueClass},
Store, ValueKey,
write::{BatchBuilder, DirectoryClass, ValueClass},
};
// FDB max value
@@ -33,7 +32,7 @@ pub async fn test(db: Store) {
);
if n % 10000 == 0 {
db.write(batch.build_batch()).await.unwrap();
db.write(batch.build_all()).await.unwrap();
batch = BatchBuilder::new();
batch
.with_account_id(0)
@@ -41,7 +40,7 @@ pub async fn test(db: Store) {
.update_document(0);
}
}
db.write(batch.build_batch()).await.unwrap();
db.write(batch.build_all()).await.unwrap();
println!("Created 900.000 keys...");
// Iterate over all keys
@@ -85,7 +84,7 @@ pub async fn test(db: Store) {
batch.clear(ValueClass::Config(format!("key{n:10}").into_bytes()));
if n % 10000 == 0 {
db.write(batch.build_batch()).await.unwrap();
db.write(batch.build_all()).await.unwrap();
batch = BatchBuilder::new();
batch
.with_account_id(0)
@@ -93,94 +92,9 @@ pub async fn test(db: Store) {
.update_document(0);
}
}
db.write(batch.build_batch()).await.unwrap();
db.write(batch.build_all()).await.unwrap();
}
// Testing ID assignment
println!("Running dynamic ID assignment tests...");
let mut builder = BatchBuilder::new();
builder
.with_account_id(0)
.with_collection(Collection::Thread)
.create_document()
.with_collection(Collection::Email)
.create_document()
.tag(Property::ThreadId, TagValue::Id(MaybeDynamicId::Dynamic(0)))
.set(Property::ThreadId, MaybeDynamicId::Dynamic(0));
let assigned_ids = db.write(builder.build_batch()).await.unwrap();
assert_eq!(assigned_ids.document_ids.len(), 2);
let thread_id = assigned_ids.first_document_id().unwrap();
let email_id = assigned_ids.last_document_id().unwrap();
let email_ids = db
.get_bitmap(BitmapKey {
account_id: 0,
collection: Collection::Email.into(),
class: BitmapClass::DocumentIds,
document_id: 0,
})
.await
.unwrap()
.unwrap();
assert_eq!(email_ids.len(), 1);
assert!(email_ids.contains(email_id));
let thread_ids = db
.get_bitmap(BitmapKey {
account_id: 0,
collection: Collection::Thread.into(),
class: BitmapClass::DocumentIds,
document_id: 0,
})
.await
.unwrap()
.unwrap();
assert_eq!(thread_ids.len(), 1);
assert!(thread_ids.contains(thread_id));
let tagged_ids = db
.get_bitmap(BitmapKey {
account_id: 0,
collection: Collection::Email.into(),
class: BitmapClass::Tag {
field: Property::ThreadId.into(),
value: TagValue::Id(thread_id),
},
document_id: 0,
})
.await
.unwrap()
.unwrap();
assert_eq!(tagged_ids.len(), 1);
assert!(tagged_ids.contains(email_id));
let stored_thread_id = db
.get_value::<u32>(ValueKey {
account_id: 0,
collection: Collection::Email.into(),
document_id: email_id,
class: ValueClass::Property(Property::ThreadId.into()),
})
.await
.unwrap()
.unwrap();
assert_eq!(stored_thread_id, thread_id);
let mut builder = BatchBuilder::new();
builder
.with_account_id(0)
.with_collection(Collection::Thread)
.delete_document(thread_id)
.with_collection(Collection::Email)
.delete_document(email_id)
.untag(
Property::ThreadId,
TagValue::Id(MaybeDynamicId::Static(thread_id)),
)
.clear(Property::ThreadId);
db.write(builder.build_batch()).await.unwrap();
// Increment a counter 1000 times concurrently
let mut handles = Vec::new();
let mut assigned_ids = HashSet::new();
@@ -195,7 +109,7 @@ pub async fn test(db: Store) {
.with_collection(0)
.update_document(0)
.add_and_get(ValueClass::Directory(DirectoryClass::UsedQuota(0)), 1);
db.write(builder.build_batch())
db.write(builder.build_all())
.await
.unwrap()
.last_counter_id()
@@ -255,7 +169,7 @@ pub async fn test(db: Store) {
.set(ValueClass::Property(1), value.as_slice())
.set(ValueClass::Property(0), "check1".as_bytes())
.set(ValueClass::Property(2), "check2".as_bytes())
.build_batch(),
.build_all(),
)
.await
.unwrap();
@@ -282,7 +196,7 @@ pub async fn test(db: Store) {
.with_collection(0)
.update_document(0)
.clear(ValueClass::Property(1))
.build_batch(),
.build_all(),
)
.await
.unwrap();
@@ -328,7 +242,7 @@ pub async fn test(db: Store) {
.clear(ValueClass::Property(0))
.clear(ValueClass::Property(2))
.clear(ValueClass::Directory(DirectoryClass::UsedQuota(0)))
.build_batch(),
.build_all(),
)
.await
.unwrap();

11
tests/src/store/query.rs
View File

@@ -139,7 +139,7 @@ pub async fn test(db: Store, fts_store: FtsStore, do_insert: bool) {
builder
.with_account_id(0)
.with_collection(COLLECTION_ID)
.create_document_with_id(document_id as u32);
.create_document(document_id as u32);
for (pos, field) in record.iter().enumerate() {
let field_id = pos as u8;
match FIELDS_OPTIONS[pos] {
@@ -186,10 +186,7 @@ pub async fn test(db: Store, fts_store: FtsStore, do_insert: bool) {
}
}
documents
.lock()
.unwrap()
.push((builder.build(), fts_builder));
documents.lock().unwrap().push((builder, fts_builder));
});
}
});
@@ -206,11 +203,11 @@ pub async fn test(db: Store, fts_store: FtsStore, do_insert: bool) {
let mut fts_chunk = Vec::new();
print!("Inserting... ",);
for (batch, fts_batch) in batches {
for (mut batch, fts_batch) in batches {
let chunk_instance = Instant::now();
chunk.push({
let db = db.clone();
tokio::spawn(async move { db.write(batch).await })
tokio::spawn(async move { db.write(batch.build_all()).await })
});
fts_chunk.push({
let fts_store = fts_store.clone();