filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

How I stopped worrying and learned to love zero-copy deserialization

Browse Source
This commit is contained in:
mdecimus
2025-02-25 17:38:02 +01:00
parent b7c0f8447b
commit 2826ff1548
157 changed files with 3822 additions and 2774 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
tests/src/jmap/email_changes.rs
View File

@@ -5,12 +5,12 @@
*/
use jmap_proto::{
parser::{json::Parser, JsonObjectParser},
parser::{JsonObjectParser, json::Parser},
types::{collection::Collection, id::Id, state::State},
};
use store::{
ahash::AHashSet,
write::{log::ChangeLogBuilder, BatchBuilder},
write::{BatchBuilder, log::ChangeLogBuilder},
};
use crate::jmap::assert_is_empty;
@@ -160,6 +160,7 @@ pub async fn test(params: &mut JMAPTest) {
.with_account_id(1)
.with_collection(Collection::Email)
.custom(changelog)
.unwrap()
.build_batch(),
)
.await

11
tests/src/jmap/email_query_changes.rs
View File

@@ -13,7 +13,7 @@ use jmap_proto::types::{collection::Collection, id::Id, property::Property, stat
use store::{
ahash::{AHashMap, AHashSet},
write::{log::ChangeLogBuilder, BatchBuilder, MaybeDynamicId, TagValue, F_BITMAP, F_CLEAR},
write::{BatchBuilder, MaybeDynamicId, TagValue, log::ChangeLogBuilder},
};
use crate::jmap::{
@@ -142,18 +142,15 @@ pub async fn test(params: &mut JMAPTest) {
.create_document()
.with_collection(Collection::Email)
.update_document(id.document_id())
.value(Property::ThreadId, id.prefix_id(), F_BITMAP | F_CLEAR)
.untag(Property::ThreadId, id.prefix_id())
.set(Property::ThreadId, MaybeDynamicId::Dynamic(0))
.tag(
Property::ThreadId,
TagValue::Id(MaybeDynamicId::Dynamic(0)),
0,
)
.tag(Property::ThreadId, TagValue::Id(MaybeDynamicId::Dynamic(0)))
.custom(server.begin_changes(1).unwrap().with_log_move(
Collection::Email,
id,
new_id,
))
.unwrap()
.build_batch(),
)
.await

146
tests/src/store/blob.rs
View File

@@ -6,12 +6,12 @@
use ahash::AHashMap;
use store::{
write::{blob::BlobQuota, now, BatchBuilder, BlobOp},
BlobClass, BlobStore, Serialize, Stores,
BlobClass, BlobStore, SerializeInfallible, Stores,
write::{BatchBuilder, BlobOp, blob::BlobQuota, now},
};
use utils::{config::Config, BlobHash};
use utils::{BlobHash, config::Config};
use crate::store::{TempDir, CONFIG};
use crate::store::{CONFIG, TempDir};
#[tokio::test]
pub async fn blob_tests() {
@@ -74,35 +74,41 @@ pub async fn blob_tests() {
// Blob hash should now exist
assert!(store.blob_exists(&hash).await.unwrap());
assert!(blob_store
.get_blob(hash.as_ref(), 0..usize::MAX)
.await
.unwrap()
.is_some());
assert!(
blob_store
.get_blob(hash.as_ref(), 0..usize::MAX)
.await
.unwrap()
.is_some()
);
// AccountId 0 should be able to read blob
assert!(store
.blob_has_access(
&hash,
BlobClass::Reserved {
account_id: 0,
expires: until
}
)
.await
.unwrap());
assert!(
store
.blob_has_access(
&hash,
BlobClass::Reserved {
account_id: 0,
expires: until
}
)
.await
.unwrap()
);
// AccountId 1 should not be able to read blob
assert!(!store
.blob_has_access(
&hash,
BlobClass::Reserved {
account_id: 1,
expires: until
}
)
.await
.unwrap());
assert!(
!store
.blob_has_access(
&hash,
BlobClass::Reserved {
account_id: 1,
expires: until
}
)
.await
.unwrap()
);
// Blob already expired, quota should be 0
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
@@ -118,23 +124,27 @@ pub async fn blob_tests() {
assert!(!store.blob_exists(&hash).await.unwrap());
// AccountId 0 should not be able to read blob
assert!(!store
.blob_has_access(
&hash,
BlobClass::Reserved {
account_id: 0,
expires: until
}
)
.await
.unwrap());
assert!(
!store
.blob_has_access(
&hash,
BlobClass::Reserved {
account_id: 0,
expires: until
}
)
.await
.unwrap()
);
// Blob should no longer be in store
assert!(blob_store
.get_blob(hash.as_ref(), 0..usize::MAX)
.await
.unwrap()
.is_none());
assert!(
blob_store
.get_blob(hash.as_ref(), 0..usize::MAX)
.await
.unwrap()
.is_none()
);
// Upload one linked blob to accountId 1, two linked blobs to accountId 0, and three unlinked (reserved) blobs to accountId 2
let expiry_times = AHashMap::from_iter([
@@ -260,17 +270,19 @@ pub async fn blob_tests() {
}
// AccountId 0 should not have access to accountId 1's blobs
assert!(!store
.blob_has_access(
BlobHash::from(b"123".as_slice()),
BlobClass::Linked {
account_id: 0,
collection: 0,
document_id: 0,
}
)
.await
.unwrap());
assert!(
!store
.blob_has_access(
BlobHash::from(b"123".as_slice()),
BlobClass::Linked {
account_id: 0,
collection: 0,
document_id: 0,
}
)
.await
.unwrap()
);
// Unlink blob
store
@@ -432,11 +444,13 @@ async fn test_store(store: BlobStore) {
std::str::from_utf8(&DATA[11..57]).unwrap()
);
assert!(store.delete_blob(hash.as_slice()).await.unwrap());
assert!(store
.get_blob(hash.as_slice(), 0..usize::MAX)
.await
.unwrap()
.is_none());
assert!(
store
.get_blob(hash.as_slice(), 0..usize::MAX)
.await
.unwrap()
.is_none()
);
// Test large blob
let mut data = Vec::with_capacity(50 * 1024 * 1024);
@@ -471,9 +485,11 @@ async fn test_store(store: BlobStore) {
std::str::from_utf8(&data[3000111..4000999]).unwrap()
);
assert!(store.delete_blob(hash.as_slice()).await.unwrap());
assert!(store
.get_blob(hash.as_slice(), 0..usize::MAX)
.await
.unwrap()
.is_none());
assert!(
store
.get_blob(hash.as_slice(), 0..usize::MAX)
.await
.unwrap()
.is_none()
);
}

13
tests/src/store/ops.rs
View File

@@ -8,10 +8,8 @@ use std::collections::HashSet;
use jmap_proto::types::{collection::Collection, property::Property};
use store::{
write::{
BatchBuilder, BitmapClass, DirectoryClass, MaybeDynamicId, TagValue, ValueClass, F_CLEAR,
},
BitmapKey, Store, ValueKey,
write::{BatchBuilder, BitmapClass, DirectoryClass, MaybeDynamicId, TagValue, ValueClass},
};
// FDB max value
@@ -107,11 +105,7 @@ pub async fn test(db: Store) {
.create_document()
.with_collection(Collection::Email)
.create_document()
.tag(
Property::ThreadId,
TagValue::Id(MaybeDynamicId::Dynamic(0)),
0,
)
.tag(Property::ThreadId, TagValue::Id(MaybeDynamicId::Dynamic(0)))
.set(Property::ThreadId, MaybeDynamicId::Dynamic(0));
let assigned_ids = db.write(builder.build_batch()).await.unwrap();
@@ -180,10 +174,9 @@ pub async fn test(db: Store) {
.delete_document(thread_id)
.with_collection(Collection::Email)
.delete_document(email_id)
.tag(
.untag(
Property::ThreadId,
TagValue::Id(MaybeDynamicId::Static(thread_id)),
F_CLEAR,
)
.clear(Property::ThreadId);
db.write(builder.build_batch()).await.unwrap();

80
tests/src/store/query.rs
View File

@@ -14,17 +14,17 @@ use std::{
use jmap_proto::types::keyword::Keyword;
use nlp::language::Language;
use store::{
FtsStore, SerializeInfallible,
ahash::AHashMap,
fts::{index::FtsDocument, Field, FtsFilter},
fts::{Field, FtsFilter, index::FtsDocument},
query::sort::Pagination,
write::ValueClass,
FtsStore,
};
use store::{
query::{Comparator, Filter},
write::{BatchBuilder, F_BITMAP, F_INDEX, F_VALUE},
Store, ValueKey,
query::{Comparator, Filter},
write::BatchBuilder,
};
use crate::store::deflate_test_resource;
@@ -145,10 +145,9 @@ pub async fn test(db: Store, fts_store: FtsStore, do_insert: bool) {
match FIELDS_OPTIONS[pos] {
FieldType::Text => {
if !field.is_empty() {
builder.value(
field_id,
field.to_lowercase(),
F_VALUE | F_BITMAP,
builder.tag(field_id, field.to_lowercase()).set(
ValueClass::Property(field_id),
field.to_lowercase().into_bytes(),
);
}
}
@@ -160,24 +159,25 @@ pub async fn test(db: Store, fts_store: FtsStore, do_insert: bool) {
Language::English,
);
if field_id == 7 {
builder.value(field_id, field.to_lowercase(), F_INDEX);
builder.index(field_id, field.to_lowercase());
}
}
}
FieldType::Integer => {
builder.value(
field_id,
field.parse::<u32>().unwrap_or(0),
F_VALUE | F_INDEX,
);
let field = field.parse::<u32>().unwrap_or(0);
builder
.index(field_id, field.serialize())
.set(ValueClass::Property(field_id), field.serialize());
}
FieldType::Keyword => {
if !field.is_empty() {
builder.value(
field_id,
Keyword::Other(field.to_lowercase()),
F_VALUE | F_INDEX | F_BITMAP,
);
builder
.set(
ValueClass::Property(field_id),
field.to_lowercase().into_bytes(),
)
.tag(field_id, Keyword::Other(field.to_lowercase()))
.index(field_id, field.to_lowercase());
}
}
}
@@ -270,7 +270,7 @@ pub async fn test_filter(db: Store, fts: FtsStore) {
.await
.unwrap(),
),
Filter::eq(fields_u8["year"], 1979u32),
Filter::eq(fields_u8["year"], 1979u32.serialize()),
],
vec!["p11293"],
),
@@ -288,9 +288,9 @@ pub async fn test_filter(db: Store, fts: FtsStore) {
.await
.unwrap(),
),
Filter::gt(fields_u8["year"], 2000u32),
Filter::lt(fields_u8["width"], 180u32),
Filter::gt(fields_u8["width"], 0u32),
Filter::gt(fields_u8["year"], 2000u32.serialize()),
Filter::lt(fields_u8["width"], 180u32.serialize()),
Filter::gt(fields_u8["width"], 0u32.serialize()),
],
vec!["p79426", "p79427", "p79428", "p79429", "p79430"],
),
@@ -332,8 +332,8 @@ pub async fn test_filter(db: Store, fts: FtsStore) {
Keyword::Other("artist".to_string()),
),
Filter::Or,
Filter::eq(fields_u8["year"], 1969u32),
Filter::eq(fields_u8["year"], 1971u32),
Filter::eq(fields_u8["year"], 1969u32.serialize()),
Filter::eq(fields_u8["year"], 1971u32.serialize()),
Filter::End,
],
vec!["p01764", "t05843"],
@@ -356,12 +356,12 @@ pub async fn test_filter(db: Store, fts: FtsStore) {
),
Filter::Or,
Filter::And,
Filter::ge(fields_u8["year"], 1900u32),
Filter::lt(fields_u8["year"], 1910u32),
Filter::ge(fields_u8["year"], 1900u32.serialize()),
Filter::lt(fields_u8["year"], 1910u32.serialize()),
Filter::End,
Filter::And,
Filter::ge(fields_u8["year"], 2000u32),
Filter::lt(fields_u8["year"], 2010u32),
Filter::ge(fields_u8["year"], 2000u32.serialize()),
Filter::lt(fields_u8["year"], 2010u32.serialize()),
Filter::End,
Filter::End,
],
@@ -391,14 +391,14 @@ pub async fn test_filter(db: Store, fts: FtsStore) {
Filter::End,
Filter::Not,
Filter::Or,
Filter::gt(fields_u8["year"], 1980u32),
Filter::gt(fields_u8["year"], 1980u32.serialize()),
Filter::And,
Filter::gt(fields_u8["width"], 500u32),
Filter::gt(fields_u8["height"], 500u32),
Filter::gt(fields_u8["width"], 500u32.serialize()),
Filter::gt(fields_u8["height"], 500u32.serialize()),
Filter::End,
Filter::End,
Filter::End,
Filter::eq(fields_u8["acquisitionYear"], 2008u32),
Filter::eq(fields_u8["acquisitionYear"], 2008u32.serialize()),
Filter::End,
],
vec!["ar00039", "t12600"],
@@ -425,8 +425,8 @@ pub async fn test_filter(db: Store, fts: FtsStore) {
.await
.unwrap(),
),
Filter::gt(fields_u8["year"], 1900u32),
Filter::gt(fields_u8["acquisitionYear"], 2000u32),
Filter::gt(fields_u8["year"], 1900u32.serialize()),
Filter::gt(fields_u8["acquisitionYear"], 2000u32.serialize()),
],
vec![
"p80042", "p80043", "p80044", "p80045", "p80203", "t11937", "t12172",
@@ -473,9 +473,9 @@ pub async fn test_sort(db: Store) {
let tests = [
(
vec![
Filter::gt(fields["year"], 0u32),
Filter::gt(fields["acquisitionYear"], 0u32),
Filter::gt(fields["width"], 0u32),
Filter::gt(fields["year"], 0u32.serialize()),
Filter::gt(fields["acquisitionYear"], 0u32.serialize()),
Filter::gt(fields["width"], 0u32.serialize()),
],
vec![
Comparator::descending(fields["year"]),
@@ -495,8 +495,8 @@ pub async fn test_sort(db: Store) {
),
(
vec![
Filter::gt(fields["width"], 0u32),
Filter::gt(fields["height"], 0u32),
Filter::gt(fields["width"], 0u32.serialize()),
Filter::gt(fields["height"], 0u32.serialize()),
],
vec![
Comparator::descending(fields["width"]),