filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

FDB blob storage support

Browse Source
This commit is contained in:
mdecimus
2023-11-28 10:40:30 +01:00
parent 0168c1dca8
commit a1f7ad891a
6 changed files with 214 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
tests/src/store/assign_id.rs
View File

@@ -50,7 +50,7 @@ async fn test_1(db: Store) {
for id in 0..100 {
handles.push({
let db = db.clone();
tokio::spawn(async move { db.assign_document_id(0, 0).await })
tokio::spawn(async move { db.assign_document_id(0, u8::MAX).await })
});
expected_ids.insert(id);
}
@@ -80,9 +80,9 @@ async fn test_2(db: Store) {
// Create document ids and try reassigning
let mut expected_ids = AHashSet::new();
let mut batch = BatchBuilder::new();
batch.with_account_id(0).with_collection(0);
batch.with_account_id(0).with_collection(u8::MAX);
for pos in 0..100 {
let id = db.assign_document_id(0, 0).await.unwrap();
let id = db.assign_document_id(0, u8::MAX).await.unwrap();
if pos % 2 == 0 {
batch.create_document(id);
} else {
@@ -95,14 +95,14 @@ async fn test_2(db: Store) {
tokio::time::sleep(Duration::from_secs(3)).await;
for _ in 0..expected_ids.len() {
let id = db.assign_document_id(0, 0).await.unwrap();
let id = db.assign_document_id(0, u8::MAX).await.unwrap();
assert!(
expected_ids.remove(&id),
"already assigned or invalid: {id}"
);
}
assert_eq!(db.assign_document_id(0, 0).await.unwrap(), 100);
assert_eq!(db.assign_document_id(0, 0).await.unwrap(), 101);
assert_eq!(db.assign_document_id(0, u8::MAX).await.unwrap(), 100);
assert_eq!(db.assign_document_id(0, u8::MAX).await.unwrap(), 101);
db.destroy().await;
}
@@ -111,7 +111,7 @@ async fn test_3(db: Store) {
// Try reassigning deleted ids
let mut expected_ids = AHashSet::new();
let mut batch = BatchBuilder::new();
batch.with_account_id(0).with_collection(0);
batch.with_account_id(0).with_collection(u8::MAX);
for id in 0..100 {
if id % 2 == 0 {
batch.create_document(id);
@@ -121,14 +121,14 @@ async fn test_3(db: Store) {
}
db.write(batch.build()).await.unwrap();
for _ in 0..expected_ids.len() {
let id = db.assign_document_id(0, 0).await.unwrap();
let id = db.assign_document_id(0, u8::MAX).await.unwrap();
assert!(
expected_ids.remove(&id),
"already assigned or invalid: {id}"
);
}
assert_eq!(db.assign_document_id(0, 0).await.unwrap(), 100);
assert_eq!(db.assign_document_id(0, 0).await.unwrap(), 101);
assert_eq!(db.assign_document_id(0, u8::MAX).await.unwrap(), 100);
assert_eq!(db.assign_document_id(0, u8::MAX).await.unwrap(), 101);
db.destroy().await;
}

82
tests/src/store/blob.rs
View File

@@ -52,7 +52,6 @@ path = "{TMP}/db.db?mode=rwc"
#[tokio::test]
pub async fn blob_tests() {
let temp_dir = TempDir::new("blob_tests", true);
let mut blob_store = None;
for (store_id, store_cfg) in [("s3", CONFIG_S3), ("fs", CONFIG_LOCAL)] {
let config =
@@ -65,20 +64,24 @@ pub async fn blob_tests() {
_ => unreachable!(),
};
println!("Testing store {}...", store_id);
println!("Testing blob store {}...", store_id);
test_store(blob_store_.clone()).await;
blob_store = Some(blob_store_);
}
let blob_store = blob_store.unwrap();
// Start SQLite store
// Init store
let store: Store = SqliteStore::open(
//let store: Store = FdbStore::open(
&Config::new(&CONFIG_DB.replace("{TMP}", temp_dir.path.as_path().to_str().unwrap()))
.unwrap(),
)
.await
.unwrap()
.into();
store.destroy().await;
// Test internal blob store
let blob_store: BlobStore = store.clone().into();
test_store(blob_store.clone()).await;
// Blob hash exists
let hash = BlobHash::from(b"abc".as_slice());
@@ -387,17 +390,76 @@ pub async fn blob_tests() {
}
async fn test_store(store: BlobStore) {
// Test small blob
const DATA: &[u8] = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Fusce erat nisl, dignissim a porttitor id, varius nec arcu. Sed mauris.";
let hash = BlobHash::from(DATA);
store.put_blob(b"abc", DATA).await.unwrap();
store.put_blob(hash.as_slice(), DATA).await.unwrap();
assert_eq!(
String::from_utf8(store.get_blob(b"abc", 0..u32::MAX).await.unwrap().unwrap()).unwrap(),
String::from_utf8(
store
.get_blob(hash.as_slice(), 0..u32::MAX)
.await
.unwrap()
.unwrap()
)
.unwrap(),
std::str::from_utf8(DATA).unwrap()
);
assert_eq!(
String::from_utf8(store.get_blob(b"abc", 11..57).await.unwrap().unwrap()).unwrap(),
String::from_utf8(
store
.get_blob(hash.as_slice(), 11..57)
.await
.unwrap()
.unwrap()
)
.unwrap(),
std::str::from_utf8(&DATA[11..57]).unwrap()
);
assert!(store.delete_blob(b"abc").await.unwrap());
assert!(store.get_blob(b"abc", 0..u32::MAX).await.unwrap().is_none());
assert!(store.delete_blob(hash.as_slice()).await.unwrap());
assert!(store
.get_blob(hash.as_slice(), 0..u32::MAX)
.await
.unwrap()
.is_none());
// Test large blob
let mut data = Vec::with_capacity(50 * 1024 * 1024);
while data.len() < 50 * 1024 * 1024 {
data.extend_from_slice(DATA);
let marker = format!(" [{}] ", data.len());
data.extend_from_slice(marker.as_bytes());
}
let hash = BlobHash::from(&data);
store.put_blob(hash.as_slice(), &data).await.unwrap();
assert_eq!(
String::from_utf8(
store
.get_blob(hash.as_slice(), 0..u32::MAX)
.await
.unwrap()
.unwrap()
)
.unwrap(),
std::str::from_utf8(&data).unwrap()
);
assert_eq!(
String::from_utf8(
store
.get_blob(hash.as_slice(), 3000111..4000999)
.await
.unwrap()
.unwrap()
)
.unwrap(),
std::str::from_utf8(&data[3000111..4000999]).unwrap()
);
assert!(store.delete_blob(hash.as_slice()).await.unwrap());
assert!(store
.get_blob(hash.as_slice(), 0..u32::MAX)
.await
.unwrap()
.is_none());
}

5
tests/src/store/mod.rs
View File

@@ -29,7 +29,7 @@ use std::io::Read;
use ::store::Store;
use store::backend::sqlite::SqliteStore;
use store::backend::{foundationdb::FdbStore, sqlite::SqliteStore};
use utils::config::Config;
pub struct TempDir {
@@ -49,7 +49,8 @@ pub async fn store_tests() {
temp_dir.path.display(),
temp_dir.path.display()
);
let db: Store = SqliteStore::open(&Config::new(&config_file).unwrap())
//let db: Store = SqliteStore::open(&Config::new(&config_file).unwrap())
let db: Store = FdbStore::open(&Config::new(&config_file).unwrap())
.await
.unwrap()
.into();