filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

FDB chunked values support

Browse Source
This commit is contained in:
mdecimus
2023-12-08 19:49:44 +01:00
parent db564ae0db
commit 166020b45b
12 changed files with 541 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/Cargo.toml
View File

@@ -6,7 +6,7 @@ resolver = "2"
[features]
#default = ["sqlite", "foundationdb", "postgres", "mysql", "rocks", "elastic", "s3"]
default = ["sqlite", "postgres", "mysql"]
default = ["sqlite", "postgres", "mysql", "foundationdb"]
sqlite = ["store/sqlite"]
foundationdb = ["store/foundation"]
postgres = ["store/postgres"]

39
tests/src/imap/mod.rs
View File

@@ -172,9 +172,9 @@ private-key = "file://{PK}"
directory = "auth"
[jmap.store]
data = "sqlite"
fts = "sqlite"
blob = "sqlite"
data = "{STORE}"
fts = "{STORE}"
blob = "{STORE}"
[jmap.protocol]
set.max-objects = 100000
@@ -262,11 +262,13 @@ pub struct IMAPTest {
shutdown_tx: watch::Sender<bool>,
}
async fn init_imap_tests(delete_if_exists: bool) -> IMAPTest {
async fn init_imap_tests(store_id: &str, delete_if_exists: bool) -> IMAPTest {
// Load and parse config
let temp_dir = TempDir::new("imap_tests", delete_if_exists);
let config = utils::config::Config::new(
&add_test_certs(SERVER).replace("{TMP}", &temp_dir.path.display().to_string()),
&add_test_certs(SERVER)
.replace("{STORE}", store_id)
.replace("{TMP}", &temp_dir.path.display().to_string()),
)
.unwrap();
let servers = config.parse_servers().unwrap();
@@ -347,16 +349,29 @@ async fn init_imap_tests(delete_if_exists: bool) -> IMAPTest {
#[tokio::test]
pub async fn imap_tests() {
/*tracing::subscriber::set_global_default(
tracing_subscriber::FmtSubscriber::builder()
.with_max_level(tracing::Level::DEBUG)
.finish(),
)
.unwrap();*/
if let Ok(level) = std::env::var("LOG") {
tracing::subscriber::set_global_default(
tracing_subscriber::FmtSubscriber::builder()
.with_env_filter(
tracing_subscriber::EnvFilter::builder()
.parse(
format!("smtp={level},imap={level},jmap={level},store={level},utils={level},directory={level}"),
)
.unwrap(),
)
.finish(),
)
.unwrap();
}
// Prepare settings
let delete = true;
let handle = init_imap_tests(delete).await;
let handle = init_imap_tests(
&std::env::var("STORE")
.expect("Missing store type. Try running `STORE=<store_type> cargo test`"),
delete,
)
.await;
// Connect to IMAP server
let mut imap_check = ImapConnection::connect(b"_y ").await;

48
tests/src/jmap/mod.rs
View File

@@ -173,9 +173,9 @@ private-key = "file://{PK}"
directory = "auth"
[jmap.store]
data = "sqlite"
fts = "sqlite"
blob = "sqlite"
data = "{STORE}"
fts = "{STORE}"
blob = "{STORE}"
[jmap.protocol]
set.max-objects = 100000
@@ -258,22 +258,28 @@ refresh-token-renew = "2s"
#[tokio::test]
pub async fn jmap_tests() {
/*let level = "warn";
tracing::subscriber::set_global_default(
tracing_subscriber::FmtSubscriber::builder()
.with_env_filter(
tracing_subscriber::EnvFilter::builder()
.parse(
format!("smtp={level},imap={level},jmap={level},store={level},utils={level},directory={level}"),
)
.unwrap(),
)
.finish(),
)
.unwrap();*/
if let Ok(level) = std::env::var("LOG") {
tracing::subscriber::set_global_default(
tracing_subscriber::FmtSubscriber::builder()
.with_env_filter(
tracing_subscriber::EnvFilter::builder()
.parse(
format!("smtp={level},imap={level},jmap={level},store={level},utils={level},directory={level}"),
)
.unwrap(),
)
.finish(),
)
.unwrap();
}
let delete = true;
let mut params = init_jmap_tests(delete).await;
let mut params = init_jmap_tests(
&std::env::var("STORE")
.expect("Missing store type. Try running `STORE=<store_type> cargo test`"),
delete,
)
.await;
email_query::test(&mut params, delete).await;
email_get::test(&mut params).await;
email_set::test(&mut params).await;
@@ -314,7 +320,7 @@ pub async fn jmap_stress_tests() {
)
.unwrap();
let params = init_jmap_tests(true).await;
let params = init_jmap_tests("foundationdb", true).await;
stress_test::test(params.server.clone(), params.client).await;
params.temp_dir.delete();
}
@@ -355,11 +361,13 @@ pub async fn assert_is_empty(server: Arc<JMAP>) {
.await;
}
async fn init_jmap_tests(delete_if_exists: bool) -> JMAPTest {
async fn init_jmap_tests(store_id: &str, delete_if_exists: bool) -> JMAPTest {
// Load and parse config
let temp_dir = TempDir::new("jmap_tests", delete_if_exists);
let config = utils::config::Config::new(
&add_test_certs(SERVER).replace("{TMP}", &temp_dir.path.display().to_string()),
&add_test_certs(SERVER)
.replace("{STORE}", store_id)
.replace("{TMP}", &temp_dir.path.display().to_string()),
)
.unwrap();
let servers = config.parse_servers().unwrap();

2
tests/src/store/mod.rs
View File

@@ -23,6 +23,7 @@
pub mod assign_id;
pub mod blob;
pub mod ops;
pub mod query;
use std::io::Read;
@@ -87,6 +88,7 @@ pub async fn store_tests() {
if insert {
store.destroy().await;
}
ops::test(store.clone()).await;
query::test(store.clone(), FtsStore::Store(store.clone()), insert).await;
assign_id::test(store).await;
}

145
tests/src/store/ops.rs Normal file
View File

@@ -0,0 +1,145 @@
/*
* Copyright (c) 2023, Stalwart Labs Ltd.
*
* This file is part of the Stalwart Mail Server.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of
* the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
* in the LICENSE file at the top-level directory of this distribution.
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* You can be released from the requirements of the AGPLv3 license by
* purchasing a commercial license. Please contact licensing@stalw.art
* for more details.
*/
use store::{
write::{BatchBuilder, ValueClass},
Store, ValueKey,
};
// FDB max value
const MAX_VALUE_SIZE: usize = 100000;
pub async fn test(db: Store) {
for (test_num, value) in [
vec![b'A'; 0],
vec![b'A'; 1],
vec![b'A'; 100],
vec![b'A'; MAX_VALUE_SIZE],
vec![b'B'; MAX_VALUE_SIZE + 1],
vec![b'C'; MAX_VALUE_SIZE]
.into_iter()
.chain(vec![b'D'; MAX_VALUE_SIZE])
.chain(vec![b'E'; MAX_VALUE_SIZE])
.collect::<Vec<_>>(),
vec![b'F'; MAX_VALUE_SIZE]
.into_iter()
.chain(vec![b'G'; MAX_VALUE_SIZE])
.chain(vec![b'H'; MAX_VALUE_SIZE + 1])
.collect::<Vec<_>>(),
]
.into_iter()
.enumerate()
{
// Write value
let test_len = value.len();
db.write(
BatchBuilder::new()
.with_account_id(0)
.with_collection(0)
.with_account_id(0)
.update_document(0)
.set(ValueClass::Property(1), value.as_slice())
.set(ValueClass::Property(0), "check1")
.set(ValueClass::Property(2), "check2")
.build_batch(),
)
.await
.unwrap();
// Fetch value
assert_eq!(
String::from_utf8(value).unwrap(),
db.get_value::<String>(ValueKey {
account_id: 0,
collection: 0,
document_id: 0,
class: ValueClass::Property(1),
})
.await
.unwrap()
.unwrap_or_else(|| panic!("no value for test {test_num} with value length {test_len}")),
"failed for test {test_num} with value length {test_len}"
);
// Delete value
db.write(
BatchBuilder::new()
.with_account_id(0)
.with_collection(0)
.with_account_id(0)
.update_document(0)
.clear(ValueClass::Property(1))
.build_batch(),
)
.await
.unwrap();
// Make sure value is deleted
assert_eq!(
None,
db.get_value::<String>(ValueKey {
account_id: 0,
collection: 0,
document_id: 0,
class: ValueClass::Property(1),
})
.await
.unwrap()
);
// Make sure other values are still there
for (class, value) in [
(ValueClass::Property(0), "check1"),
(ValueClass::Property(2), "check2"),
] {
assert_eq!(
Some(value.to_string()),
db.get_value::<String>(ValueKey {
account_id: 0,
collection: 0,
document_id: 0,
class,
})
.await
.unwrap()
);
}
// Delete everything
db.write(
BatchBuilder::new()
.with_account_id(0)
.with_collection(0)
.with_account_id(0)
.update_document(0)
.clear(ValueClass::Property(0))
.clear(ValueClass::Property(2))
.build_batch(),
)
.await
.unwrap();
// Make sure everything is deleted
db.assert_is_empty(db.clone().into()).await;
}
}