filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Database schema optimization - part 9

Browse Source
This commit is contained in:
mdecimus
2025-11-09 19:28:19 +01:00
parent d9e6927606
commit 836bc5b7fd
27 changed files with 973 additions and 600 deletions
Download Patch File Download Diff File Expand all files Collapse all files

165
tests/src/store/mod.rs
View File

@@ -19,63 +19,11 @@ pub struct TempDir {
pub path: std::path::PathBuf,
}
const CONFIG: &str = r#"
[store."s3"]
type = "s3"
access-key = "minioadmin"
secret-key = "minioadmin"
region = "eu-central-1"
endpoint = "http://localhost:9000"
bucket = "tmp"
[store."fs"]
type = "fs"
path = "{TMP}"
[store."rocksdb"]
type = "rocksdb"
path = "{TMP}/rocksdb"
[store."foundationdb"]
type = "foundationdb"
[store."sqlite"]
type = "sqlite"
path = "{TMP}/sqlite.db"
[store."postgresql"]
type = "postgresql"
host = "localhost"
port = 5432
database = "stalwart"
user = "postgres"
password = "mysecretpassword"
[store."mysql"]
type = "mysql"
host = "localhost"
port = 3307
database = "stalwart"
user = "root"
password = "password"
[store."redis"]
type = "redis"
urls = "redis://127.0.0.1"
redis-type = "single"
[storage]
lookup = "mysql"
data = "postgresql"
blob = "sqlite"
"#;
#[tokio::test(flavor = "multi_thread")]
pub async fn store_tests() {
let insert = true;
let temp_dir = TempDir::new("store_tests", insert);
let mut config = Config::new(CONFIG.replace("{TMP}", &temp_dir.path.to_string_lossy()))
let mut config = Config::new(build_store_config(&temp_dir.path.to_string_lossy()))
.unwrap()
.assert_no_errors();
let stores = Stores::parse_all(&mut config, false).await;
@@ -95,7 +43,38 @@ pub async fn store_tests() {
//import_export::test(store.clone()).await;
ops::test(store.clone()).await;
query::test(SearchStore::Store(store.clone()), insert).await;
if insert {
temp_dir.delete();
}
}
#[tokio::test(flavor = "multi_thread")]
pub async fn search_tests() {
let insert = true;
let temp_dir = TempDir::new("search_store_tests", insert);
let mut config = Config::new(build_store_config(&temp_dir.path.to_string_lossy()))
.unwrap()
.assert_no_errors();
let stores = Stores::parse_all(&mut config, false).await;
let store_id = std::env::var("SEARCH_STORE")
.expect("Missing store type. Try running `SEARCH_STORE=<store_type> cargo test`");
let store = stores
.search_stores
.get(&store_id)
.expect("Store not found")
.clone();
println!("Testing store {}...", store_id);
if insert {
match &store {
SearchStore::Store(store) => store.destroy().await,
SearchStore::ElasticSearch(_) => (),
}
}
query::test(store, insert).await;
if insert {
temp_dir.delete();
@@ -130,3 +109,81 @@ impl TempDir {
std::fs::remove_dir_all(&self.path).unwrap();
}
}
pub fn build_store_config(temp_dir: &str) -> String {
let store = std::env::var("STORE")
.expect("Missing store type. Try running `STORE=<store_type> cargo test`");
let fts_store = std::env::var("SEARCH_STORE").unwrap_or_else(|_| store.clone());
let blob_store = std::env::var("BLOB_STORE").unwrap_or_else(|_| store.clone());
let lookup_store = std::env::var("LOOKUP_STORE").unwrap_or_else(|_| store.clone());
CONFIG
.replace("{STORE}", &store)
.replace("{SEARCH_STORE}", &fts_store)
.replace("{BLOB_STORE}", &blob_store)
.replace("{LOOKUP_STORE}", &lookup_store)
.replace("{TMP}", temp_dir)
}
const CONFIG: &str = r#"
[store."sqlite"]
type = "sqlite"
path = "{TMP}/sqlite.db"
[store."rocksdb"]
type = "rocksdb"
path = "{TMP}/rocks.db"
[store."foundationdb"]
type = "foundationdb"
[store."postgresql"]
type = "postgresql"
host = "localhost"
port = 5432
database = "stalwart"
user = "postgres"
password = "mysecretpassword"
[store."mysql"]
type = "mysql"
host = "localhost"
port = 3307
database = "stalwart"
user = "root"
password = "password"
[store."elastic"]
type = "elasticsearch"
url = "https://localhost:9200"
tls.allow-invalid-certs = true
[store."elastic".auth]
username = "elastic"
secret = "changeme"
[store."s3"]
type = "s3"
access-key = "minioadmin"
secret-key = "minioadmin"
region = "eu-central-1"
endpoint = "http://localhost:9000"
bucket = "tmp"
[store."fs"]
type = "fs"
path = "{TMP}"
[store."redis"]
type = "redis"
urls = "redis://127.0.0.1"
redis-type = "single"
[storage]
data = "{STORE}"
fts = "{SEARCH_STORE}"
blob = "{BLOB_STORE}"
lookup = "{LOOKUP_STORE}"
directory = "{STORE}"
"#;

167
tests/src/store/ops.rs
View File

@@ -19,39 +19,50 @@ use types::collection::{Collection, SyncCollection};
// FDB max value
const MAX_VALUE_SIZE: usize = 100000;
fn value_gen(chunks: impl IntoIterator<Item = (u8, usize)>) -> Vec<u8> {
let mut value = Vec::new();
for (byte, size) in chunks {
value.extend(std::iter::repeat_n(byte, size));
}
value
}
pub async fn test(db: Store) {
#[cfg(feature = "foundationdb")]
if matches!(db, Store::FoundationDb(_)) && std::env::var("SLOW_FDB_TRX").is_ok() {
if matches!(db, Store::FoundationDb(_)) {
use types::collection::Collection;
println!("Running slow FoundationDB transaction tests...");
// Create 900000 keys
println!("Running FoundationDB chunked iterator test...");
let kvs = [
("a", value_gen([(b'a', 1)])),
("b", value_gen([(b'b', MAX_VALUE_SIZE), (b'0', 1)])),
(
"c",
value_gen([
(b'c', MAX_VALUE_SIZE),
(b'1', MAX_VALUE_SIZE),
(b'2', MAX_VALUE_SIZE),
]),
),
(
"d",
value_gen([(b'd', MAX_VALUE_SIZE), (b'3', MAX_VALUE_SIZE)]),
),
("e", value_gen([(b'e', 1)])),
];
let mut batch = BatchBuilder::new();
batch
.with_account_id(0)
.with_collection(Collection::Email)
.with_document(0);
for n in 0..900000 {
batch.set(
ValueClass::Config(format!("key{n:10}").into_bytes()),
format!("value{n:10}").into_bytes(),
);
if n % 10000 == 0 {
db.write(batch.build_all()).await.unwrap();
batch = BatchBuilder::new();
batch
.with_account_id(0)
.with_collection(Collection::Email)
.with_document(0);
}
for (key, value) in &kvs {
batch.set(ValueClass::Config(key.as_bytes().to_vec()), value.clone());
}
db.write(batch.build_all()).await.unwrap();
println!("Created 900.000 keys...");
// Iterate over all keys
let mut n = 0;
let mut results = Vec::new();
db.iterate(
store::IterateParams::new(
ValueKey {
@@ -68,38 +79,110 @@ pub async fn test(db: Store) {
},
),
|key, value| {
assert_eq!(std::str::from_utf8(key).unwrap(), format!("key{n:10}"));
assert_eq!(std::str::from_utf8(value).unwrap(), format!("value{n:10}"));
n += 1;
if n % 10000 == 0 {
println!("Iterated over {n} keys");
std::thread::sleep(std::time::Duration::from_millis(1000));
}
results.push((String::from_utf8(key.to_vec()).unwrap(), value.to_vec()));
Ok(true)
},
)
.await
.unwrap();
// Delete 100 keys
let mut batch = BatchBuilder::new();
batch
.with_account_id(0)
.with_collection(Collection::Email)
.with_document(0);
for n in 0..900000 {
batch.clear(ValueClass::Config(format!("key{n:10}").into_bytes()));
assert_eq!(results.len(), kvs.len());
if n % 10000 == 0 {
db.write(batch.build_all()).await.unwrap();
batch = BatchBuilder::new();
batch
.with_account_id(0)
.with_collection(Collection::Email)
.with_document(0);
db.delete_range(
ValueKey {
account_id: 0,
collection: 0,
document_id: 0,
class: ValueClass::Config(b"".to_vec()),
},
ValueKey {
account_id: 0,
collection: 0,
document_id: 0,
class: ValueClass::Config(b"\xFF".to_vec()),
},
)
.await
.unwrap();
if std::env::var("SLOW_FDB_TRX").is_ok() {
println!("Running FoundationDB slow transaction tests...");
// Create 900000 keys
let mut batch = BatchBuilder::new();
batch
.with_account_id(0)
.with_collection(Collection::Email)
.with_document(0);
for n in 0..900000 {
batch.set(
ValueClass::Config(format!("key{n:10}").into_bytes()),
format!("value{n:10}").into_bytes(),
);
if n % 10000 == 0 {
db.write(batch.build_all()).await.unwrap();
batch = BatchBuilder::new();
batch
.with_account_id(0)
.with_collection(Collection::Email)
.with_document(0);
}
}
db.write(batch.build_all()).await.unwrap();
println!("Created 900.000 keys...");
// Iterate over all keys
let mut n = 0;
db.iterate(
store::IterateParams::new(
ValueKey {
account_id: 0,
collection: 0,
document_id: 0,
class: ValueClass::Config(b"".to_vec()),
},
ValueKey {
account_id: 0,
collection: 0,
document_id: 0,
class: ValueClass::Config(b"\xFF".to_vec()),
},
),
|key, value| {
assert_eq!(std::str::from_utf8(key).unwrap(), format!("key{n:10}"));
assert_eq!(std::str::from_utf8(value).unwrap(), format!("value{n:10}"));
n += 1;
if n % 10000 == 0 {
println!("Iterated over {n} keys");
std::thread::sleep(std::time::Duration::from_millis(1000));
}
Ok(true)
},
)
.await
.unwrap();
// Delete 100 keys
let mut batch = BatchBuilder::new();
batch
.with_account_id(0)
.with_collection(Collection::Email)
.with_document(0);
for n in 0..900000 {
batch.clear(ValueClass::Config(format!("key{n:10}").into_bytes()));
if n % 10000 == 0 {
db.write(batch.build_all()).await.unwrap();
batch = BatchBuilder::new();
batch
.with_account_id(0)
.with_collection(Collection::Email)
.with_document(0);
}
}
db.write(batch.build_all()).await.unwrap();
}
db.write(batch.build_all()).await.unwrap();
}
// Merge values 1000 times concurrently

378
tests/src/store/query.rs
View File

@@ -5,25 +5,23 @@
*/
use crate::store::deflate_test_resource;
use ahash::AHashSet;
use nlp::language::Language;
use std::{
fmt::Display,
io::Write,
sync::{Arc, Mutex},
time::Instant,
};
use store::{
SearchStore, SerializeInfallible,
SearchStore, Store,
ahash::AHashMap,
roaring::RoaringBitmap,
search::{
EmailSearchField, IndexDocument, SearchComparator, SearchField, SearchFilter,
SearchOperator, SearchQuery, SearchValue,
SearchOperator, SearchQuery, SearchValue, TracingSearchField,
},
write::{Operation, SearchIndex, ValueClass},
write::SearchIndex,
};
use store::{Store, ValueKey, write::BatchBuilder};
use types::collection::Collection;
use utils::map::vec_map::VecMap;
pub const FIELDS: [&str; 20] = [
@@ -85,6 +83,26 @@ const FIELD_MAPPINGS: [EmailSearchField; 20] = [
EmailSearchField::HasAttachment, // "url",
];
const ALL_IDS: &[&str] = &[
"p11293", "p79426", "p79427", "p79428", "p79429", "p79430", "d05503", "d00399", "d05352",
"p01764", "t05843", "n02478", "n02479", "n03568", "n03658", "n04327", "n04328", "n04721",
"n04739", "n05095", "n05096", "n05145", "n05157", "n05158", "n05159", "n05298", "n05303",
"n06070", "t01181", "t03571", "t05805", "t05806", "t12147", "t12154", "t12155", "ar00039",
"t12600", "p80203", "t13209", "t13560", "t13561", "t13655", "t13811", "p13352", "p13351",
"p13350", "p13349", "p13348", "p13347", "p13346", "p13345", "p13344", "p13342", "p13341",
"p13340", "p13339", "p13338", "p13337", "p13336", "p13335", "p13334", "p13333", "p13332",
"p13331", "p13330", "p13329", "p13328", "p13327", "p13326", "p13325", "p13324", "p13323",
"t13786", "p13322", "p13321", "p13320", "p13319", "p13318", "p13317", "p13316", "p13315",
"p13314", "t13588", "t13587", "t13586", "t13585", "t13584", "t13540", "t13444", "ar01154",
"ar01153", "t03681", "t12601", "ar00166", "t12625", "t12915", "p04182", "t06483", "ar00703",
"t07671", "ar00021", "t05557", "t07918", "p06298", "p05465", "p06640", "t12855", "t01355",
"t12800", "t12557", "t02078", "ar00052", "ar00627", "t00352", "t07275", "t12318", "t04931",
"t13683", "t13686", "t13687", "t13688", "t13689", "t13690", "t13691", "t13769", "t13773",
"t07151", "t13684", "t07523", "t12369", "t12567", "ar00627", "ar00052", "t00352", "t07275",
"t12318", "t04931", "t13683", "t13686", "t13687", "t13688", "t13689", "t13690", "t13691",
"t07766", "t07918", "t12993", "ar00044", "t13326", "t07614", "t12414",
];
#[allow(clippy::mutex_atomic)]
pub async fn test(store: SearchStore, do_insert: bool) {
println!("Running Store query tests...");
@@ -95,8 +113,34 @@ pub async fn test(store: SearchStore, do_insert: bool) {
.unwrap();
let now = Instant::now();
let documents = Arc::new(Mutex::new(Vec::new()));
let mut mask = RoaringBitmap::new();
let mut fields = AHashMap::new();
// Global ids test
println!("Running global id filtering tests...");
test_global(store.clone()).await;
if do_insert {
let filter_ids = std::env::var("QUICK_TEST").is_ok().then(|| {
let mut ids = AHashSet::new();
for &id in ALL_IDS {
ids.insert(id.to_string());
let id = id.as_bytes();
if id.last().unwrap() > &b'0' {
let mut alt_id = id.to_vec();
*alt_id.last_mut().unwrap() -= 1;
ids.insert(String::from_utf8(alt_id).unwrap());
}
if id.last().unwrap() < &b'9' {
let mut alt_id = id.to_vec();
*alt_id.last_mut().unwrap() += 1;
ids.insert(String::from_utf8(alt_id).unwrap());
}
}
ids
});
pool.scope_fifo(|s| {
for (document_id, record) in csv::ReaderBuilder::new()
.has_headers(true)
@@ -107,18 +151,25 @@ pub async fn test(store: SearchStore, do_insert: bool) {
let record = record.unwrap();
let documents = documents.clone();
if let Some(filter_ids) = &filter_ids {
let id = record.get(1).unwrap().to_lowercase();
if !filter_ids.contains(&id) {
continue;
}
}
s.spawn_fifo(move |_| {
let mut document = IndexDocument::new(SearchIndex::Email)
.with_account_id(0)
.with_document_id(document_id as u32);
for (pos, field) in record.iter().enumerate() {
let field_id = pos as u8;
match FIELD_MAPPINGS[pos] {
EmailSearchField::From
| EmailSearchField::To
| EmailSearchField::Cc => {
| EmailSearchField::Cc
| EmailSearchField::Bcc => {
document.index_text(
FIELD_MAPPINGS[pos],
FIELD_MAPPINGS[pos].clone(),
&field.to_lowercase(),
Language::None,
);
@@ -127,7 +178,7 @@ pub async fn test(store: SearchStore, do_insert: bool) {
| EmailSearchField::Body
| EmailSearchField::Attachment => {
document.index_text(
FIELD_MAPPINGS[pos],
FIELD_MAPPINGS[pos].clone(),
&field.to_lowercase(),
Language::English,
);
@@ -143,7 +194,7 @@ pub async fn test(store: SearchStore, do_insert: bool) {
| EmailSearchField::SentAt
| EmailSearchField::Size => {
document.index_unsigned(
FIELD_MAPPINGS[pos],
FIELD_MAPPINGS[pos].clone(),
field.parse::<u64>().unwrap_or(0),
);
}
@@ -166,36 +217,60 @@ pub async fn test(store: SearchStore, do_insert: bool) {
let now = Instant::now();
let batches = documents.lock().unwrap().drain(..).collect::<Vec<_>>();
let mut chunk = Vec::new();
let mut fts_chunk = Vec::new();
print!("Inserting... ",);
let mut chunks = Vec::new();
let mut chunk = Vec::new();
for document in batches {
let chunk_instance = Instant::now();
chunk.push({
let db = db.clone();
tokio::spawn(async move { db.write(batch.build_all()).await })
});
fts_chunk.push({
let fts_store = fts_store.clone();
tokio::spawn(async move { fts_store.index(fts_batch).await })
});
if chunk.len() == 1000 {
for handle in chunk {
handle.await.unwrap().unwrap();
let mut document_id = None;
let mut to_field = None;
for (key, value) in document.fields() {
if key == &SearchField::DocumentId {
if let SearchValue::Uint(id) = value {
document_id = Some(*id as u32);
}
} else if key == &SearchField::Email(EmailSearchField::To)
&& let SearchValue::Text { value, .. } = value
{
to_field = Some(value.to_string());
}
for handle in fts_chunk {
}
let document_id = document_id.unwrap();
let to_field = to_field.unwrap();
mask.insert(document_id);
fields.insert(document_id, to_field);
chunk.push(document);
if chunk.len() == 10 {
chunks.push(chunk);
chunk = Vec::new();
}
}
if !chunk.is_empty() {
chunks.push(chunk);
}
let mut tasks = Vec::new();
for chunk in chunks {
let chunk_instance = Instant::now();
tasks.push({
let db = store.clone();
tokio::spawn(async move { db.index(chunk).await })
});
if tasks.len() == 100 {
for handle in tasks {
handle.await.unwrap().unwrap();
}
print!(" [{} ms]", chunk_instance.elapsed().as_millis());
std::io::stdout().flush().unwrap();
chunk = Vec::new();
fts_chunk = Vec::new();
tasks = Vec::new();
}
}
if !chunk.is_empty() {
for handle in chunk {
if !tasks.is_empty() {
for handle in tasks {
handle.await.unwrap().unwrap();
}
}
@@ -203,25 +278,29 @@ pub async fn test(store: SearchStore, do_insert: bool) {
println!("\nInsert took {} ms.", now.elapsed().as_millis());
}
println!("Running filter tests...");
println!("Running account filter tests...");
let now = Instant::now();
test_filter(db.clone(), fts_store).await;
test_filter(store.clone(), &fields, &mask).await;
println!("Filtering took {} ms.", now.elapsed().as_millis());
println!("Running sort tests...");
println!("Running account sort tests...");
let now = Instant::now();
test_sort(db).await;
test_sort(store.clone(), &fields, &mask).await;
println!("Sorting took {} ms.", now.elapsed().as_millis());
println!("Running unindex tests...");
let now = Instant::now();
test_unindex(store.clone(), &fields).await;
println!("Unindexing took {} ms.", now.elapsed().as_millis());
}
pub async fn test_filter(
store: SearchStore,
fields: &AHashMap<u32, &'static str>,
mask: &RoaringBitmap,
) {
async fn test_filter(store: SearchStore, fields: &AHashMap<u32, String>, mask: &RoaringBitmap) {
let can_stem = !matches!(store, SearchStore::Store(Store::MySQL(_)));
let tests = [
(
vec![
SearchFilter::eq(SearchField::AccountId, 0u32),
SearchFilter::has_english_text(EmailSearchField::Subject, "water"),
SearchFilter::eq(EmailSearchField::ReceivedAt, 1979u32),
],
@@ -229,6 +308,7 @@ pub async fn test_filter(
),
(
vec![
SearchFilter::eq(SearchField::AccountId, 0u32),
SearchFilter::has_keyword(EmailSearchField::From, "gelatin"),
SearchFilter::gt(EmailSearchField::ReceivedAt, 2000u32),
SearchFilter::lt(EmailSearchField::Size, 180u32),
@@ -237,27 +317,32 @@ pub async fn test_filter(
vec!["p79426", "p79427", "p79428", "p79429", "p79430"],
),
(
vec![SearchFilter::has_english_text(
EmailSearchField::Subject,
"'rustic bridge'",
)],
vec![
SearchFilter::eq(SearchField::AccountId, 0u32),
SearchFilter::has_english_text(EmailSearchField::Subject, "'rustic bridge'"),
],
vec!["d05503"],
),
(
vec![
SearchFilter::eq(SearchField::AccountId, 0u32),
SearchFilter::has_english_text(EmailSearchField::Subject, "'rustic'"),
SearchFilter::has_english_text(EmailSearchField::Subject, "study"),
SearchFilter::has_english_text(
EmailSearchField::Subject,
if can_stem { "study" } else { "studies" },
),
],
vec!["d00399", "d05352"],
),
(
vec![
SearchFilter::eq(SearchField::AccountId, 0u32),
SearchFilter::cond(
EmailSearchField::Headers,
SearchOperator::Contains,
SearchValue::KeyValues(VecMap::from_iter([(
"artist".to_string(),
"kunst mauro".to_string(),
"kunst, mauro".to_string(),
)])),
),
SearchFilter::has_keyword(EmailSearchField::Cc, "artist"),
@@ -270,10 +355,14 @@ pub async fn test_filter(
),
(
vec![
SearchFilter::eq(SearchField::AccountId, 0u32),
SearchFilter::Not,
SearchFilter::has_keyword(EmailSearchField::From, "oil"),
SearchFilter::End,
SearchFilter::has_english_text(EmailSearchField::Body, "bequeath"),
SearchFilter::has_english_text(
EmailSearchField::Body,
if can_stem { "bequeath" } else { "bequeathed" },
),
SearchFilter::Or,
SearchFilter::And,
SearchFilter::ge(EmailSearchField::ReceivedAt, 1900u32),
@@ -294,6 +383,7 @@ pub async fn test_filter(
(
vec![
SearchFilter::And,
SearchFilter::eq(SearchField::AccountId, 0u32),
SearchFilter::cond(
EmailSearchField::Headers,
SearchOperator::Contains,
@@ -320,20 +410,45 @@ pub async fn test_filter(
vec!["ar00039", "t12600"],
),
(
vec![
SearchFilter::has_english_text(EmailSearchField::Subject, "study"),
SearchFilter::has_keyword(EmailSearchField::From, "paper"),
SearchFilter::has_english_text(EmailSearchField::Body, "'purchased'"),
SearchFilter::Not,
SearchFilter::has_english_text(EmailSearchField::Subject, "'anatomical'"),
SearchFilter::has_english_text(EmailSearchField::Subject, "'for'"),
SearchFilter::End,
SearchFilter::gt(EmailSearchField::ReceivedAt, 1900u32),
SearchFilter::gt(EmailSearchField::Bcc, "2008".to_string()),
],
vec![
"p80042", "p80043", "p80044", "p80045", "p80203", "t11937", "t12172",
],
if can_stem {
vec![
SearchFilter::eq(SearchField::AccountId, 0u32),
SearchFilter::has_english_text(EmailSearchField::Subject, "study"),
SearchFilter::has_keyword(EmailSearchField::From, "paper"),
SearchFilter::has_english_text(EmailSearchField::Body, "'purchased'"),
SearchFilter::Not,
SearchFilter::Or,
SearchFilter::has_english_text(EmailSearchField::Subject, "'anatomical'"),
SearchFilter::has_english_text(EmailSearchField::Subject, "'discarded'"),
SearchFilter::has_english_text(EmailSearchField::Subject, "'untitled'"),
SearchFilter::has_english_text(EmailSearchField::Subject, "'girl'"),
SearchFilter::End,
SearchFilter::End,
SearchFilter::gt(EmailSearchField::ReceivedAt, 1900u32),
SearchFilter::gt(EmailSearchField::Bcc, "2008".to_string()),
]
} else {
vec![
SearchFilter::eq(SearchField::AccountId, 0u32),
SearchFilter::Or,
SearchFilter::has_english_text(EmailSearchField::Subject, "study"),
SearchFilter::has_english_text(EmailSearchField::Subject, "studies"),
SearchFilter::End,
SearchFilter::has_keyword(EmailSearchField::From, "paper"),
SearchFilter::has_english_text(EmailSearchField::Body, "'purchased'"),
SearchFilter::Not,
SearchFilter::Or,
SearchFilter::has_english_text(EmailSearchField::Subject, "'anatomical'"),
SearchFilter::has_english_text(EmailSearchField::Subject, "'discarded'"),
SearchFilter::has_english_text(EmailSearchField::Subject, "'untitled'"),
SearchFilter::has_english_text(EmailSearchField::Subject, "'girl'"),
SearchFilter::End,
SearchFilter::End,
SearchFilter::gt(EmailSearchField::ReceivedAt, 1900u32),
SearchFilter::gt(EmailSearchField::Bcc, "2008".to_string()),
]
},
vec!["p80203", "t13209", "t13560", "t13561"],
),
];
@@ -351,17 +466,16 @@ pub async fn test_filter(
let mut results = Vec::new();
for document_id in ids {
results.push(*fields.get(&document_id).unwrap());
results.push(fields.get(&document_id).unwrap());
}
assert_eq!(results, expected_results);
}
}
pub async fn test_sort(
store: SearchStore,
fields: &AHashMap<u32, &'static str>,
mask: &RoaringBitmap,
) {
async fn test_sort(store: SearchStore, fields: &AHashMap<u32, String>, mask: &RoaringBitmap) {
let is_reversed =
matches!(store, SearchStore::Store(Store::MySQL(_))) || store.internal_fts().is_some();
let tests = [
(
vec![
@@ -409,11 +523,19 @@ pub async fn test_sort(
SearchComparator::descending(EmailSearchField::Cc),
SearchComparator::ascending(EmailSearchField::To),
],
vec![
"ar00627", "ar00052", "t00352", "t07275", "t12318", "t04931", "t13683", "t13686",
"t13687", "t13688", "t13689", "t13690", "t13691", "t07766", "t07918", "t12993",
"ar00044", "t13326", "t07614", "t12414",
],
if !is_reversed {
vec![
"ar00052", "ar00627", "t00352", "t07275", "t12318", "t04931", "t13683",
"t13686", "t13687", "t13688", "t13689", "t13690", "t13691", "t13769", "t13773",
"t07151", "t13684", "t07523", "t12369", "t12567",
]
} else {
vec![
"ar00627", "ar00052", "t00352", "t07275", "t12318", "t04931", "t13683",
"t13686", "t13687", "t13688", "t13689", "t13690", "t13691", "t07766", "t07918",
"t12993", "ar00044", "t13326", "t07614", "t12414",
]
},
),
];
@@ -430,9 +552,119 @@ pub async fn test_sort(
.unwrap();
let mut results = Vec::new();
for document_id in ids {
results.push(*fields.get(&document_id).unwrap());
for document_id in ids.into_iter().take(expected_results.len()) {
results.push(fields.get(&document_id).unwrap());
}
assert_eq!(results, expected_results);
}
}
async fn test_unindex(store: SearchStore, fields: &AHashMap<u32, String>) {
let ids = store
.query_account(
SearchQuery::new(SearchIndex::Email)
.with_mask(RoaringBitmap::from_iter(fields.keys().copied()))
.with_account_id(0)
.with_filter(SearchFilter::has_keyword(EmailSearchField::From, "paper")),
)
.await
.unwrap();
assert!(!ids.is_empty());
let expected_count = ids.len().saturating_sub(10);
let mut query = SearchQuery::new(SearchIndex::Email)
.with_account_id(0)
.with_filter(SearchFilter::Or);
for id in ids.into_iter().take(10) {
query = query.with_filter(SearchFilter::eq(SearchField::DocumentId, id));
}
query = query.with_filter(SearchFilter::End);
store.unindex(query).await.unwrap();
assert_eq!(
store
.query_account(
SearchQuery::new(SearchIndex::Email)
.with_account_id(0)
.with_filter(SearchFilter::has_keyword(EmailSearchField::From, "paper"))
.with_mask(RoaringBitmap::from_iter(fields.keys().copied())),
)
.await
.unwrap()
.len(),
expected_count
);
}
async fn test_global(store: SearchStore) {
// Insert global ids
for (id, queue_id, etyp, keywords) in [
(0, 1000u64, 1u64, "init start"),
(1, 1000u64, 2u64, "init complete"),
(2, 1001u64, 1u64, "process start"),
(3, 1001u64, 2u64, "process complete"),
(4, 1002u64, 1u64, "cleanup start"),
(5, 1002u64, 2u64, "cleanup complete"),
] {
let mut document = IndexDocument::new(SearchIndex::Tracing).with_id(id);
document.index_unsigned(TracingSearchField::QueueId, queue_id);
document.index_unsigned(TracingSearchField::EventType, etyp);
document.index_text(TracingSearchField::Keywords, keywords, Language::None);
store.index(vec![document]).await.unwrap();
}
// Query all
assert_eq!(
store
.query_global(
SearchQuery::new(SearchIndex::Tracing)
.with_filter(SearchFilter::ge(SearchField::Id, 0u64))
)
.await
.unwrap()
.into_iter()
.collect::<AHashSet<_>>(),
AHashSet::from_iter([0, 1, 2, 3, 4, 5])
);
// Query with filter
assert_eq!(
store
.query_global(
SearchQuery::new(SearchIndex::Tracing)
.with_filter(SearchFilter::gt(SearchField::Id, 1u64))
.with_filter(SearchFilter::lt(SearchField::Id, 5u64))
.with_filter(SearchFilter::has_keyword(
TracingSearchField::Keywords,
"start",
)),
)
.await
.unwrap()
.into_iter()
.collect::<AHashSet<_>>(),
AHashSet::from_iter([2, 4])
);
// Delete by filter
store
.unindex(
SearchQuery::new(SearchIndex::Tracing)
.with_filter(SearchFilter::lt(SearchField::Id, 3u64)),
)
.await
.unwrap();
assert_eq!(
store
.query_global(
SearchQuery::new(SearchIndex::Tracing)
.with_filter(SearchFilter::ge(SearchField::Id, 0u64))
)
.await
.unwrap()
.into_iter()
.collect::<AHashSet<_>>(),
AHashSet::from_iter([3, 4, 5])
);
}