filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Email query and thread merge tests passing

Browse Source
This commit is contained in:
Mauro D
2023-04-21 15:40:11 +00:00
parent 51b14ed79e
commit 46b5dc0425
20 changed files with 1401 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
crates/store/src/backend/foundationdb/read.rs
View File

@@ -42,6 +42,7 @@ impl ReadTransaction<'_> {
let begin = key.serialize();
key.block_num = u32::MAX;
let end = key.serialize();
let key_len = begin.len();
let mut values = self.trx.get_ranges(
RangeOption {
begin: KeySelector::first_greater_or_equal(begin),
@@ -56,10 +57,12 @@ impl ReadTransaction<'_> {
while let Some(values) = values.next().await {
for value in values? {
let key = value.key();
bm.deserialize_block(
value.value(),
key.deserialize_be_u32(key.len() - std::mem::size_of::<u32>())?,
);
if key.len() == key_len {
bm.deserialize_block(
value.value(),
key.deserialize_be_u32(key.len() - std::mem::size_of::<u32>())?,
);
}
}
}

32
crates/store/src/backend/sqlite/id_assign.rs
View File

@@ -42,28 +42,46 @@ impl IdCacheKey {
#[derive(Clone)]
pub struct IdAssigner {
pub available_document_ids: RoaringBitmap,
pub freed_document_ids: Option<RoaringBitmap>,
pub next_document_id: u32,
pub next_change_id: u64,
}
impl IdAssigner {
pub fn new(used_ids: Option<RoaringBitmap>, next_change_id: u64) -> Self {
let mut assigner = IdAssigner {
available_document_ids: RoaringBitmap::full(),
freed_document_ids: None,
next_document_id: 0,
next_change_id,
};
if let Some(used_ids) = used_ids {
assigner.available_document_ids ^= &used_ids;
if let Some(max) = used_ids.max() {
assigner.next_document_id = max + 1;
let mut freed_ids =
RoaringBitmap::from_sorted_iter(0..assigner.next_document_id).unwrap();
freed_ids ^= used_ids;
if !freed_ids.is_empty() {
assigner.freed_document_ids = Some(freed_ids);
}
}
}
assigner
}
pub fn assign_document_id(&mut self) -> u32 {
let id = self.available_document_ids.min().unwrap();
self.available_document_ids.remove(id);
id
if let Some(freed_ids) = &mut self.freed_document_ids {
let id = freed_ids.min().unwrap();
freed_ids.remove(id);
if freed_ids.is_empty() {
self.freed_document_ids = None;
}
id
} else {
let id = self.next_document_id;
self.next_document_id += 1;
id
}
}
pub fn assign_change_id(&mut self) -> u64 {

41
crates/store/src/backend/sqlite/read.rs
View File

@@ -38,6 +38,7 @@ impl ReadTransaction<'_> {
) -> crate::Result<()> {
let begin = key.serialize();
key.block_num = u32::MAX;
let key_len = begin.len();
let end = key.serialize();
let mut query = self
.conn
@@ -46,27 +47,29 @@ impl ReadTransaction<'_> {
while let Some(row) = rows.next()? {
let key = row.get_ref(0)?.as_bytes()?;
let block_num = key.deserialize_be_u32(key.len() - std::mem::size_of::<u32>())?;
if key.len() == key_len {
let block_num = key.deserialize_be_u32(key.len() - std::mem::size_of::<u32>())?;
for word_num in 0..WORDS_PER_BLOCK {
match row.get::<_, i64>((word_num + 1) as usize)? as u64 {
0 => (),
u64::MAX => {
bm.insert_range(
block_num * BITS_PER_BLOCK + word_num * WORD_SIZE_BITS
..(block_num * BITS_PER_BLOCK + word_num * WORD_SIZE_BITS)
+ WORD_SIZE_BITS,
);
}
mut word => {
while word != 0 {
let trailing_zeros = word.trailing_zeros();
bm.insert(
block_num * BITS_PER_BLOCK
+ word_num * WORD_SIZE_BITS
+ trailing_zeros,
for word_num in 0..WORDS_PER_BLOCK {
match row.get::<_, i64>((word_num + 1) as usize)? as u64 {
0 => (),
u64::MAX => {
bm.insert_range(
block_num * BITS_PER_BLOCK + word_num * WORD_SIZE_BITS
..(block_num * BITS_PER_BLOCK + word_num * WORD_SIZE_BITS)
+ WORD_SIZE_BITS,
);
word ^= 1 << trailing_zeros;
}
mut word => {
while word != 0 {
let trailing_zeros = word.trailing_zeros();
bm.insert(
block_num * BITS_PER_BLOCK
+ word_num * WORD_SIZE_BITS
+ trailing_zeros,
);
word ^= 1 << trailing_zeros;
}
}
}
}

4
crates/store/src/fts/builder.rs
View File

@@ -65,6 +65,10 @@ impl<'x> FtsIndexBuilder<'x> {
self.tokens.insert((field, token));
}
}
pub fn index_raw_token(&mut self, field: impl Into<u8>, token: impl Into<String>) {
self.tokens.insert((field.into(), token.into()));
}
}
impl<'x> IntoOperations for FtsIndexBuilder<'x> {

4
crates/store/src/query/filter.rs
View File

@@ -69,6 +69,10 @@ impl ReadTransaction<'_> {
)
.await?
}
TextMatch::Raw => {
self.get_bitmap(BitmapKey::hash(&text, account_id, collection, 0, field))
.await?
}
},
Filter::InBitmap { family, field, key } => {
self.get_bitmap(BitmapKey {

52
crates/store/src/query/mod.rs
View File

@@ -49,6 +49,7 @@ pub enum TextMatch {
Exact(Language),
Stemmed(Language),
Tokenized,
Raw,
}
#[derive(Debug)]
@@ -129,27 +130,32 @@ impl Filter {
}
}
pub fn has_text(field: impl Into<u8>, text: impl Into<String>, mut language: Language) -> Self {
pub fn has_text_detect(
field: impl Into<u8>,
text: impl Into<String>,
default_language: Language,
) -> Self {
let mut text = text.into();
let language = if let Some((l, t)) = text
.split_once(':')
.and_then(|(l, t)| (Language::from_iso_639(l)?, t.to_string()).into())
{
text = t;
l
} else {
LanguageDetector::detect_single(&text)
.and_then(|(l, c)| if c > 0.3 { Some(l) } else { None })
.unwrap_or(default_language)
};
Self::has_text(field, text, language)
}
pub fn has_text(field: impl Into<u8>, text: impl Into<String>, language: Language) -> Self {
let text = text.into();
let op = if !matches!(language, Language::None) {
let match_phrase = (text.starts_with('"') && text.ends_with('"'))
|| (text.starts_with('\'') && text.ends_with('\''));
if !match_phrase && language == Language::Unknown {
language = if let Some((l, t)) = text
.split_once(':')
.and_then(|(l, t)| (Language::from_iso_639(l)?, t.to_string()).into())
{
text = t;
l
} else {
LanguageDetector::detect_single(&text)
.and_then(|(l, c)| if c > 0.3 { Some(l) } else { None })
.unwrap_or(Language::Unknown)
};
}
if match_phrase {
if (text.starts_with('"') && text.ends_with('"'))
|| (text.starts_with('\'') && text.ends_with('\''))
{
TextMatch::Exact(language)
} else {
TextMatch::Stemmed(language)
@@ -165,6 +171,14 @@ impl Filter {
}
}
pub fn has_raw_text(field: impl Into<u8>, text: impl Into<String>) -> Self {
Filter::HasText {
field: field.into(),
text: text.into(),
op: TextMatch::Raw,
}
}
pub fn has_english_text(field: impl Into<u8>, text: impl Into<String>) -> Self {
Self::has_text(field, text, Language::English)
}

21
crates/store/src/query/sort.rs
View File

@@ -1,3 +1,5 @@
use std::cmp::Ordering;
use ahash::{AHashMap, AHashSet};
use crate::{ReadTransaction, Store, ValueKey};
@@ -155,7 +157,10 @@ impl ReadTransaction<'_> {
let mut seen_prefixes = AHashSet::new();
let mut sorted_ids = sorted_ids.into_iter().collect::<Vec<_>>();
sorted_ids.sort_by(|a, b| a.1.cmp(&b.1));
sorted_ids.sort_by(|a, b| match a.1.cmp(&b.1) {
Ordering::Equal => a.0.cmp(&b.0),
other => other,
});
for (document_id, _) in sorted_ids {
// Obtain document prefixId
let prefix_id = if let Some(prefix_key) = &paginate.prefix_key {
@@ -252,13 +257,17 @@ impl Pagination {
// Pagination
if !self.has_anchor {
if self.position > 0 {
self.position -= 1;
if self.position >= 0 {
if self.position > 0 {
self.position -= 1;
} else {
self.ids.push(id);
if self.ids.len() == self.limit {
return false;
}
}
} else {
self.ids.push(id);
if self.ids.len() == self.limit {
return false;
}
}
} else if self.anchor_offset >= 0 {
if !self.anchor_found {

9
crates/store/src/write/mod.rs
View File

@@ -347,7 +347,14 @@ impl AssertValue {
pub fn matches(&self, bytes: &[u8]) -> bool {
match self {
AssertValue::U32(v) => {
bytes.len() == std::mem::size_of::<u32>() && u32::deserialize(bytes).unwrap() == *v
let coco = "fd";
let a = u32::deserialize(bytes).unwrap();
let b = *v;
if a != b {
println!("has {} != expected {}", a, b);
}
a == b
//bytes.len() == std::mem::size_of::<u32>() && u32::deserialize(bytes).unwrap() == *v
}
AssertValue::U64(v) => {
bytes.len() == std::mem::size_of::<u64>() && u64::deserialize(bytes).unwrap() == *v