filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

FoundationDB first impl

Browse Source
This commit is contained in:
Mauro D
2023-03-30 06:26:53 +00:00
parent 05d53da6fb
commit d74731f3c6
25 changed files with 1555 additions and 725 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
src/query/filter.rs
View File

@@ -1,10 +1,13 @@
use std::ops::{BitAndAssign, BitOrAssign, BitXorAssign};
use std::{
ops::{BitAndAssign, BitOrAssign, BitXorAssign},
time::Instant,
};
use roaring::RoaringBitmap;
use crate::{
write::{key::KeySerializer, Tokenize},
BitmapKey, IndexKey, Store, BM_TERM, TERM_EXACT,
backend::foundationdb::read::ReadTransaction, write::Tokenize, BitmapKey, Store, BM_TERM,
TERM_EXACT,
};
use super::{Filter, ResultSet};
@@ -15,17 +18,21 @@ struct State {
}
impl Store {
pub fn filter(
pub async fn filter(
&self,
account_id: u32,
collection: u8,
filters: Vec<Filter>,
) -> crate::Result<ResultSet> {
let document_ids = self
.get_document_ids(account_id, collection)?
let mut trx = self.read_transaction().await?;
let document_ids = trx
.get_document_ids(account_id, collection)
.await?
.unwrap_or_else(RoaringBitmap::new);
if filters.is_empty() {
return Ok(ResultSet {
account_id,
collection,
results: document_ids.clone(),
document_ids,
});
@@ -36,52 +43,42 @@ impl Store {
let mut filters = filters.into_iter().peekable();
while let Some(filter) = filters.next() {
match filter {
trx.refresh_if_old().await?;
let result = match filter {
Filter::HasKeyword { field, value } => {
state.op.apply(
&mut state.bm,
self.get_bitmap(BitmapKey {
account_id,
collection,
family: BM_TERM | TERM_EXACT,
field,
key: value.as_bytes(),
})?,
&document_ids,
);
trx.get_bitmap(BitmapKey {
account_id,
collection,
family: BM_TERM | TERM_EXACT,
field,
key: value.as_bytes(),
#[cfg(feature = "foundation")]
block_num: 0,
})
.await?
}
Filter::HasKeywords { field, value } => {
let tokens = value.tokenize();
state.op.apply(
&mut state.bm,
self.get_bitmaps_intersection(
tokens
.iter()
.map(|key| BitmapKey {
account_id,
collection,
family: BM_TERM | TERM_EXACT,
field,
key: key.as_bytes(),
})
.collect(),
)?,
&document_ids,
);
trx.get_bitmaps_intersection(
value
.tokenize()
.into_iter()
.map(|key| BitmapKey {
account_id,
collection,
family: BM_TERM | TERM_EXACT,
field,
key: key.into_bytes(),
#[cfg(feature = "foundation")]
block_num: 0,
})
.collect(),
)
.await?
}
Filter::MatchValue { field, op, value } => {
let key =
KeySerializer::new(std::mem::size_of::<IndexKey<&[u8]>>() + value.len())
.write(account_id)
.write(collection)
.write(field)
.write(&value[..])
.finalize();
state.op.apply(
&mut state.bm,
self.range_to_bitmap(&key, &value, op)?,
&document_ids,
);
trx.range_to_bitmap(account_id, collection, field, value, op)
.await?
}
Filter::HasText {
field,
@@ -89,51 +86,39 @@ impl Store {
language,
match_phrase,
} => {
state.op.apply(
&mut state.bm,
self.fts_query(
account_id,
collection,
field,
&text,
language,
match_phrase,
)?,
&document_ids,
);
self.fts_query(account_id, collection, field, &text, language, match_phrase)
.await?
}
Filter::InBitmap { family, field, key } => {
state.op.apply(
&mut state.bm,
self.get_bitmap(BitmapKey {
account_id,
collection,
family,
field,
key: &key,
})?,
&document_ids,
);
}
Filter::DocumentSet(set) => {
state.op.apply(&mut state.bm, Some(set), &document_ids);
trx.get_bitmap(BitmapKey {
account_id,
collection,
family,
field,
key: &key,
#[cfg(feature = "foundation")]
block_num: 0,
})
.await?
}
Filter::DocumentSet(set) => Some(set),
op @ (Filter::And | Filter::Or | Filter::Not) => {
stack.push(state);
state = op.into();
continue;
}
Filter::End => {
if let Some(mut prev_state) = stack.pop() {
prev_state
.op
.apply(&mut prev_state.bm, state.bm, &document_ids);
if let Some(prev_state) = stack.pop() {
let bm = state.bm;
state = prev_state;
bm
} else {
break;
}
}
}
};
state.op.apply(&mut state.bm, result, &document_ids);
//println!("{:?}: {:?}", state.op, state.bm);
@@ -149,6 +134,8 @@ impl Store {
}
Ok(ResultSet {
account_id,
collection,
results: state.bm.unwrap_or_else(RoaringBitmap::new),
document_ids,
})

109
src/query/log.rs Normal file
View File

@@ -0,0 +1,109 @@
use utils::codec::leb128::Leb128Iterator;
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
pub enum Change {
Insert(u64),
Update(u64),
ChildUpdate(u64),
Delete(u64),
}
pub struct Changes {
pub changes: Vec<Change>,
pub from_change_id: u64,
pub to_change_id: u64,
}
#[derive(Debug)]
pub enum Query {
All,
Since(u64),
SinceInclusive(u64),
RangeInclusive(u64, u64),
}
impl Default for Changes {
fn default() -> Self {
Self {
changes: Vec::with_capacity(10),
from_change_id: 0,
to_change_id: 0,
}
}
}
impl Changes {
pub fn deserialize(&mut self, bytes: &[u8]) -> Option<()> {
let mut bytes_it = bytes.iter();
let total_inserts: usize = bytes_it.next_leb128()?;
let total_updates: usize = bytes_it.next_leb128()?;
let total_child_updates: usize = bytes_it.next_leb128()?;
let total_deletes: usize = bytes_it.next_leb128()?;
if total_inserts > 0 {
for _ in 0..total_inserts {
self.changes.push(Change::Insert(bytes_it.next_leb128()?));
}
}
if total_updates > 0 || total_child_updates > 0 {
'update_outer: for change_pos in 0..(total_updates + total_child_updates) {
let id = bytes_it.next_leb128()?;
let mut is_child_update = change_pos >= total_updates;
for (idx, change) in self.changes.iter().enumerate() {
match change {
Change::Insert(insert_id) if *insert_id == id => {
// Item updated after inserted, no need to count this change.
continue 'update_outer;
}
Change::Update(update_id) if *update_id == id => {
// Move update to the front
is_child_update = false;
self.changes.remove(idx);
break;
}
Change::ChildUpdate(update_id) if *update_id == id => {
// Move update to the front
self.changes.remove(idx);
break;
}
_ => (),
}
}
self.changes.push(if !is_child_update {
Change::Update(id)
} else {
Change::ChildUpdate(id)
});
}
}
if total_deletes > 0 {
'delete_outer: for _ in 0..total_deletes {
let id = bytes_it.next_leb128()?;
'delete_inner: for (idx, change) in self.changes.iter().enumerate() {
match change {
Change::Insert(insert_id) if *insert_id == id => {
self.changes.remove(idx);
continue 'delete_outer;
}
Change::Update(update_id) | Change::ChildUpdate(update_id)
if *update_id == id =>
{
self.changes.remove(idx);
break 'delete_inner;
}
_ => (),
}
}
self.changes.push(Change::Delete(id));
}
}
Some(())
}
}

38
src/query/mod.rs
View File

@@ -1,4 +1,5 @@
pub mod filter;
pub mod log;
pub mod sort;
use roaring::RoaringBitmap;
@@ -58,8 +59,10 @@ pub enum Comparator {
#[derive(Debug)]
pub struct ResultSet {
results: RoaringBitmap,
document_ids: RoaringBitmap,
account_id: u32,
collection: u8,
pub results: RoaringBitmap,
pub document_ids: RoaringBitmap,
}
pub struct SortedResultRet {
@@ -68,6 +71,17 @@ pub struct SortedResultRet {
pub found_anchor: bool,
}
pub enum SortedId {
Id(u32),
GroupedId(Vec<u32>),
}
#[allow(clippy::len_without_is_empty)]
pub trait UnsortedIds {
fn contains_id(&self, id: u32) -> bool;
fn len(&self) -> usize;
}
impl Filter {
pub fn cond(field: impl Into<u8>, op: Operator, value: impl Serialize) -> Self {
Filter::MatchValue {
@@ -183,3 +197,23 @@ impl Comparator {
}
}
}
impl UnsortedIds for RoaringBitmap {
fn contains_id(&self, id: u32) -> bool {
self.contains(id)
}
fn len(&self) -> usize {
self.len() as usize
}
}
impl UnsortedIds for Vec<u32> {
fn contains_id(&self, id: u32) -> bool {
self.iter().any(|&i| i == id)
}
fn len(&self) -> usize {
self.len()
}
}

225
src/query/sort.rs
View File

@@ -1,18 +1,24 @@
use std::ops::{BitAndAssign, BitXorAssign};
use foundationdb::{future::FdbValue, options, FdbResult, KeySelector, RangeOption};
use futures::{Stream, StreamExt};
use roaring::RoaringBitmap;
#[cfg(feature = "rocks")]
use rocksdb::{
DBIteratorWithThreadMode, Direction, IteratorMode, MultiThreaded, OptimisticTransactionDB,
};
#[cfg(feature = "rocks")]
use crate::backend::rocksdb::{ACCOUNT_KEY_LEN, CF_INDEXES};
use crate::{
backend::rocksdb::{ACCOUNT_KEY_LEN, CF_INDEXES},
write::key::KeySerializer,
Error, Store,
backend::foundationdb::read::ReadTransaction, write::key::DeserializeBigEndian, Error,
IndexKeyPrefix, Serialize, Store,
};
use super::{Comparator, ResultSet, SortedResultRet};
#[cfg(feature = "rocks")]
enum IndexType<'x> {
DocumentSet {
set: RoaringBitmap,
@@ -21,25 +27,47 @@ enum IndexType<'x> {
DB {
it: Option<DBIteratorWithThreadMode<'x, OptimisticTransactionDB<MultiThreaded>>>,
prefix: Vec<u8>,
start_key: Vec<u8>,
from_key: Vec<u8>,
ascending: bool,
prev_item: Option<u32>,
prev_key: Option<Box<[u8]>>,
},
}
#[cfg(feature = "rocks")]
struct IndexIterator<'x> {
index: IndexType<'x>,
remaining: RoaringBitmap,
eof: bool,
}
#[cfg(feature = "foundation")]
enum IndexType<'x, T: Stream<Item = FdbResult<FdbValue>> + Unpin + 'x> {
DocumentSet {
set: RoaringBitmap,
it: Option<roaring::bitmap::IntoIter>,
},
DB {
it: Option<T>,
from_key: Vec<u8>,
to_key: Vec<u8>,
ascending: bool,
prev_item: Option<u32>,
prev_key: Option<Box<[u8]>>,
phantom: std::marker::PhantomData<&'x ()>,
},
}
#[cfg(feature = "foundation")]
struct IndexIterator<'x, T: Stream<Item = FdbResult<FdbValue>> + Unpin + 'x> {
index: IndexType<'x, T>,
remaining: RoaringBitmap,
eof: bool,
}
impl Store {
#[allow(clippy::too_many_arguments)]
pub fn sort(
pub async fn sort(
&self,
account_id: u32,
collection: u8,
mut result_set: ResultSet,
comparators: Vec<Comparator>,
limit: usize,
@@ -50,6 +78,7 @@ impl Store {
let has_anchor = anchor.is_some();
let mut anchor_found = false;
let requested_position = position;
let trx = self.read_transaction().await?;
let mut result = SortedResultRet {
position,
@@ -60,37 +89,23 @@ impl Store {
.into_iter()
.map(|comp| IndexIterator {
index: match comp {
Comparator::Field { field, ascending } => {
let prefix = KeySerializer::new(ACCOUNT_KEY_LEN)
.write(account_id)
.write(collection)
.write(field)
.finalize();
IndexType::DB {
it: None,
start_key: if !ascending {
let (key_account_id, key_collection, key_field) = if field < u8::MAX
{
(account_id, collection, field + 1)
} else if (collection) < u8::MAX {
(account_id, (collection) + 1, field)
} else {
(account_id + 1, collection, field)
};
KeySerializer::new(ACCOUNT_KEY_LEN)
.write(key_account_id)
.write(key_collection)
.write(key_field)
.finalize()
} else {
prefix.clone()
},
prefix,
ascending,
prev_item: None,
prev_key: None,
}
}
Comparator::Field { field, ascending } => IndexType::DB {
it: None,
from_key: if !ascending {
result_set.from_key(field).serialize()
} else {
result_set.to_key(field).serialize()
},
to_key: if !ascending {
result_set.to_key(field).serialize()
} else {
result_set.from_key(field).serialize()
},
ascending,
prev_item: None,
prev_key: None,
phantom: std::marker::PhantomData,
},
Comparator::DocumentSet { mut set, ascending } => IndexType::DocumentSet {
set: if !ascending {
if !set.is_empty() {
@@ -111,12 +126,13 @@ impl Store {
.collect::<Vec<_>>();
let mut current = 0;
let iter_len = iterators.len() - 1;
'outer: loop {
let mut doc_id;
'inner: loop {
let (it_opts, mut next_it_opts) = if current < iterators.len() - 1 {
let (it_opts, mut next_it_opts) = if current < iter_len {
let (iterators_first, iterators_last) = iterators.split_at_mut(current + 1);
(
iterators_first.last_mut().unwrap(),
@@ -145,26 +161,46 @@ impl Store {
match &mut it_opts.index {
IndexType::DB {
it,
prefix,
start_key,
from_key,
to_key,
ascending,
prev_item,
prev_key,
..
} => {
let it = if let Some(it) = it {
it
} else {
*it = Some(self.db.iterator_cf(
&self.db.cf_handle(CF_INDEXES).unwrap(),
IteratorMode::From(
start_key,
if *ascending {
Direction::Forward
} else {
Direction::Reverse
#[cfg(feature = "foundation")]
{
*it = Some(trx.trx.get_ranges_keyvalues(
RangeOption {
begin: KeySelector::first_greater_or_equal(
from_key.clone(),
),
end: KeySelector::last_less_than(to_key.clone()),
mode: options::StreamingMode::Iterator,
reverse: !*ascending,
..Default::default()
},
),
));
true,
));
}
#[cfg(feature = "rocks")]
{
*it = Some(self.db.iterator_cf(
&self.db.cf_handle(CF_INDEXES).unwrap(),
IteratorMode::From(
from_key,
if *ascending {
Direction::Forward
} else {
Direction::Reverse
},
),
));
}
it.as_mut().unwrap()
};
@@ -184,24 +220,20 @@ impl Store {
let mut is_eof = false;
loop {
if let Some(result) = it.next() {
let (key, _) = result.map_err(|e| {
if let Some(result) = it.next().await {
let key = result?.key().to_vec().into_boxed_slice();
/*let (key, _) = result.map_err(|e| {
Error::InternalError(format!("Iterator error: {}", e))
})?;
if !key.starts_with(prefix) {
*prev_key = None;
is_eof = true;
break;
}
}*/
doc_id = u32::from_be_bytes(
key.get(key.len() - std::mem::size_of::<u32>()..)
.ok_or_else(|| {
Error::InternalError("Invalid index entry".to_string())
})?
.try_into()
.unwrap(),
);
doc_id = key
.as_ref()
.deserialize_be_u32(key.len() - std::mem::size_of::<u32>())?;
if it_opts.remaining.contains(doc_id) {
it_opts.remaining.remove(doc_id);
@@ -305,24 +337,45 @@ impl Store {
match &mut next_it_opts.index {
IndexType::DB {
it,
start_key,
from_key,
to_key,
ascending,
prev_item,
prev_key,
..
} => {
if let Some(it) = it {
*it = self.db.iterator_cf(
&self.db.cf_handle(CF_INDEXES).unwrap(),
IteratorMode::From(
start_key,
if *ascending {
Direction::Forward
} else {
Direction::Reverse
#[cfg(feature = "rocks")]
{
*it = self.db.iterator_cf(
&self.db.cf_handle(CF_INDEXES).unwrap(),
IteratorMode::From(
from_key,
if *ascending {
Direction::Forward
} else {
Direction::Reverse
},
),
);
}
#[cfg(feature = "foundation")]
{
*it = trx.trx.get_ranges_keyvalues(
RangeOption {
begin: KeySelector::first_greater_or_equal(
from_key.clone(),
),
end: KeySelector::last_less_than(
to_key.clone(),
),
mode: options::StreamingMode::Iterator,
reverse: !*ascending,
..Default::default()
},
),
);
true,
);
}
}
*prev_item = None;
*prev_key = None;
@@ -422,3 +475,27 @@ impl Store {
Ok(result)
}
}
impl ResultSet {
pub fn from_key(&self, field: u8) -> IndexKeyPrefix {
IndexKeyPrefix {
account_id: self.account_id,
collection: self.collection,
field,
}
}
pub fn to_key(&self, field: u8) -> IndexKeyPrefix {
let (account_id, collection, field) = if field < u8::MAX {
(self.account_id, self.collection, field + 1)
} else if (self.collection) < u8::MAX {
(self.account_id, (self.collection) + 1, field)
} else {
(self.account_id + 1, self.collection, field)
};
IndexKeyPrefix {
account_id,
collection,
field,
}
}
}