filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

SQLite support

Browse Source
This commit is contained in:
Mauro D
2023-04-02 17:10:59 +00:00
parent b4e392d1c2
commit 4d192de2fe
19 changed files with 1086 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
src/query/filter.rs
View File

@@ -2,7 +2,7 @@ use std::ops::{BitAndAssign, BitOrAssign, BitXorAssign};
use roaring::RoaringBitmap;
use crate::{write::Tokenize, BitmapKey, Store, BM_KEYWORD};
use crate::{write::Tokenize, BitmapKey, ReadTransaction, Store, BM_KEYWORD};
use super::{Filter, ResultSet};
@@ -11,22 +11,22 @@ struct State {
bm: Option<RoaringBitmap>,
}
impl Store {
impl ReadTransaction<'_> {
#[maybe_async::maybe_async]
pub async fn filter(
&self,
&mut self,
account_id: u32,
collection: u8,
filters: Vec<Filter>,
) -> crate::Result<ResultSet> {
let mut trx = self.read_transaction().await?;
let mut not_mask = RoaringBitmap::new();
let mut not_fetch = false;
if filters.is_empty() {
return Ok(ResultSet {
account_id,
collection,
results: trx
.get_document_ids(account_id, collection)
results: self
.get_bitmap(BitmapKey::new_document_ids(account_id, collection))
.await?
.unwrap_or_else(RoaringBitmap::new),
});
@@ -37,23 +37,22 @@ impl Store {
let mut filters = filters.into_iter().peekable();
while let Some(filter) = filters.next() {
trx.refresh_if_old().await?;
self.refresh_if_old().await?;
let result = match filter {
Filter::HasKeyword { field, value } => {
trx.get_bitmap(BitmapKey {
self.get_bitmap(BitmapKey {
account_id,
collection,
family: BM_KEYWORD,
field,
key: value.as_bytes(),
#[cfg(feature = "foundation")]
block_num: 0,
})
.await?
}
Filter::HasKeywords { field, value } => {
trx.get_bitmaps_intersection(
self.get_bitmaps_intersection(
value
.tokenize()
.into_iter()
@@ -63,7 +62,6 @@ impl Store {
family: BM_KEYWORD,
field,
key: key.into_bytes(),
#[cfg(feature = "foundation")]
block_num: 0,
})
.collect(),
@@ -71,7 +69,7 @@ impl Store {
.await?
}
Filter::MatchValue { field, op, value } => {
trx.range_to_bitmap(account_id, collection, field, value, op)
self.range_to_bitmap(account_id, collection, field, value, op)
.await?
}
Filter::HasText {
@@ -80,17 +78,16 @@ impl Store {
language,
match_phrase,
} => {
trx.fts_query(account_id, collection, field, &text, language, match_phrase)
self.fts_query(account_id, collection, field, &text, language, match_phrase)
.await?
}
Filter::InBitmap { family, field, key } => {
trx.get_bitmap(BitmapKey {
self.get_bitmap(BitmapKey {
account_id,
collection,
family,
field,
key: &key,
#[cfg(feature = "foundation")]
block_num: 0,
})
.await?
@@ -113,8 +110,8 @@ impl Store {
};
if matches!(state.op, Filter::Not) && !not_fetch {
not_mask = trx
.get_document_ids(account_id, collection)
not_mask = self
.get_bitmap(BitmapKey::new_document_ids(account_id, collection))
.await?
.unwrap_or_else(RoaringBitmap::new);
not_fetch = true;
@@ -122,8 +119,6 @@ impl Store {
state.op.apply(&mut state.bm, result, &not_mask);
//println!("{:?}: {:?}", state.op, state.bm);
if matches!(state.op, Filter::And) && state.bm.as_ref().unwrap().is_empty() {
while let Some(filter) = filters.peek() {
if matches!(filter, Filter::End) {
@@ -143,6 +138,30 @@ impl Store {
}
}
impl Store {
pub async fn filter(
&self,
account_id: u32,
collection: u8,
filters: Vec<Filter>,
) -> crate::Result<ResultSet> {
#[cfg(feature = "is_async")]
{
self.read_transaction()
.await?
.filter(account_id, collection, filters)
.await
}
#[cfg(feature = "is_sync")]
{
let mut trx = self.read_transaction()?;
self.spawn_worker(move || trx.filter(account_id, collection, filters))
.await
}
}
}
impl Filter {
#[inline(always)]
pub fn apply(

51
src/query/get.rs Normal file
View File

@@ -0,0 +1,51 @@
use crate::{Deserialize, Store, ValueKey};
impl Store {
pub async fn get_value<U>(&self, key: ValueKey) -> crate::Result<Option<U>>
where
U: Deserialize + 'static,
{
#[cfg(feature = "is_async")]
{
self.read_transaction().await?.get_value(key).await
}
#[cfg(feature = "is_sync")]
{
let trx = self.read_transaction()?;
self.spawn_worker(move || trx.get_value(key)).await
}
}
pub async fn get_values<U>(&self, key: Vec<ValueKey>) -> crate::Result<Vec<Option<U>>>
where
U: Deserialize + 'static,
{
#[cfg(feature = "is_async")]
{
let mut trx = self.read_transaction().await?;
let mut results = Vec::with_capacity(key.len());
for key in key {
trx.refresh_if_old().await?;
results.push(trx.get_value(key).await?);
}
Ok(results)
}
#[cfg(feature = "is_sync")]
{
let trx = self.read_transaction()?;
self.spawn_worker(move || {
let mut results = Vec::with_capacity(key.len());
for key in key {
results.push(trx.get_value(key)?);
}
Ok(results)
})
.await
}
}
}

16
src/query/mod.rs
View File

@@ -1,4 +1,5 @@
pub mod filter;
pub mod get;
pub mod log;
pub mod sort;
@@ -6,7 +7,7 @@ use roaring::RoaringBitmap;
use crate::{
fts::{lang::LanguageDetector, Language},
Serialize,
BitmapKey, Serialize, BM_DOCUMENT_IDS,
};
#[derive(Debug, Clone, Copy)]
@@ -185,3 +186,16 @@ impl Comparator {
}
}
}
impl BitmapKey<&'static [u8]> {
pub fn new_document_ids(account_id: u32, collection: u8) -> Self {
BitmapKey {
account_id,
collection,
family: BM_DOCUMENT_IDS,
field: u8::MAX,
key: b"",
block_num: 0,
}
}
}

100
src/query/sort.rs
View File

@@ -1,12 +1,24 @@
use ahash::AHashMap;
use crate::Store;
use crate::{ReadTransaction, Store};
use super::{Comparator, ResultSet, SortedResultRet};
impl Store {
pub struct Pagination {
requested_position: i32,
position: i32,
limit: usize,
anchor: u32,
anchor_offset: i32,
has_anchor: bool,
anchor_found: bool,
ids: Vec<u32>,
}
impl ReadTransaction<'_> {
#[maybe_async::maybe_async]
pub async fn sort(
&self,
&mut self,
result_set: ResultSet,
mut comparators: Vec<Comparator>,
limit: usize,
@@ -14,27 +26,14 @@ impl Store {
anchor: Option<u32>,
anchor_offset: i32,
) -> crate::Result<SortedResultRet> {
let limit = match (result_set.results.len(), limit) {
(0, _) => {
return Ok(SortedResultRet {
position,
ids: vec![],
found_anchor: true,
});
}
(_, 0) => result_set.results.len() as usize,
(a, b) => std::cmp::min(a as usize, b),
};
let mut paginate = Pagination::new(limit, position, anchor, anchor_offset);
if comparators.len() == 1 {
match comparators.pop().unwrap() {
Comparator::Field { field, ascending } => {
let trx = self.read_transaction().await?;
let mut results = result_set.results;
trx.sort_index(
self.sort_index(
result_set.account_id,
result_set.collection,
field,
@@ -70,7 +69,6 @@ impl Store {
}
}
} else {
let mut trx = self.read_transaction().await?;
let mut sorted_ids = AHashMap::with_capacity(paginate.limit);
for (pos, comparator) in comparators.into_iter().take(4).enumerate() {
@@ -81,8 +79,8 @@ impl Store {
let mut has_grouped_ids = false;
let mut idx = 0;
trx.refresh_if_old().await?;
trx.sort_index(
self.refresh_if_old().await?;
self.sort_index(
result_set.account_id,
result_set.collection,
field,
@@ -153,15 +151,59 @@ impl Store {
}
}
pub struct Pagination {
requested_position: i32,
position: i32,
limit: usize,
anchor: u32,
anchor_offset: i32,
has_anchor: bool,
anchor_found: bool,
ids: Vec<u32>,
impl Store {
pub async fn sort(
&self,
result_set: ResultSet,
comparators: Vec<Comparator>,
limit: usize,
position: i32,
anchor: Option<u32>,
anchor_offset: i32,
) -> crate::Result<SortedResultRet> {
let limit = match (result_set.results.len(), limit) {
(0, _) => {
return Ok(SortedResultRet {
position,
ids: vec![],
found_anchor: true,
});
}
(_, 0) => result_set.results.len() as usize,
(a, b) => std::cmp::min(a as usize, b),
};
#[cfg(feature = "is_async")]
{
self.read_transaction()
.await?
.sort(
result_set,
comparators,
limit,
position,
anchor,
anchor_offset,
)
.await
}
#[cfg(feature = "is_sync")]
{
let mut trx = self.read_transaction()?;
self.spawn_worker(move || {
trx.sort(
result_set,
comparators,
limit,
position,
anchor,
anchor_offset,
)
})
.await
}
}
}
impl Pagination {