filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Database schema optimization - part 3

Browse Source
This commit is contained in:
mdecimus
2025-11-01 19:19:36 +01:00
parent d5c5f12adf
commit 8e1316895b
41 changed files with 1979 additions and 1276 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crates/store/src/backend/foundationdb/read.rs
View File

@@ -47,6 +47,8 @@ impl FdbStore {
let begin = params.begin.serialize(WITH_SUBSPACE);
let end = params.end.serialize(WITH_SUBSPACE);
let todo = "fix fdb range scan to support chunked reads";
if !params.first {
let mut last_key = vec![];

1
crates/store/src/backend/mysql/main.rs
View File

@@ -104,7 +104,6 @@ impl MysqlStore {
SUBSPACE_LOGS,
SUBSPACE_TELEMETRY_SPAN,
SUBSPACE_TELEMETRY_METRIC,
SUBSPACE_TELEMETRY_INDEX,
] {
let table = char::from(table);
conn.query_drop(format!(

1
crates/store/src/backend/postgres/main.rs
View File

@@ -92,7 +92,6 @@ impl PostgresStore {
SUBSPACE_BLOBS,
SUBSPACE_TELEMETRY_SPAN,
SUBSPACE_TELEMETRY_METRIC,
SUBSPACE_TELEMETRY_INDEX,
] {
let table = char::from(table);
conn.execute(

1
crates/store/src/backend/rocksdb/main.rs
View File

@@ -74,7 +74,6 @@ impl RocksDbStore {
SUBSPACE_BLOBS,
SUBSPACE_TELEMETRY_SPAN,
SUBSPACE_TELEMETRY_METRIC,
SUBSPACE_TELEMETRY_INDEX,
] {
let cf_opts = Options::default();
cfs.push(ColumnFamilyDescriptor::new(

1
crates/store/src/backend/sqlite/main.rs
View File

@@ -105,7 +105,6 @@ impl SqliteStore {
SUBSPACE_BLOBS,
SUBSPACE_TELEMETRY_SPAN,
SUBSPACE_TELEMETRY_METRIC,
SUBSPACE_TELEMETRY_INDEX,
] {
let table = char::from(table);
conn.execute(

72
crates/store/src/dispatch/search.rs
View File

@@ -7,51 +7,44 @@
use super::DocumentSet;
use crate::{
SearchStore,
search::{IndexDocument, SearchComparator, SearchFilter},
backend::elastic::query,
search::{IndexDocument, SearchComparator, SearchDocumentId, SearchFilter, SearchQuery},
};
use trc::AddContext;
use types::collection::Collection;
impl SearchStore {
pub async fn index(&self, document: IndexDocument) -> trc::Result<()> {
match self {
pub async fn query<R: SearchDocumentId>(&self, query: SearchQuery) -> trc::Result<Vec<R>> {
todo!()
/*match self {
SearchStore::Store(store) => {
store
.index_query(account_id, collection, filters, comparators)
.await
}
#[cfg(feature = "elastic")]
SearchStore::ElasticSearch(store) => {
store
.index_query(account_id, collection, filters, comparators)
.await
}
}
.caused_by(trc::location!())*/
}
pub async fn index(&self, documents: Vec<IndexDocument>) -> trc::Result<()> {
todo!()
/*match self {
SearchStore::Store(store) => store.index_insert(document).await,
#[cfg(feature = "elastic")]
SearchStore::ElasticSearch(store) => store.index_insert(document).await,
}
.caused_by(trc::location!())
.caused_by(trc::location!())*/
}
pub async fn query(
&self,
account_id: u32,
collection: Collection,
filters: Vec<SearchFilter>,
comparators: Vec<SearchComparator>,
) -> trc::Result<Vec<u32>> {
match self {
SearchStore::Store(store) => {
store
.index_query(account_id, collection, filters, comparators)
.await
}
#[cfg(feature = "elastic")]
SearchStore::ElasticSearch(store) => {
store
.index_query(account_id, collection, filters, comparators)
.await
}
}
.caused_by(trc::location!())
}
pub async fn remove(
&self,
account_id: u32,
collection: Collection,
document_ids: &impl DocumentSet,
) -> trc::Result<()> {
match self {
pub async fn unindex(&self, query: SearchQuery) -> trc::Result<()> {
todo!()
/*match self {
SearchStore::Store(store) => {
store
.index_remove(account_id, collection, document_ids)
@@ -64,15 +57,6 @@ impl SearchStore {
.await
}
}
.caused_by(trc::location!())
}
pub async fn remove_all(&self, account_id: u32) -> trc::Result<()> {
match self {
SearchStore::Store(store) => store.index_remove_all(account_id).await,
#[cfg(feature = "elastic")]
SearchStore::ElasticSearch(store) => store.index_remove_all(account_id).await,
}
.caused_by(trc::location!())
.caused_by(trc::location!())*/
}
}

2
crates/store/src/dispatch/store.rs
View File

@@ -488,7 +488,6 @@ impl Store {
SUBSPACE_REPORT_IN,
SUBSPACE_TELEMETRY_SPAN,
SUBSPACE_TELEMETRY_METRIC,
SUBSPACE_TELEMETRY_INDEX,
] {
self.delete_range(
AnyKey {
@@ -667,7 +666,6 @@ impl Store {
(SUBSPACE_INDEXES, false),
(SUBSPACE_TELEMETRY_SPAN, true),
(SUBSPACE_TELEMETRY_METRIC, true),
(SUBSPACE_TELEMETRY_INDEX, true),
] {
let from_key = crate::write::AnyKey {
subspace,

3
crates/store/src/lib.rs
View File

@@ -102,7 +102,6 @@ pub const SUBSPACE_QUOTA: u8 = b'u';
pub const SUBSPACE_REPORT_OUT: u8 = b'h';
pub const SUBSPACE_REPORT_IN: u8 = b'r';
pub const SUBSPACE_TELEMETRY_SPAN: u8 = b'o';
pub const SUBSPACE_TELEMETRY_INDEX: u8 = b'w';
pub const SUBSPACE_TELEMETRY_METRIC: u8 = b'x';
pub const SUBSPACE_RESERVED_2: u8 = b'z';
@@ -111,7 +110,7 @@ pub const SUBSPACE_BITMAP_ID: u8 = b'b';
pub const SUBSPACE_BITMAP_TAG: u8 = b'c';
pub const SUBSPACE_BITMAP_TEXT: u8 = b'v';
pub const SUBSPACE_FTS_INDEX: u8 = b'g';
pub const SUBSPACE_TELEMETRY_INDEX: u8 = b'w';
*/
#[derive(Clone)]

6
crates/store/src/search/index.rs
View File

@@ -29,7 +29,7 @@ use crate::{
pub const TERM_INDEX_VERSION: u8 = 1;
impl Store {
pub async fn index_insert(&self, document: IndexDocument) -> trc::Result<()> {
pub(crate) async fn index_insert(&self, document: IndexDocument) -> trc::Result<()> {
/*let mut detect = LanguageDetector::new();
let mut tokens: AHashMap<BitmapHash, Postings> = AHashMap::new();
let mut parts = Vec::new();
@@ -142,7 +142,7 @@ impl Store {
Ok(())
}
pub async fn index_remove(
pub(crate) async fn index_remove(
&self,
account_id: u32,
collection: Collection,
@@ -244,7 +244,7 @@ impl Store {
Ok(())
}
pub async fn index_remove_all(&self, _: u32) -> trc::Result<()> {
pub(crate) async fn index_remove_all(&self, _: u32) -> trc::Result<()> {
// No-op
// Term indexes are stored in the same key range as the document

144
crates/store/src/search/mod.rs
View File

@@ -13,6 +13,8 @@ use nlp::language::Language;
use roaring::RoaringBitmap;
use std::{borrow::Cow, collections::hash_map::Entry};
use crate::write::SearchIndex;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SearchOperator {
LowerThan,
@@ -26,6 +28,9 @@ pub enum SearchOperator {
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum SearchField {
AccountId,
DocumentId,
Id,
Email(EmailSearchField),
Calendar(CalendarSearchField),
Contact(ContactSearchField),
@@ -84,10 +89,25 @@ pub enum FileSearchField {
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SearchValue {
Text { value: String, language: Language },
Number(i64),
Int(i64),
Uint(u64),
Boolean(bool),
}
pub trait SearchDocumentId: Sized {
fn from_u32(id: u32) -> Self;
fn from_u64(id: u64) -> Self;
fn field(&self) -> SearchField;
}
#[derive(Debug)]
pub struct SearchQuery {
index: SearchIndex,
filters: Vec<SearchFilter>,
comparators: Vec<SearchComparator>,
mask: RoaringBitmap,
}
#[derive(Debug)]
pub enum SearchFilter {
Operator {
@@ -111,8 +131,6 @@ pub enum SearchComparator {
#[derive(Debug)]
pub struct IndexDocument {
pub(crate) account_id: u32,
pub(crate) document_id: u32,
pub(crate) fields: AHashMap<SearchField, SearchValue>,
pub(crate) default_language: Language,
}
@@ -275,18 +293,25 @@ impl IndexDocument {
Self {
fields: Default::default(),
default_language,
account_id: 0,
document_id: 0,
}
}
pub fn with_account_id(mut self, account_id: u32) -> Self {
self.account_id = account_id;
self.fields
.insert(SearchField::AccountId, SearchValue::Uint(account_id as u64));
self
}
pub fn with_document_id(mut self, document_id: u32) -> Self {
self.document_id = document_id;
self.fields.insert(
SearchField::DocumentId,
SearchValue::Uint(document_id as u64),
);
self
}
pub fn with_id(mut self, id: u64) -> Self {
self.fields.insert(SearchField::Id, SearchValue::Uint(id));
self
}
@@ -316,9 +341,14 @@ impl IndexDocument {
.insert(field.into(), SearchValue::Boolean(value));
}
pub fn index_number<N: Into<i64>>(&mut self, field: impl Into<SearchField>, value: N) {
pub fn index_integer<N: Into<i64>>(&mut self, field: impl Into<SearchField>, value: N) {
self.fields
.insert(field.into(), SearchValue::Number(value.into()));
.insert(field.into(), SearchValue::Int(value.into()));
}
pub fn index_unsigned<N: Into<u64>>(&mut self, field: impl Into<SearchField>, value: N) {
self.fields
.insert(field.into(), SearchValue::Uint(value.into()));
}
pub fn has_field(&self, field: &SearchField) -> bool {
@@ -326,6 +356,64 @@ impl IndexDocument {
}
}
impl SearchQuery {
pub fn new(index: SearchIndex) -> Self {
Self {
index,
filters: Vec::new(),
comparators: Vec::new(),
mask: RoaringBitmap::new(),
}
}
pub fn with_filters(mut self, filters: Vec<SearchFilter>) -> Self {
if self.filters.is_empty() {
self.filters = filters;
} else {
self.filters.extend(filters);
}
self
}
pub fn with_comparators(mut self, comparators: Vec<SearchComparator>) -> Self {
if self.comparators.is_empty() {
self.comparators = comparators;
} else {
self.comparators.extend(comparators);
}
self
}
pub fn with_filter(mut self, filter: SearchFilter) -> Self {
self.filters.push(filter);
self
}
pub fn with_comparator(mut self, comparator: SearchComparator) -> Self {
self.comparators.push(comparator);
self
}
pub fn with_mask(mut self, mask: RoaringBitmap) -> Self {
self.mask = mask;
self
}
pub fn with_account_id(mut self, account_id: u32) -> Self {
self.filters.push(SearchFilter::cond(
SearchField::AccountId,
SearchOperator::Equal,
SearchValue::Uint(account_id as u64),
));
self
}
pub fn execute(&self) -> RoaringBitmap {
let todo = "implement search execution logic";
todo!()
}
}
impl From<EmailSearchField> for SearchField {
fn from(field: EmailSearchField) -> Self {
SearchField::Email(field)
@@ -352,31 +440,31 @@ impl From<FileSearchField> for SearchField {
impl From<u64> for SearchValue {
fn from(value: u64) -> Self {
SearchValue::Number(value as i64)
SearchValue::Uint(value)
}
}
impl From<i64> for SearchValue {
fn from(value: i64) -> Self {
SearchValue::Number(value)
SearchValue::Int(value)
}
}
impl From<u32> for SearchValue {
fn from(value: u32) -> Self {
SearchValue::Number(value as i64)
SearchValue::Uint(value as u64)
}
}
impl From<i32> for SearchValue {
fn from(value: i32) -> Self {
SearchValue::Number(value as i64)
SearchValue::Int(value as i64)
}
}
impl From<usize> for SearchValue {
fn from(value: usize) -> Self {
SearchValue::Number(value as i64)
SearchValue::Uint(value as u64)
}
}
@@ -394,3 +482,31 @@ impl From<String> for SearchValue {
}
}
}
impl SearchDocumentId for u32 {
fn from_u32(id: u32) -> Self {
id
}
fn from_u64(id: u64) -> Self {
id as u32
}
fn field(&self) -> SearchField {
SearchField::DocumentId
}
}
impl SearchDocumentId for u64 {
fn from_u32(id: u32) -> Self {
id as u64
}
fn from_u64(id: u64) -> Self {
id
}
fn field(&self) -> SearchField {
SearchField::Id
}
}

2
crates/store/src/search/query.rs
View File

@@ -44,7 +44,7 @@ enum FtsTokenized {
}*/
impl Store {
pub async fn index_query(
pub(crate) async fn index_query(
&self,
account_id: u32,
collection: Collection,

35
crates/store/src/write/key.rs
View File

@@ -13,9 +13,8 @@ use crate::{
SUBSPACE_BLOB_RESERVE, SUBSPACE_COUNTER, SUBSPACE_DIRECTORY, SUBSPACE_IN_MEMORY_COUNTER,
SUBSPACE_IN_MEMORY_VALUE, SUBSPACE_INDEXES, SUBSPACE_LOGS, SUBSPACE_PROPERTY,
SUBSPACE_QUEUE_EVENT, SUBSPACE_QUEUE_MESSAGE, SUBSPACE_QUOTA, SUBSPACE_REPORT_IN,
SUBSPACE_REPORT_OUT, SUBSPACE_SETTINGS, SUBSPACE_TASK_QUEUE, SUBSPACE_TELEMETRY_INDEX,
SUBSPACE_TELEMETRY_METRIC, SUBSPACE_TELEMETRY_SPAN, U16_LEN, U32_LEN, U64_LEN, ValueKey,
WITH_SUBSPACE,
SUBSPACE_REPORT_OUT, SUBSPACE_SETTINGS, SUBSPACE_TASK_QUEUE, SUBSPACE_TELEMETRY_METRIC,
SUBSPACE_TELEMETRY_SPAN, U16_LEN, U32_LEN, U64_LEN, ValueKey, WITH_SUBSPACE,
write::{IndexPropertyClass, SearchIndex},
};
use std::convert::TryInto;
@@ -322,6 +321,11 @@ impl ValueClass {
.write(*due)
}
}
TaskQueueClass::MergeThreads { due } => serializer
.write(*due)
.write(account_id)
.write(9u8)
.write(document_id),
},
ValueClass::Blob(op) => match op {
BlobOp::Reserve { hash, until } => serializer
@@ -418,9 +422,6 @@ impl ValueClass {
},
ValueClass::Telemetry(telemetry) => match telemetry {
TelemetryClass::Span { span_id } => serializer.write(*span_id),
TelemetryClass::Index { span_id, value } => {
serializer.write(value.as_slice()).write(*span_id)
}
TelemetryClass::Metric {
timestamp,
metric_id,
@@ -512,7 +513,9 @@ impl ValueClass {
ValueClass::TaskQueue(e) => match e {
TaskQueueClass::UpdateIndex { .. } => (U64_LEN * 2) + 2,
TaskQueueClass::BayesTrain { .. } => (U64_LEN * 2) + 1,
TaskQueueClass::SendAlarm { .. } => U64_LEN + (U32_LEN * 3) + 1,
TaskQueueClass::SendAlarm { .. } | TaskQueueClass::MergeThreads { .. } => {
U64_LEN + (U32_LEN * 3) + 1
}
TaskQueueClass::SendImip { is_payload, .. } => {
if *is_payload {
(U64_LEN * 2) + (U32_LEN * 2) + 1
@@ -535,7 +538,6 @@ impl ValueClass {
ValueClass::Report(_) => U64_LEN * 2 + 1,
ValueClass::Telemetry(telemetry) => match telemetry {
TelemetryClass::Span { .. } => U64_LEN + 1,
TelemetryClass::Index { value, .. } => U64_LEN + value.len() + 1,
TelemetryClass::Metric { .. } => U64_LEN * 2 + 1,
},
ValueClass::DocumentId => U32_LEN + 1,
@@ -584,7 +586,6 @@ impl ValueClass {
ValueClass::Report(_) => SUBSPACE_REPORT_IN,
ValueClass::Telemetry(telemetry) => match telemetry {
TelemetryClass::Span { .. } => SUBSPACE_TELEMETRY_SPAN,
TelemetryClass::Index { .. } => SUBSPACE_TELEMETRY_INDEX,
TelemetryClass::Metric { .. } => SUBSPACE_TELEMETRY_METRIC,
},
ValueClass::DocumentId | ValueClass::ChangeId => SUBSPACE_COUNTER,
@@ -666,7 +667,8 @@ impl SearchIndex {
SearchIndex::Calendar => 1,
SearchIndex::Contacts => 2,
SearchIndex::File => 3,
SearchIndex::DeliveryHistory => 4,
SearchIndex::TracingSpan => 4,
SearchIndex::InMemory => unreachable!(),
}
}
@@ -676,7 +678,18 @@ impl SearchIndex {
1 => Some(SearchIndex::Calendar),
2 => Some(SearchIndex::Contacts),
3 => Some(SearchIndex::File),
4 => Some(SearchIndex::DeliveryHistory),
4 => Some(SearchIndex::TracingSpan),
_ => None,
}
}
pub fn try_from_str(value: &str) -> Option<Self> {
match value {
"email" => Some(SearchIndex::Email),
"calendar" => Some(SearchIndex::Calendar),
"contacts" => Some(SearchIndex::Contacts),
"file" => Some(SearchIndex::File),
"tracing" => Some(SearchIndex::TracingSpan),
_ => None,
}
}

10
crates/store/src/write/mod.rs
View File

@@ -209,6 +209,9 @@ pub enum TaskQueueClass {
due: u64,
is_payload: bool,
},
MergeThreads {
due: u64,
},
}
#[derive(Debug, PartialEq, Clone, Copy, Eq, Hash)]
@@ -217,7 +220,8 @@ pub enum SearchIndex {
Calendar,
Contacts,
File,
DeliveryHistory,
TracingSpan,
InMemory,
}
#[derive(Debug, PartialEq, Clone, Eq, Hash)]
@@ -272,10 +276,6 @@ pub enum TelemetryClass {
metric_id: u64,
node_id: u64,
},
Index {
span_id: u64,
value: Vec<u8>,
},
}
#[derive(Debug, PartialEq, Clone, Eq, Hash)]