filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Database schema optimization - part 3

Browse Source
This commit is contained in:
mdecimus
2025-11-01 19:19:36 +01:00
parent d5c5f12adf
commit 8e1316895b
41 changed files with 1979 additions and 1276 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
crates/common/src/config/jmap/settings.rs
View File

@@ -5,9 +5,11 @@
*/
use crate::config::groupware::GroupwareConfig;
use ahash::{AHashMap, AHashSet};
use jmap_proto::request::capability::BaseCapabilities;
use nlp::language::Language;
use std::{str::FromStr, time::Duration};
use store::{search::SearchField, write::SearchIndex};
use types::{collection::Collection, special_use::SpecialUse};
use utils::{
config::{Config, Rate, cron::SimpleCron, utils::ParseValue},
@@ -79,6 +81,10 @@ pub struct JmapConfig {
pub encrypt: bool,
pub encrypt_append: bool,
pub index_batch_size: usize,
pub index_all_headers: bool,
pub index_fields: AHashMap<SearchIndex, AHashSet<SearchField>>,
pub capabilities: BaseCapabilities,
pub account_purge_frequency: SimpleCron,
}
@@ -351,10 +357,45 @@ impl JmapConfig {
calendar_parse_max_items: config
.property("jmap.calendar.parse.max-items")
.unwrap_or(10),
index_batch_size: config.property("jmap.index.batch-size").unwrap_or(100),
index_all_headers: config
.property_or_default("jmap.index.email.all-headers", "false")
.unwrap_or(false),
index_fields: AHashMap::new(),
default_folders,
shared_folder,
};
// Parse index fields
for index in [
SearchIndex::Email,
SearchIndex::Contacts,
SearchIndex::Calendar,
] {
let mut fields = AHashSet::new();
let todo = "implement";
/*for field_str in config.values(&format!(
"jmap.index.{}.fields",
index.as_config_case()
)) {
match SearchField::try_from(field_str.1.as_str()) {
Ok(field) => {
fields.insert(field);
}
Err(_) => {
config.new_parse_error(
&format!(
"jmap.index.{}.fields",
index.as_config_case()
),
format!("Invalid search field: {}", field_str.1),
);
}
}
}*/
jmap.index_fields.insert(index, fields);
}
for collection in Bitmap::<Collection>::all() {
let key = format!("object-quota.{}", collection.as_config_case());
jmap.max_objects[collection as usize] =

5
crates/common/src/config/network.rs
View File

@@ -44,6 +44,7 @@ pub struct ClusterRoles {
pub fts_indexing: ClusterRole,
pub bayes_training: ClusterRole,
pub imip_processing: ClusterRole,
pub merge_threads: ClusterRole,
pub calendar_alerts: ClusterRole,
pub renew_acme: ClusterRole,
pub calculate_metrics: ClusterRole,
@@ -256,6 +257,10 @@ impl Network {
&mut network.roles.calendar_alerts,
"cluster.roles.calendar-alerts",
),
(
&mut network.roles.merge_threads,
"cluster.roles.merge-threads",
),
] {
let shards = config
.properties::<NodeList>(key)

178
crates/common/src/telemetry/tracers/index.rs Normal file
View File

@@ -0,0 +1,178 @@
fn build_index() {
let mut queue_ids = AHashSet::new();
let mut values = AHashSet::new();
for event in events.iter().chain([span, &event]) {
for (key, value) in &event.keys {
match (key, value) {
(Key::QueueId, Value::UInt(queue_id)) => {
queue_ids.insert(*queue_id);
}
(Key::From | Key::To | Key::Domain | Key::Hostname, Value::String(address)) => {
values.insert(address.clone());
}
(Key::To, Value::Array(value)) => {
for value in value {
if let Value::String(address) = value {
values.insert(address.clone());
}
}
}
(Key::RemoteIp, Value::Ipv4(ip)) => {
values.insert(ip.to_string().into());
}
(Key::RemoteIp, Value::Ipv6(ip)) => {
values.insert(ip.to_string().into());
}
_ => {}
}
}
}
// Build index
batch.set(
ValueClass::Telemetry(TelemetryClass::Index {
span_id,
value: (span.inner.typ.code() as u16).to_be_bytes().to_vec(),
}),
vec![],
);
for queue_id in queue_ids {
batch.set(
ValueClass::Telemetry(TelemetryClass::Index {
span_id,
value: queue_id.to_be_bytes().to_vec(),
}),
vec![],
);
}
for value in values {
batch.set(
ValueClass::Telemetry(TelemetryClass::Index {
span_id,
value: value.as_bytes().to_vec(),
}),
vec![],
);
}
}
/*
enum SpanCollector {
Vec(Vec<u64>),
HashSet(AHashSet<u64>),
Empty,
}
impl SpanCollector {
fn new(num_params: usize) -> Self {
if num_params == 1 {
Self::Vec(Vec::new())
} else {
Self::HashSet(AHashSet::new())
}
}
fn insert(&mut self, span_id: u64) {
match self {
Self::Vec(vec) => vec.push(span_id),
Self::HashSet(set) => {
set.insert(span_id);
}
_ => unreachable!(),
}
}
fn into_vec(self) -> Vec<u64> {
match self {
Self::Vec(mut vec) => {
vec.sort_unstable_by(|a, b| b.cmp(a));
vec
}
Self::HashSet(set) => {
let mut vec: Vec<u64> = set.into_iter().collect();
vec.sort_unstable_by(|a, b| b.cmp(a));
vec
}
Self::Empty => Vec::new(),
}
}
fn intersect(&mut self, other_span: Self) -> bool {
match (self, other_span) {
(Self::HashSet(set), Self::HashSet(other_set)) => {
set.retain(|span_id| other_set.contains(span_id));
set.is_empty()
}
_ => unreachable!(),
}
}
}
let mut spans = SpanCollector::Empty;
let num_params = params.len();
let todo = "use FTS";
for (param_num, param) in params.iter().enumerate() {
let (value, exact_len) = match param {
TracingQuery::EventType(event) => (
(event.code() as u16).to_be_bytes().to_vec(),
std::mem::size_of::<u16>() + U64_LEN,
),
TracingQuery::QueueId(id) => (
id.to_be_bytes().to_vec(),
std::mem::size_of::<u64>() + U64_LEN,
),
TracingQuery::Keywords(value) => {
if let Some(value) = value.strip_prefix('"').and_then(|v| v.strip_suffix('"')) {
(value.as_bytes().to_vec(), value.len() + U64_LEN)
} else {
(value.as_bytes().to_vec(), 0)
}
}
};
let mut param_spans = SpanCollector::new(num_params);
self.iterate(
IterateParams::new(
ValueKey::from(ValueClass::Telemetry(TelemetryClass::Index {
span_id: 0,
value: value.clone(),
})),
ValueKey::from(ValueClass::Telemetry(TelemetryClass::Index {
span_id: u64::MAX,
value,
})),
)
.no_values(),
|key, _| {
if exact_len == 0 || key.len() == exact_len {
let span_id = key
.deserialize_be_u64(key.len() - U64_LEN)
.caused_by(trc::location!())?;
if (from_span_id == 0 || span_id >= from_span_id)
&& (to_span_id == 0 || span_id <= to_span_id)
{
param_spans.insert(span_id);
}
}
Ok(true)
},
)
.await
.caused_by(trc::location!())?;
if param_num == 0 {
spans = param_spans;
} else if spans.intersect(param_spans) {
return Ok(Vec::new());
}
}
Ok(spans.into_vec())
*/

256
crates/common/src/telemetry/tracers/store.rs
View File

@@ -8,12 +8,12 @@
*
*/
use std::{future::Future, time::Duration};
use crate::config::telemetry::StoreTracer;
use ahash::{AHashMap, AHashSet};
use std::{future::Future, time::Duration};
use store::{
Deserialize, IterateParams, Store, U64_LEN, ValueKey,
write::{BatchBuilder, TelemetryClass, ValueClass, key::DeserializeBigEndian},
Deserialize, Store, ValueKey,
write::{BatchBuilder, SearchIndex, TaskQueueClass, TelemetryClass, ValueClass, now},
};
use trc::{
AddContext, AuthEvent, Event, EventDetails, EventType, Key, MessageIngestEvent,
@@ -23,8 +23,6 @@ use trc::{
};
use utils::snowflake::SnowflakeIdGenerator;
use crate::config::telemetry::StoreTracer;
const MAX_EVENTS: usize = 2048;
pub(crate) fn spawn_store_tracer(builder: SubscriberBuilder, settings: StoreTracer) {
@@ -35,6 +33,8 @@ pub(crate) fn spawn_store_tracer(builder: SubscriberBuilder, settings: StoreTrac
let mut batch = BatchBuilder::new();
while let Some(events) = rx.recv().await {
let now = now();
for event in events {
if let Some(span) = &event.inner.span {
let span_id = span.span_id().unwrap();
@@ -43,44 +43,16 @@ pub(crate) fn spawn_store_tracer(builder: SubscriberBuilder, settings: StoreTrac
if events.len() < MAX_EVENTS {
events.push(event);
}
} else if let Some(events) = active_spans.remove(&span_id) {
let mut queue_ids = AHashSet::new();
let mut values = AHashSet::new();
for event in events.iter().chain([span, &event]) {
for (key, value) in &event.keys {
match (key, value) {
(Key::QueueId, Value::UInt(queue_id)) => {
queue_ids.insert(*queue_id);
}
(
Key::From | Key::To | Key::Domain | Key::Hostname,
Value::String(address),
) => {
values.insert(address.clone());
}
(Key::To, Value::Array(value)) => {
for value in value {
if let Value::String(address) = value {
values.insert(address.clone());
}
}
}
(Key::RemoteIp, Value::Ipv4(ip)) => {
values.insert(ip.to_string().into());
}
(Key::RemoteIp, Value::Ipv6(ip)) => {
values.insert(ip.to_string().into());
}
_ => {}
}
}
}
if !queue_ids.is_empty() {
// Serialize events
batch.set(
} else if let Some(events) = active_spans.remove(&span_id)
&& events
.iter()
.chain([span, &event])
.flat_map(|event| event.keys.iter())
.any(|(k, v)| matches!((k, v), (Key::QueueId, Value::UInt(_))))
{
// Serialize events
batch
.set(
ValueClass::Telemetry(TelemetryClass::Span { span_id }),
serialize_events(
[span.as_ref()]
@@ -89,35 +61,17 @@ pub(crate) fn spawn_store_tracer(builder: SubscriberBuilder, settings: StoreTrac
.chain([event.as_ref()].into_iter()),
events.len() + 2,
),
);
// Build index
batch.set(
ValueClass::Telemetry(TelemetryClass::Index {
span_id,
value: (span.inner.typ.code() as u16).to_be_bytes().to_vec(),
)
.with_account_id((span_id >> 32) as u32) // TODO: This is hacky, improve
.with_document(span_id as u32)
.set(
ValueClass::TaskQueue(TaskQueueClass::UpdateIndex {
due: now,
index: SearchIndex::TracingSpan,
is_insert: true,
}),
vec![],
);
for queue_id in queue_ids {
batch.set(
ValueClass::Telemetry(TelemetryClass::Index {
span_id,
value: queue_id.to_be_bytes().to_vec(),
}),
vec![],
);
}
for value in values {
batch.set(
ValueClass::Telemetry(TelemetryClass::Index {
span_id,
value: value.as_bytes().to_vec(),
}),
vec![],
);
}
}
}
}
}
@@ -181,69 +135,7 @@ impl TracingStore for Store {
from_span_id: u64,
to_span_id: u64,
) -> trc::Result<Vec<u64>> {
let mut spans = SpanCollector::Empty;
let num_params = params.len();
let todo = "use FTS";
for (param_num, param) in params.iter().enumerate() {
let (value, exact_len) = match param {
TracingQuery::EventType(event) => (
(event.code() as u16).to_be_bytes().to_vec(),
std::mem::size_of::<u16>() + U64_LEN,
),
TracingQuery::QueueId(id) => (
id.to_be_bytes().to_vec(),
std::mem::size_of::<u64>() + U64_LEN,
),
TracingQuery::Keywords(value) => {
if let Some(value) = value.strip_prefix('"').and_then(|v| v.strip_suffix('"')) {
(value.as_bytes().to_vec(), value.len() + U64_LEN)
} else {
(value.as_bytes().to_vec(), 0)
}
}
};
let mut param_spans = SpanCollector::new(num_params);
self.iterate(
IterateParams::new(
ValueKey::from(ValueClass::Telemetry(TelemetryClass::Index {
span_id: 0,
value: value.clone(),
})),
ValueKey::from(ValueClass::Telemetry(TelemetryClass::Index {
span_id: u64::MAX,
value,
})),
)
.no_values(),
|key, _| {
if exact_len == 0 || key.len() == exact_len {
let span_id = key
.deserialize_be_u64(key.len() - U64_LEN)
.caused_by(trc::location!())?;
if (from_span_id == 0 || span_id >= from_span_id)
&& (to_span_id == 0 || span_id <= to_span_id)
{
param_spans.insert(span_id);
}
}
Ok(true)
},
)
.await
.caused_by(trc::location!())?;
if param_num == 0 {
spans = param_spans;
} else if spans.intersect(param_spans) {
return Ok(Vec::new());
}
}
Ok(spans.into_vec())
todo!()
}
async fn purge_spans(&self, period: Duration) -> trc::Result<()> {
@@ -262,108 +154,12 @@ impl TracingStore for Store {
.await
.caused_by(trc::location!())?;
let mut delete_keys: Vec<ValueClass> = Vec::new();
self.iterate(
IterateParams::new(
ValueKey::from(ValueClass::Telemetry(TelemetryClass::Index {
span_id: 0,
value: vec![],
})),
ValueKey::from(ValueClass::Telemetry(TelemetryClass::Index {
span_id: u64::MAX,
value: vec![u8::MAX; 16],
})),
)
.no_values(),
|key, _| {
let span_id = key
.deserialize_be_u64(key.len() - U64_LEN)
.caused_by(trc::location!())?;
if span_id < until_span_id {
delete_keys.push(ValueClass::Telemetry(TelemetryClass::Index {
span_id,
value: key[0..key.len() - U64_LEN].to_vec(),
}));
}
Ok(true)
},
)
.await
.caused_by(trc::location!())?;
if !delete_keys.is_empty() {
// Commit index
let mut batch = BatchBuilder::new();
for key in delete_keys {
if batch.is_large_batch() {
self.write(batch.build_all()).await?;
batch = BatchBuilder::new();
}
batch.clear(key);
}
if !batch.is_empty() {
self.write(batch.build_all()).await?;
}
}
let todo = "delete from index";
Ok(())
}
}
enum SpanCollector {
Vec(Vec<u64>),
HashSet(AHashSet<u64>),
Empty,
}
impl SpanCollector {
fn new(num_params: usize) -> Self {
if num_params == 1 {
Self::Vec(Vec::new())
} else {
Self::HashSet(AHashSet::new())
}
}
fn insert(&mut self, span_id: u64) {
match self {
Self::Vec(vec) => vec.push(span_id),
Self::HashSet(set) => {
set.insert(span_id);
}
_ => unreachable!(),
}
}
fn into_vec(self) -> Vec<u64> {
match self {
Self::Vec(mut vec) => {
vec.sort_unstable_by(|a, b| b.cmp(a));
vec
}
Self::HashSet(set) => {
let mut vec: Vec<u64> = set.into_iter().collect();
vec.sort_unstable_by(|a, b| b.cmp(a));
vec
}
Self::Empty => Vec::new(),
}
}
fn intersect(&mut self, other_span: Self) -> bool {
match (self, other_span) {
(Self::HashSet(set), Self::HashSet(other_set)) => {
set.retain(|span_id| other_set.contains(span_id));
set.is_empty()
}
_ => unreachable!(),
}
}
}
impl StoreTracer {
pub fn default_events() -> impl IntoIterator<Item = EventType> {
EventType::variants().into_iter().filter(|event| {