filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Database schema optimization - part 2

Browse Source
This commit is contained in:
mdecimus
2025-10-31 18:25:01 +01:00
parent 3e1d2d54e4
commit d5c5f12adf
43 changed files with 2266 additions and 1853 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
crates/groupware/src/cache/calcard.rs vendored
View File

@@ -9,7 +9,7 @@ use crate::{
DavResourceName, RFC_3986,
calendar::{
ArchivedCalendar, ArchivedCalendarEvent, Calendar, CalendarEvent, SCHEDULE_INBOX_ID,
SCHEDULE_OUTBOX_ID,
SCHEDULE_OUTBOX_ID, storage::ItipAutoExpunge,
},
contact::{AddressBook, ArchivedAddressBook, ArchivedContactCard, ContactCard},
};
@@ -192,11 +192,7 @@ pub(super) async fn build_scheduling_resources(
.unwrap_or_else(|| format!("_{account_id}"));
let item_ids = server
.document_ids(
account_id,
Collection::CalendarEventNotification,
CalendarNotificationField::CreatedToId,
)
.itip_ids(account_id)
.await
.caused_by(trc::location!())?;

69
crates/groupware/src/calendar/index.rs
View File

@@ -17,8 +17,10 @@ use calcard::icalendar::{
ICalendarParameterValue, ICalendarProperty, ICalendarValue,
};
use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject};
use nlp::language::Language;
use store::{
write::{IndexPropertyClass, ValueClass},
search::{CalendarSearchField, IndexDocument},
write::{IndexPropertyClass, SearchIndex, ValueClass},
xxhash_rust::xxh3,
};
use types::{acl::AclGrant, collection::SyncCollection, field::CalendarNotificationField};
@@ -76,7 +78,11 @@ impl IndexableObject for CalendarEvent {
fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
[
IndexValue::SearchIndex {
hashes: self.hashes().collect(),
index: SearchIndex::Calendar,
hash: self
.hashes()
.chain([self.data.event_range_start() as u64])
.fold(0, |acc, hash| acc ^ hash),
},
IndexValue::Quota {
used: self.dead_properties.size() as u32
@@ -98,7 +104,11 @@ impl IndexableObject for &ArchivedCalendarEvent {
fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
[
IndexValue::SearchIndex {
hashes: self.hashes().collect(),
index: SearchIndex::Calendar,
hash: self
.hashes()
.chain([self.data.event_range_start() as u64])
.fold(0, |acc, hash| acc ^ hash),
},
IndexValue::Quota {
used: self.dead_properties.size() as u32
@@ -263,6 +273,7 @@ impl CalendarEvent {
| ICalendarProperty::Comment
| ICalendarProperty::Attendee
| ICalendarProperty::Organizer
| ICalendarProperty::Uid
)
})
})
@@ -302,6 +313,7 @@ impl ArchivedCalendarEvent {
| ArchivedICalendarProperty::Comment
| ArchivedICalendarProperty::Attendee
| ArchivedICalendarProperty::Organizer
| ArchivedICalendarProperty::Uid
)
})
})
@@ -322,3 +334,54 @@ impl ArchivedCalendarEvent {
.map(|v| xxh3::xxh3_64(v.as_bytes()))
}
}
impl ArchivedCalendarEvent {
pub fn index_document(&self) -> IndexDocument {
let mut document = IndexDocument::with_default_language(Language::Unknown);
document.index_number(CalendarSearchField::Start, self.data.event_range_start());
for component in self
.data
.event
.components
.iter()
.filter(|e| e.component_type.is_scheduling_object())
{
for entry in component.entries.iter() {
let field = match entry.name {
ArchivedICalendarProperty::Summary => CalendarSearchField::Title,
ArchivedICalendarProperty::Description => CalendarSearchField::Description,
ArchivedICalendarProperty::Location => CalendarSearchField::Location,
ArchivedICalendarProperty::Organizer => CalendarSearchField::Owner,
ArchivedICalendarProperty::Attendee => CalendarSearchField::Attendee,
ArchivedICalendarProperty::Uid => CalendarSearchField::Uid,
_ => continue,
};
for value in entry
.values
.iter()
.filter_map(|v| match v {
ArchivedICalendarValue::Text(v) => Some(v.as_str()),
ArchivedICalendarValue::Uri(uri) => uri.as_str(),
_ => None,
})
.chain(entry.params.iter().filter_map(|p| match &p.value {
ArchivedICalendarParameterValue::Text(v) => Some(v.as_str()),
ArchivedICalendarParameterValue::Uri(uri) => uri.as_str(),
_ => None,
}))
{
document.index_text(
field,
value.strip_prefix("mailto:").unwrap_or(value),
Language::Unknown,
);
}
}
}
document
}
}

70
crates/groupware/src/calendar/storage.rs
View File

@@ -18,10 +18,10 @@ use crate::{
use calcard::common::timezone::Tz;
use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder};
use store::{
IndexKey, IterateParams, SerializeInfallible, U16_LEN, U32_LEN, U64_LEN,
IterateParams, U16_LEN, U32_LEN, U64_LEN, ValueKey,
roaring::RoaringBitmap,
write::{
Archive, BatchBuilder, TaskQueueClass, ValueClass,
Archive, BatchBuilder, IndexPropertyClass, TaskQueueClass, ValueClass,
key::{DeserializeBigEndian, KeySerializer},
now,
},
@@ -33,6 +33,8 @@ use types::{
};
pub trait ItipAutoExpunge: Sync + Send {
fn itip_ids(&self, account_id: u32) -> impl Future<Output = trc::Result<RoaringBitmap>> + Send;
fn itip_auto_expunge(
&self,
account_id: u32,
@@ -41,33 +43,71 @@ pub trait ItipAutoExpunge: Sync + Send {
}
impl ItipAutoExpunge for Server {
async fn itip_auto_expunge(&self, account_id: u32, hold_period: u64) -> trc::Result<()> {
let mut destroy_ids = RoaringBitmap::new();
async fn itip_ids(&self, account_id: u32) -> trc::Result<RoaringBitmap> {
let mut document_ids = RoaringBitmap::new();
self.store()
.iterate(
IterateParams::new(
IndexKey {
ValueKey {
account_id,
collection: Collection::CalendarEventNotification.into(),
document_id: 0,
field: CalendarNotificationField::CreatedToId.into(),
key: 0u64.serialize(),
class: ValueClass::IndexProperty(IndexPropertyClass::Integer {
property: CalendarNotificationField::CreatedToId.into(),
value: 0,
}),
},
IndexKey {
ValueKey {
account_id,
collection: Collection::CalendarEventNotification.into(),
document_id: u32::MAX,
field: CalendarNotificationField::CreatedToId.into(),
key: now().saturating_sub(hold_period).serialize(),
document_id: 0,
class: ValueClass::IndexProperty(IndexPropertyClass::Integer {
property: CalendarNotificationField::CreatedToId.into(),
value: u64::MAX,
}),
},
)
.no_values()
.ascending(),
|key, _| {
destroy_ids.insert(
key.deserialize_be_u32(key.len() - U32_LEN)
.caused_by(trc::location!())?,
);
document_ids.insert(key.deserialize_be_u32(key.len() - U32_LEN)?);
Ok(true)
},
)
.await
.caused_by(trc::location!())
.map(|_| document_ids)
}
async fn itip_auto_expunge(&self, account_id: u32, hold_period: u64) -> trc::Result<()> {
let mut destroy_ids = RoaringBitmap::new();
self.store()
.iterate(
IterateParams::new(
ValueKey {
account_id,
collection: Collection::CalendarEventNotification.into(),
document_id: 0,
class: ValueClass::IndexProperty(IndexPropertyClass::Integer {
property: CalendarNotificationField::CreatedToId.into(),
value: 0,
}),
},
ValueKey {
account_id,
collection: Collection::CalendarEventNotification.into(),
document_id: 0,
class: ValueClass::IndexProperty(IndexPropertyClass::Integer {
property: CalendarNotificationField::CreatedToId.into(),
value: now().saturating_sub(hold_period),
}),
},
)
.no_values()
.ascending(),
|key, _| {
destroy_ids.insert(key.deserialize_be_u32(key.len() - U32_LEN)?);
Ok(true)
},

105
crates/groupware/src/contact/index.rs
View File

@@ -5,9 +5,20 @@
*/
use super::{AddressBook, ArchivedAddressBook, ArchivedContactCard, ContactCard};
use calcard::vcard::{ArchivedVCardProperty, VCardProperty};
use calcard::{
common::IanaString,
vcard::{
ArchivedVCardParameterValue, ArchivedVCardProperty, ArchivedVCardValue,
VCardParameterValue, VCardProperty,
},
};
use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject};
use store::xxhash_rust::xxh3;
use nlp::language::Language;
use store::{
search::{ContactSearchField, IndexDocument},
write::SearchIndex,
xxhash_rust::xxh3,
};
use types::{acl::AclGrant, collection::SyncCollection, field::ContactField};
use utils::sanitize_email;
@@ -86,7 +97,8 @@ impl IndexableObject for ContactCard {
value: self.emails().next().into(),
},
IndexValue::SearchIndex {
hashes: self.hashes().collect(),
index: SearchIndex::Contacts,
hash: self.hashes().fold(0, |acc, hash| acc ^ hash),
},
IndexValue::Quota {
used: self.dead_properties.size() as u32
@@ -115,7 +127,8 @@ impl IndexableObject for &ArchivedContactCard {
value: self.emails().next().into(),
},
IndexValue::SearchIndex {
hashes: self.hashes().collect(),
index: SearchIndex::Contacts,
hash: self.hashes().fold(0, |acc, hash| acc ^ hash),
},
IndexValue::Quota {
used: self.dead_properties.size() as u32
@@ -154,9 +167,23 @@ impl ContactCard {
| VCardProperty::Note
| VCardProperty::Nickname
| VCardProperty::Email
| VCardProperty::Kind
| VCardProperty::Uid
| VCardProperty::Member
| VCardProperty::Impp
| VCardProperty::Socialprofile
| VCardProperty::Tel
)
})
.flat_map(|e| e.values.iter().filter_map(|v| v.as_text()))
.flat_map(|e| {
e.values
.iter()
.filter_map(|v| v.as_text())
.chain(e.params.iter().filter_map(|p| match &p.value {
VCardParameterValue::Text(v) => Some(v.as_str()),
_ => None,
}))
})
.map(|v| xxh3::xxh3_64(v.as_bytes()))
}
@@ -185,9 +212,23 @@ impl ArchivedContactCard {
| ArchivedVCardProperty::Note
| ArchivedVCardProperty::Nickname
| ArchivedVCardProperty::Email
| ArchivedVCardProperty::Kind
| ArchivedVCardProperty::Uid
| ArchivedVCardProperty::Member
| ArchivedVCardProperty::Impp
| ArchivedVCardProperty::Socialprofile
| ArchivedVCardProperty::Tel
)
})
.flat_map(|e| e.values.iter().filter_map(|v| v.as_text()))
.flat_map(|e| {
e.values
.iter()
.filter_map(|v| v.as_text())
.chain(e.params.iter().filter_map(|p| match &p.value {
ArchivedVCardParameterValue::Text(v) => Some(v.as_str()),
_ => None,
}))
})
.map(|v| xxh3::xxh3_64(v.as_bytes()))
}
@@ -199,3 +240,55 @@ impl ArchivedContactCard {
})
}
}
impl ArchivedContactCard {
pub fn index_document(&self) -> IndexDocument {
let mut document = IndexDocument::with_default_language(Language::Unknown);
document.index_number(ContactSearchField::Created, self.created.to_native());
for entry in self.card.entries.iter() {
let field = match entry.name {
ArchivedVCardProperty::N => ContactSearchField::Name,
ArchivedVCardProperty::Nickname => ContactSearchField::Nickname,
ArchivedVCardProperty::Org => ContactSearchField::Organization,
ArchivedVCardProperty::Email => ContactSearchField::Email,
ArchivedVCardProperty::Tel => ContactSearchField::Phone,
ArchivedVCardProperty::Impp | ArchivedVCardProperty::Socialprofile => {
ContactSearchField::OnlineService
}
ArchivedVCardProperty::Adr => ContactSearchField::Address,
ArchivedVCardProperty::Note => ContactSearchField::Note,
ArchivedVCardProperty::Kind => ContactSearchField::Kind,
ArchivedVCardProperty::Uid => ContactSearchField::Uid,
ArchivedVCardProperty::Member => ContactSearchField::Member,
_ => continue,
};
for value in entry.values.iter() {
match value {
ArchivedVCardValue::Text(v) => {
document.index_text(field, v, Language::Unknown);
}
ArchivedVCardValue::Kind(v) => {
document.index_text(field, v.as_str(), Language::Unknown);
}
ArchivedVCardValue::Component(v) => {
for item in v.iter() {
document.index_text(field, item, Language::Unknown);
}
}
_ => (),
}
}
for param in entry.params.iter() {
if let ArchivedVCardParameterValue::Text(value) = &param.value {
document.index_text(field, value, Language::Unknown);
}
}
}
document
}
}