filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Groupware caching improvements

Browse Source
This commit is contained in:
mdecimus
2025-05-08 17:30:57 +02:00
parent e69b5ec2b7
commit fe7d646966
54 changed files with 2138 additions and 1963 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
crates/groupware/Cargo.toml
View File

@@ -14,6 +14,7 @@ directory = { path = "../directory" }
dav-proto = { path = "../dav-proto" }
calcard = { path = "/Users/me/code/calcard", features = ["rkyv"] }
hashify = "0.2"
tokio = { version = "1.23", features = ["net", "macros"] }
rkyv = { version = "0.8.10", features = ["little_endian"] }
percent-encoding = "2.3.1"
compact_str = "0.9.0"

300
crates/groupware/src/cache/calcard.rs vendored Normal file
View File

@@ -0,0 +1,300 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use super::GroupwareCache;
use crate::{
DavResourceName,
calendar::{ArchivedCalendar, ArchivedCalendarEvent, Calendar, CalendarEvent},
contact::{AddressBook, ArchivedAddressBook, ArchivedContactCard, ContactCard},
};
use calcard::common::timezone::Tz;
use common::{
DavName, DavPath, DavResource, DavResourceMetadata, DavResources, Server, auth::AccessToken,
};
use directory::backend::internal::manage::ManageDirectory;
use jmap_proto::types::{
collection::{Collection, SyncCollection},
value::AclGrant,
};
use percent_encoding::NON_ALPHANUMERIC;
use std::sync::Arc;
use store::ahash::{AHashMap, AHashSet};
use tokio::sync::Semaphore;
use trc::AddContext;
use utils::map::bitmap::Bitmap;
pub(super) async fn build_calcard_resources(
server: &Server,
access_token: &AccessToken,
account_id: u32,
sync_collection: SyncCollection,
container_collection: Collection,
item_collection: Collection,
update_lock: Arc<Semaphore>,
) -> trc::Result<DavResources> {
let mut last_change_id = server
.core
.storage
.data
.get_last_change_id(account_id, sync_collection)
.await
.caused_by(trc::location!())?
.unwrap_or_default();
// Create default folders
let is_calendar = matches!(sync_collection, SyncCollection::Calendar);
let mut container_ids = server
.get_document_ids(account_id, container_collection)
.await
.caused_by(trc::location!())?
.unwrap_or_default();
if container_ids.is_empty() {
if is_calendar {
server
.create_default_calendar(access_token, account_id)
.await?
} else {
server
.create_default_addressbook(access_token, account_id)
.await?
}
last_change_id = server
.core
.storage
.data
.get_last_change_id(account_id, sync_collection)
.await
.caused_by(trc::location!())?
.unwrap_or_default();
container_ids = server
.get_document_ids(account_id, container_collection)
.await
.caused_by(trc::location!())?
.unwrap_or_default();
}
let item_ids = server
.get_document_ids(account_id, item_collection)
.await
.caused_by(trc::location!())?
.unwrap_or_default();
let name = server
.store()
.get_principal_name(account_id)
.await
.caused_by(trc::location!())?
.unwrap_or_else(|| format!("_{account_id}"));
let mut cache = DavResources {
base_path: format!(
"{}/{}/",
if is_calendar {
DavResourceName::Cal
} else {
DavResourceName::Card
}
.base_path(),
percent_encoding::utf8_percent_encode(&name, NON_ALPHANUMERIC),
),
paths: AHashSet::with_capacity((container_ids.len() + item_ids.len()) as usize),
resources: Vec::with_capacity((container_ids.len() + item_ids.len()) as usize),
item_change_id: last_change_id,
container_change_id: last_change_id,
highest_change_id: last_change_id,
size: std::mem::size_of::<DavResources>() as u64,
update_lock,
};
for document_id in container_ids {
if let Some(archive) = server
.get_archive(account_id, container_collection, document_id)
.await
.caused_by(trc::location!())?
{
let resource = if is_calendar {
resource_from_calendar(archive.unarchive::<Calendar>()?, document_id)
} else {
resource_from_addressbook(archive.unarchive::<AddressBook>()?, document_id)
};
let path = DavPath {
path: resource.container_name().unwrap().to_string(),
parent_id: None,
hierarchy_seq: 1,
resource_idx: cache.resources.len(),
};
cache.size += (std::mem::size_of::<DavPath>()
+ std::mem::size_of::<DavResource>()
+ (path.path.len()) * 2) as u64;
cache.paths.insert(path);
cache.resources.push(resource);
}
}
let parent_range = cache.resources.len();
for document_id in item_ids {
if let Some(archive) = server
.get_archive(account_id, item_collection, document_id)
.await
.caused_by(trc::location!())?
{
let resource = if is_calendar {
resource_from_event(archive.unarchive::<CalendarEvent>()?, document_id)
} else {
resource_from_card(archive.unarchive::<ContactCard>()?, document_id)
};
let resource_idx = cache.resources.len();
for name in resource.child_names().unwrap_or_default().iter() {
if let Some(parent) = cache.resources.get(..parent_range).and_then(|resources| {
resources.iter().find(|r| r.document_id == name.parent_id)
}) {
let path = DavPath {
path: format!("{}/{}", parent.container_name().unwrap(), name.name),
parent_id: Some(name.parent_id),
hierarchy_seq: 0,
resource_idx,
};
cache.size +=
(std::mem::size_of::<DavPath>() + name.name.len() + path.path.len()) as u64;
cache.paths.insert(path);
}
}
cache.size += std::mem::size_of::<DavResource>() as u64;
cache.resources.push(resource);
}
}
Ok(cache)
}
pub(super) fn build_simple_hierarchy(cache: &mut DavResources) {
cache.paths = AHashSet::with_capacity(cache.resources.len());
let name_idx = cache
.resources
.iter()
.filter_map(|resource| {
resource
.container_name()
.map(|name| (resource.document_id, name))
})
.collect::<AHashMap<_, _>>();
for (resource_idx, resource) in cache.resources.iter().enumerate() {
match &resource.data {
DavResourceMetadata::Calendar { name, .. }
| DavResourceMetadata::AddressBook { name, .. } => {
let path = DavPath {
path: name.to_string(),
parent_id: None,
hierarchy_seq: 1,
resource_idx,
};
cache.size +=
(std::mem::size_of::<DavPath>() + name.len() + path.path.len()) as u64;
cache.paths.insert(path);
}
DavResourceMetadata::CalendarEvent { names, .. }
| DavResourceMetadata::ContactCard { names } => {
for name in names {
if let Some(parent_name) = name_idx.get(&name.parent_id) {
let path = DavPath {
path: format!("{parent_name}/{}", name.name),
parent_id: Some(name.parent_id),
hierarchy_seq: 1,
resource_idx,
};
cache.size += (std::mem::size_of::<DavPath>()
+ name.name.len()
+ path.path.len()) as u64;
cache.paths.insert(path);
}
}
}
_ => unreachable!(),
}
cache.size += std::mem::size_of::<DavResource>() as u64;
}
}
pub(super) fn resource_from_calendar(calendar: &ArchivedCalendar, document_id: u32) -> DavResource {
DavResource {
document_id,
data: DavResourceMetadata::Calendar {
name: calendar.name.to_string(),
acls: calendar
.acls
.iter()
.map(|acl| AclGrant {
account_id: acl.account_id.to_native(),
grants: Bitmap::from(&acl.grants),
})
.collect(),
tz: calendar
.preferences
.first()
.and_then(|pref| pref.time_zone.tz())
.unwrap_or(Tz::UTC),
},
}
}
pub(super) fn resource_from_event(event: &ArchivedCalendarEvent, document_id: u32) -> DavResource {
let (start, duration) = event.data.event_range().unwrap_or_default();
DavResource {
document_id,
data: DavResourceMetadata::CalendarEvent {
names: event
.names
.iter()
.map(|name| DavName {
name: name.name.to_string(),
parent_id: name.parent_id.to_native(),
})
.collect(),
start,
duration,
},
}
}
pub(super) fn resource_from_addressbook(
book: &ArchivedAddressBook,
document_id: u32,
) -> DavResource {
DavResource {
document_id,
data: DavResourceMetadata::AddressBook {
name: book.name.to_string(),
acls: book
.acls
.iter()
.map(|acl| AclGrant {
account_id: acl.account_id.to_native(),
grants: Bitmap::from(&acl.grants),
})
.collect(),
},
}
}
pub(super) fn resource_from_card(card: &ArchivedContactCard, document_id: u32) -> DavResource {
DavResource {
document_id,
data: DavResourceMetadata::ContactCard {
names: card
.names
.iter()
.map(|name| DavName {
name: name.name.to_string(),
parent_id: name.parent_id.to_native(),
})
.collect(),
},
}
}

182
crates/groupware/src/cache/file.rs vendored Normal file
View File

@@ -0,0 +1,182 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::{
DavResourceName,
file::{ArchivedFileNode, FileNode},
};
use common::{DavPath, DavResource, DavResourceMetadata, DavResources, Server};
use directory::backend::internal::manage::ManageDirectory;
use jmap_proto::types::{
collection::{Collection, SyncCollection},
property::Property,
value::AclGrant,
};
use percent_encoding::NON_ALPHANUMERIC;
use std::sync::Arc;
use store::{
Deserialize, IterateParams, U32_LEN, ValueKey,
ahash::{AHashMap, AHashSet},
write::{AlignedBytes, Archive, ValueClass, key::DeserializeBigEndian},
};
use tokio::sync::Semaphore;
use trc::AddContext;
use utils::{map::bitmap::Bitmap, topological::TopologicalSort};
pub(super) async fn build_file_resources(
server: &Server,
account_id: u32,
update_lock: Arc<Semaphore>,
) -> trc::Result<DavResources> {
let last_change_id = server
.core
.storage
.data
.get_last_change_id(account_id, SyncCollection::FileNode)
.await
.caused_by(trc::location!())?
.unwrap_or_default();
let name = server
.store()
.get_principal_name(account_id)
.await
.caused_by(trc::location!())?
.unwrap_or_else(|| format!("_{account_id}"));
let resources = fetch_files(server, account_id).await?;
let mut files = DavResources {
base_path: format!(
"{}/{}/",
DavResourceName::File.base_path(),
percent_encoding::utf8_percent_encode(&name, NON_ALPHANUMERIC),
),
size: std::mem::size_of::<DavResources>() as u64,
paths: AHashSet::with_capacity(resources.len()),
resources,
item_change_id: last_change_id,
container_change_id: last_change_id,
highest_change_id: last_change_id,
update_lock,
};
build_nested_hierarchy(&mut files);
Ok(files)
}
pub(super) fn build_nested_hierarchy(resources: &mut DavResources) {
let mut topological_sort = TopologicalSort::with_capacity(resources.resources.len());
let mut names = AHashMap::with_capacity(resources.resources.len());
for (resource_idx, resource) in resources.resources.iter().enumerate() {
if let DavResourceMetadata::File { parent_id, .. } = resource.data {
topological_sort.insert(
parent_id.map(|id| id + 1).unwrap_or_default(),
resource.document_id + 1,
);
names.insert(
resource.document_id,
DavPath {
path: resource.container_name().unwrap().to_string(),
parent_id,
hierarchy_seq: 0,
resource_idx,
},
);
}
}
for (hierarchy_sequence, folder_id) in topological_sort.into_iterator().enumerate() {
if folder_id != 0 {
let folder_id = folder_id - 1;
if let Some((name, parent_name)) = names
.get(&folder_id)
.and_then(|folder| folder.parent_id.map(|parent_id| (&folder.path, parent_id)))
.and_then(|(name, parent_id)| {
names.get(&parent_id).map(|folder| (name, &folder.path))
})
{
let name = format!("{parent_name}/{name}");
let folder = names.get_mut(&folder_id).unwrap();
folder.path = name;
folder.hierarchy_seq = hierarchy_sequence as u32;
} else {
names.get_mut(&folder_id).unwrap().hierarchy_seq = hierarchy_sequence as u32;
}
}
}
resources.paths = names
.into_values()
.inspect(|v| {
resources.size += (std::mem::size_of::<DavPath>()
+ std::mem::size_of::<u32>()
+ std::mem::size_of::<usize>()
+ std::mem::size_of::<DavResource>()
+ v.path.len()) as u64;
})
.collect();
}
async fn fetch_files(server: &Server, account_id: u32) -> trc::Result<Vec<DavResource>> {
let mut files = Vec::with_capacity(16);
server
.store()
.iterate(
IterateParams::new(
ValueKey {
account_id,
collection: Collection::FileNode.into(),
document_id: 0,
class: ValueClass::Property(Property::Value.into()),
},
ValueKey {
account_id,
collection: Collection::FileNode.into(),
document_id: u32::MAX,
class: ValueClass::Property(Property::Value.into()),
},
),
|key, value| {
let archive = <Archive<AlignedBytes> as Deserialize>::deserialize(value)?;
files.push(resource_from_file(
archive.unarchive::<FileNode>()?,
key.deserialize_be_u32(key.len() - U32_LEN)?,
));
Ok(true)
},
)
.await
.caused_by(trc::location!())?;
Ok(files)
}
pub(super) fn resource_from_file(node: &ArchivedFileNode, document_id: u32) -> DavResource {
let parent_id = node.parent_id.to_native();
DavResource {
document_id,
data: DavResourceMetadata::File {
name: node.name.as_str().to_string(),
size: node.file.as_ref().map(|f| f.size.to_native()),
parent_id: if parent_id > 0 {
Some(parent_id - 1)
} else {
None
},
acls: node
.acls
.iter()
.map(|acl| AclGrant {
account_id: acl.account_id.to_native(),
grants: Bitmap::from(&acl.grants),
})
.collect(),
},
}
}

414
crates/groupware/src/cache/mod.rs vendored Normal file
View File

@@ -0,0 +1,414 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::{
calendar::{Calendar, CalendarEvent, CalendarPreferences},
contact::{AddressBook, ContactCard},
file::FileNode,
};
use calcard::{
build_calcard_resources, build_simple_hierarchy, resource_from_addressbook,
resource_from_calendar, resource_from_card, resource_from_event,
};
use common::{CacheSwap, DavResource, DavResources, Server, auth::AccessToken};
use file::{build_file_resources, build_nested_hierarchy, resource_from_file};
use jmap_proto::types::collection::{Collection, SyncCollection};
use std::sync::Arc;
use store::{
ahash::AHashMap,
query::log::{Change, Query},
write::{AlignedBytes, Archive, BatchBuilder},
};
use tokio::sync::Semaphore;
use trc::AddContext;
pub mod calcard;
pub mod file;
pub trait GroupwareCache: Sync + Send {
fn fetch_dav_resources(
&self,
access_token: &AccessToken,
account_id: u32,
collection: SyncCollection,
) -> impl Future<Output = trc::Result<Arc<DavResources>>> + Send;
fn create_default_addressbook(
&self,
access_token: &AccessToken,
account_id: u32,
) -> impl Future<Output = trc::Result<()>> + Send;
fn create_default_calendar(
&self,
access_token: &AccessToken,
account_id: u32,
) -> impl Future<Output = trc::Result<()>> + Send;
fn cached_dav_resources(
&self,
account_id: u32,
collection: SyncCollection,
) -> Option<Arc<DavResources>>;
}
impl GroupwareCache for Server {
async fn fetch_dav_resources(
&self,
access_token: &AccessToken,
account_id: u32,
collection: SyncCollection,
) -> trc::Result<Arc<DavResources>> {
let cache_store = match collection {
SyncCollection::Calendar => &self.inner.cache.events,
SyncCollection::AddressBook => &self.inner.cache.contacts,
SyncCollection::FileNode => &self.inner.cache.files,
_ => unreachable!(),
};
let cache_ = match cache_store.get_value_or_guard_async(&account_id).await {
Ok(cache) => cache,
Err(guard) => {
let cache = full_cache_build(
self,
account_id,
collection,
Arc::new(Semaphore::new(1)),
access_token,
)
.await?;
if guard.insert(CacheSwap::new(cache.clone())).is_err() {
cache_store.insert(account_id, CacheSwap::new(cache.clone()));
}
return Ok(cache);
}
};
// Perform full refresh on stale ids
let cache = cache_.load_full();
if cache.highest_change_id > 0
&& self
.core
.jmap
.changes_max_history
.and_then(|history| self.inner.data.jmap_id_gen.past_id(history))
.is_some_and(|last_change_id| cache.highest_change_id < last_change_id)
{
let cache = full_cache_build(
self,
account_id,
collection,
cache.update_lock.clone(),
access_token,
)
.await?;
cache_.update(cache.clone());
return Ok(cache);
}
// Obtain current state
let changes = self
.core
.storage
.data
.changes(
account_id,
collection,
Query::Since(cache.highest_change_id),
)
.await
.caused_by(trc::location!())?;
// Verify changes
if changes.changes.is_empty() {
return Ok(cache);
}
// Lock for updates
let _permit = cache.update_lock.acquire().await;
let cache = cache_.load_full();
if cache.highest_change_id >= changes.to_change_id {
return Ok(cache);
}
let mut updated_resources = AHashMap::with_capacity(8);
let has_no_children = collection == SyncCollection::FileNode;
for change in changes.changes {
match change {
Change::InsertItem(id) | Change::UpdateItem(id) => {
let document_id = id as u32;
if let Some(archive) = self
.get_archive(account_id, collection.collection(false), document_id)
.await
.caused_by(trc::location!())?
{
updated_resources.insert(
(has_no_children, document_id),
Some(resource_from_archive(
archive,
document_id,
collection,
false,
)?),
);
} else {
updated_resources.insert((has_no_children, document_id), None);
}
}
Change::DeleteItem(id) => {
updated_resources.insert((has_no_children, id as u32), None);
}
Change::InsertContainer(id) | Change::UpdateContainer(id) => {
let document_id = id as u32;
if let Some(archive) = self
.get_archive(account_id, collection.collection(true), document_id)
.await
.caused_by(trc::location!())?
{
updated_resources.insert(
(true, document_id),
Some(resource_from_archive(
archive,
document_id,
collection,
true,
)?),
);
} else {
updated_resources.insert((true, document_id), None);
}
}
Change::DeleteContainer(id) => {
updated_resources.insert((true, id as u32), None);
}
Change::UpdateContainerProperty(_) => (),
}
}
let mut rebuild_hierarchy = false;
let mut resources = Vec::with_capacity(cache.resources.len());
for resource in &cache.resources {
let is_container = has_no_children || resource.is_container();
if let Some(updated_resource) =
updated_resources.remove(&(is_container, resource.document_id))
{
if let Some(updated_resource) = updated_resource {
rebuild_hierarchy =
rebuild_hierarchy || updated_resource.has_hierarchy_changes(resource);
resources.push(updated_resource);
} else {
// Deleted resource
rebuild_hierarchy = true;
}
} else {
resources.push(resource.clone());
}
}
// Add new resources
for resource in updated_resources.into_values().flatten() {
resources.push(resource);
rebuild_hierarchy = true;
}
let cache = if rebuild_hierarchy {
let mut cache = DavResources {
base_path: cache.base_path.clone(),
paths: Default::default(),
resources,
item_change_id: changes.item_change_id.unwrap_or(cache.item_change_id),
container_change_id: changes
.container_change_id
.unwrap_or(cache.container_change_id),
highest_change_id: changes.to_change_id,
size: std::mem::size_of::<DavResources>() as u64,
update_lock: cache.update_lock.clone(),
};
if matches!(collection, SyncCollection::FileNode) {
build_nested_hierarchy(&mut cache);
} else {
build_simple_hierarchy(&mut cache);
}
cache
} else {
DavResources {
base_path: cache.base_path.clone(),
paths: cache.paths.clone(),
resources,
item_change_id: changes.item_change_id.unwrap_or(cache.item_change_id),
container_change_id: changes
.container_change_id
.unwrap_or(cache.container_change_id),
highest_change_id: changes.to_change_id,
size: cache.size,
update_lock: cache.update_lock.clone(),
}
};
let cache = Arc::new(cache);
cache_.update(cache.clone());
Ok(cache)
}
async fn create_default_addressbook(
&self,
access_token: &AccessToken,
account_id: u32,
) -> trc::Result<()> {
if let Some(name) = &self.core.groupware.default_addressbook_name {
let mut batch = BatchBuilder::new();
let document_id = self
.store()
.assign_document_ids(account_id, Collection::AddressBook, 1)
.await?;
AddressBook {
name: name.clone(),
display_name: self.core.groupware.default_addressbook_display_name.clone(),
is_default: true,
..Default::default()
}
.insert(access_token, account_id, document_id, &mut batch)?;
self.commit_batch(batch).await?;
}
Ok(())
}
async fn create_default_calendar(
&self,
access_token: &AccessToken,
account_id: u32,
) -> trc::Result<()> {
if let Some(name) = &self.core.groupware.default_calendar_name {
let mut batch = BatchBuilder::new();
let document_id = self
.store()
.assign_document_ids(account_id, Collection::Calendar, 1)
.await?;
Calendar {
name: name.clone(),
preferences: vec![CalendarPreferences {
account_id,
name: name.clone(),
description: self.core.groupware.default_calendar_display_name.clone(),
..Default::default()
}],
..Default::default()
}
.insert(access_token, account_id, document_id, &mut batch)?;
self.commit_batch(batch).await?;
}
Ok(())
}
fn cached_dav_resources(
&self,
account_id: u32,
collection: SyncCollection,
) -> Option<Arc<DavResources>> {
(match collection {
SyncCollection::Calendar => &self.inner.cache.events,
SyncCollection::AddressBook => &self.inner.cache.contacts,
SyncCollection::FileNode => &self.inner.cache.files,
_ => unreachable!(),
})
.get(&account_id)
.map(|cache| cache.load_full())
}
}
async fn full_cache_build(
server: &Server,
account_id: u32,
collection: SyncCollection,
update_lock: Arc<Semaphore>,
access_token: &AccessToken,
) -> trc::Result<Arc<DavResources>> {
match collection {
SyncCollection::Calendar => {
build_calcard_resources(
server,
access_token,
account_id,
SyncCollection::Calendar,
Collection::Calendar,
Collection::CalendarEvent,
update_lock,
)
.await
}
SyncCollection::AddressBook => {
build_calcard_resources(
server,
access_token,
account_id,
SyncCollection::AddressBook,
Collection::AddressBook,
Collection::ContactCard,
update_lock,
)
.await
}
SyncCollection::FileNode => build_file_resources(server, account_id, update_lock).await,
_ => unreachable!(),
}
.map(Arc::new)
}
fn resource_from_archive(
archive: Archive<AlignedBytes>,
document_id: u32,
collection: SyncCollection,
is_container: bool,
) -> trc::Result<DavResource> {
Ok(match collection {
SyncCollection::Calendar => {
if is_container {
resource_from_calendar(
archive
.unarchive::<Calendar>()
.caused_by(trc::location!())?,
document_id,
)
} else {
resource_from_event(
archive
.unarchive::<CalendarEvent>()
.caused_by(trc::location!())?,
document_id,
)
}
}
SyncCollection::AddressBook => {
if is_container {
resource_from_addressbook(
archive
.unarchive::<AddressBook>()
.caused_by(trc::location!())?,
document_id,
)
} else {
resource_from_card(
archive
.unarchive::<ContactCard>()
.caused_by(trc::location!())?,
document_id,
)
}
}
SyncCollection::FileNode => resource_from_file(
archive
.unarchive::<FileNode>()
.caused_by(trc::location!())?,
document_id,
),
_ => unreachable!(),
})
}

1
crates/groupware/src/calendar/dates.rs
View File

@@ -19,7 +19,6 @@ use calcard::{
};
use chrono::{DateTime, TimeZone};
use dav_proto::schema::property::TimeRange;
use rkyv::time;
use std::str::FromStr;
use store::{
ahash::AHashMap,

81
crates/groupware/src/calendar/index.rs
View File

@@ -4,37 +4,17 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use common::storage::index::{
IndexItem, IndexValue, IndexableAndSerializableObject, IndexableObject,
};
use jmap_proto::types::{collection::SyncCollection, value::AclGrant};
use store::{SerializeInfallible, write::key::KeySerializer};
use crate::{IDX_NAME, IDX_TIME, IDX_UID};
use super::{
ArchivedCalendar, ArchivedCalendarEvent, ArchivedCalendarPreferences, ArchivedDefaultAlert,
ArchivedTimezone, Calendar, CalendarEvent, CalendarPreferences, DefaultAlert, Timezone,
};
use crate::IDX_UID;
use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject};
use jmap_proto::types::{collection::SyncCollection, value::AclGrant};
impl IndexableObject for Calendar {
fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
// Note: When adding a new value with index id above 0u8, tune `build_hierarchy`` to skip
// this value during iteration.
[
IndexValue::Index {
field: IDX_NAME,
value: self.name.as_str().into(),
},
IndexValue::Index {
field: IDX_TIME,
value: self
.preferences
.first()
.and_then(|p| p.time_zone.tz())
.map(|tz| tz.as_id().serialize())
.into(),
},
IndexValue::Acl {
value: (&self.acls).into(),
},
@@ -55,19 +35,6 @@ impl IndexableObject for Calendar {
impl IndexableObject for &ArchivedCalendar {
fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
[
IndexValue::Index {
field: IDX_NAME,
value: self.name.as_str().into(),
},
IndexValue::Index {
field: IDX_TIME,
value: self
.preferences
.first()
.and_then(|p| p.time_zone.tz())
.map(|tz| tz.as_id().serialize())
.into(),
},
IndexValue::Acl {
value: self
.acls
@@ -95,31 +62,10 @@ impl IndexableAndSerializableObject for Calendar {}
impl IndexableObject for CalendarEvent {
fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
[
IndexValue::IndexList {
field: IDX_NAME,
value: self
.names
.iter()
.map(|v| IndexItem::Vec(v.serialize()))
.collect::<Vec<_>>(),
},
IndexValue::Index {
field: IDX_UID,
value: self.data.event.uids().next().into(),
},
IndexValue::Index {
field: IDX_TIME,
value: self
.data
.event_range()
.map(|(start, duration)| {
KeySerializer::new(std::mem::size_of::<i64>() + std::mem::size_of::<u32>())
.write(start as u64)
.write(duration)
.finalize()
})
.into(),
},
IndexValue::Quota {
used: self.dead_properties.size() as u32
+ self.display_name.as_ref().map_or(0, |n| n.len() as u32)
@@ -138,31 +84,10 @@ impl IndexableObject for CalendarEvent {
impl IndexableObject for &ArchivedCalendarEvent {
fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
[
IndexValue::IndexList {
field: IDX_NAME,
value: self
.names
.iter()
.map(|v| IndexItem::Vec(v.serialize()))
.collect::<Vec<_>>(),
},
IndexValue::Index {
field: IDX_UID,
value: self.data.event.uids().next().into(),
},
IndexValue::Index {
field: IDX_TIME,
value: self
.data
.event_range()
.map(|(start, duration)| {
KeySerializer::new(std::mem::size_of::<i64>() + std::mem::size_of::<u32>())
.write(start as u64)
.write(duration)
.finalize()
})
.into(),
},
IndexValue::Quota {
used: self.dead_properties.size() as u32
+ self.display_name.as_ref().map_or(0, |n| n.len() as u32)

2
crates/groupware/src/calendar/mod.rs
View File

@@ -8,8 +8,8 @@ pub mod dates;
pub mod index;
pub mod storage;
use crate::DavName;
use calcard::icalendar::ICalendar;
use common::DavName;
use dav_proto::schema::request::DeadProperty;
use jmap_proto::types::{acl::Acl, value::AclGrant};
use store::{SERIALIZE_CALENDAR_V1, SERIALIZE_CALENDAREVENT_V1, SerializedVersion};

37
crates/groupware/src/contact/index.rs
View File

@@ -4,25 +4,14 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use common::storage::index::{
IndexItem, IndexValue, IndexableAndSerializableObject, IndexableObject,
};
use jmap_proto::types::{collection::SyncCollection, value::AclGrant};
use store::SerializeInfallible;
use crate::{IDX_NAME, IDX_UID};
use super::{AddressBook, ArchivedAddressBook, ArchivedContactCard, ContactCard};
use crate::IDX_UID;
use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject};
use jmap_proto::types::{collection::SyncCollection, value::AclGrant};
impl IndexableObject for AddressBook {
fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
// Note: When adding a new value with index id above 0u8, tune `build_hierarchy`` to skip
// this value during iteration.
[
IndexValue::Index {
field: IDX_NAME,
value: self.name.as_str().into(),
},
IndexValue::Acl {
value: (&self.acls).into(),
},
@@ -43,10 +32,6 @@ impl IndexableObject for AddressBook {
impl IndexableObject for &ArchivedAddressBook {
fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
[
IndexValue::Index {
field: IDX_NAME,
value: self.name.as_str().into(),
},
IndexValue::Acl {
value: self
.acls
@@ -74,14 +59,6 @@ impl IndexableAndSerializableObject for AddressBook {}
impl IndexableObject for ContactCard {
fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
[
IndexValue::IndexList {
field: IDX_NAME,
value: self
.names
.iter()
.map(|v| IndexItem::Vec(v.serialize()))
.collect::<Vec<_>>(),
},
IndexValue::Index {
field: IDX_UID,
value: self.card.uid().into(),
@@ -104,14 +81,6 @@ impl IndexableObject for ContactCard {
impl IndexableObject for &ArchivedContactCard {
fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
[
IndexValue::IndexList {
field: IDX_NAME,
value: self
.names
.iter()
.map(|v| IndexItem::Vec(v.serialize()))
.collect::<Vec<_>>(),
},
IndexValue::Index {
field: IDX_UID,
value: self.card.uid().into(),

4
crates/groupware/src/contact/mod.rs
View File

@@ -8,13 +8,11 @@ pub mod index;
pub mod storage;
use calcard::vcard::VCard;
use common::DavName;
use dav_proto::schema::request::DeadProperty;
use jmap_proto::types::{acl::Acl, value::AclGrant};
use store::{SERIALIZE_ADDRESSBOOK_V1, SERIALIZE_CALENDAREVENT_V1, SerializedVersion};
use crate::DavName;
#[derive(
rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq,
)]

96
crates/groupware/src/file/index.rs
View File

@@ -4,57 +4,30 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use common::storage::{
folder::FolderHierarchy,
index::{IndexValue, IndexableAndSerializableObject, IndexableObject},
};
use jmap_proto::types::{collection::SyncCollection, property::Property, value::AclGrant};
use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject};
use jmap_proto::types::{collection::SyncCollection, value::AclGrant};
use super::{ArchivedFileNode, FileNode};
impl IndexableObject for FileNode {
fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
let size = self.dead_properties.size() as u32
+ self.display_name.as_ref().map_or(0, |n| n.len() as u32)
+ self.name.len() as u32;
let mut values = Vec::with_capacity(6);
values.extend([
IndexValue::Index {
field: Property::Name.into(),
value: percent_encoding::percent_decode_str(&self.name)
.decode_utf8()
.unwrap_or_else(|_| self.name.as_str().into())
.to_lowercase()
.into(),
},
IndexValue::Index {
field: Property::ParentId.into(),
value: self.parent_id.into(),
},
IndexValue::Acl {
value: (&self.acls).into(),
},
IndexValue::LogContainer {
IndexValue::LogItem {
prefix: None,
sync_collection: SyncCollection::FileNode.into(),
},
IndexValue::Quota { used: self.size() },
]);
if let Some(file) = &self.file {
let size = size + file.size;
values.extend([
IndexValue::Blob {
value: file.blob_hash.clone(),
},
IndexValue::Index {
field: Property::Size.into(),
value: size.into(),
},
IndexValue::Quota { used: size },
]);
} else {
values.push(IndexValue::Quota { used: size });
values.extend([IndexValue::Blob {
value: file.blob_hash.clone(),
}]);
}
values.into_iter()
@@ -66,14 +39,6 @@ impl IndexableObject for &ArchivedFileNode {
let mut values = Vec::with_capacity(6);
values.extend([
IndexValue::Index {
field: Property::Name.into(),
value: self.name.to_lowercase().into(),
},
IndexValue::Index {
field: Property::ParentId.into(),
value: u32::from(self.parent_id).into(),
},
IndexValue::Acl {
value: self
.acls
@@ -82,25 +47,17 @@ impl IndexableObject for &ArchivedFileNode {
.collect::<Vec<_>>()
.into(),
},
IndexValue::LogContainer {
IndexValue::LogItem {
prefix: None,
sync_collection: SyncCollection::FileNode.into(),
},
IndexValue::Quota { used: self.size() },
]);
let size = self.size();
if let Some(file) = self.file.as_ref() {
values.extend([
IndexValue::Blob {
value: (&file.blob_hash).into(),
},
IndexValue::Index {
field: Property::Size.into(),
value: size.into(),
},
IndexValue::Quota { used: size },
]);
} else {
values.push(IndexValue::Quota { used: size });
values.extend([IndexValue::Blob {
value: (&file.blob_hash).into(),
}]);
}
values.into_iter()
@@ -109,19 +66,11 @@ impl IndexableObject for &ArchivedFileNode {
impl IndexableAndSerializableObject for FileNode {}
impl FolderHierarchy for ArchivedFileNode {
fn name(&self) -> String {
self.name.to_string()
}
fn parent_id(&self) -> u32 {
u32::from(self.parent_id)
}
fn is_container(&self) -> bool {
self.file.is_none()
}
pub trait NodeSize {
fn size(&self) -> u32;
}
impl NodeSize for ArchivedFileNode {
fn size(&self) -> u32 {
self.dead_properties.size() as u32
+ self.display_name.as_ref().map_or(0, |n| n.len() as u32)
@@ -129,3 +78,12 @@ impl FolderHierarchy for ArchivedFileNode {
+ self.file.as_ref().map_or(0, |f| u32::from(f.size))
}
}
impl NodeSize for FileNode {
fn size(&self) -> u32 {
self.dead_properties.size() as u32
+ self.display_name.as_ref().map_or(0, |n| n.len() as u32)
+ self.name.len() as u32
+ self.file.as_ref().map_or(0, |f| f.size)
}
}

372
crates/groupware/src/hierarchy.rs
View File

@@ -1,372 +0,0 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::{
DavName, DavResourceName, IDX_NAME, IDX_TIME,
calendar::{Calendar, CalendarPreferences},
contact::AddressBook,
file::FileNode,
};
use calcard::common::timezone::Tz;
use common::{
DavResource, DavResourceId, DavResourceMetadata, DavResources, Server, auth::AccessToken,
};
use directory::backend::internal::manage::ManageDirectory;
use jmap_proto::types::collection::Collection;
use percent_encoding::NON_ALPHANUMERIC;
use std::sync::Arc;
use store::{
Deserialize, IndexKey, IndexKeyPrefix, IterateParams, SerializeInfallible, U32_LEN, U64_LEN,
ahash::AHashMap,
write::{BatchBuilder, key::DeserializeBigEndian},
};
use trc::AddContext;
use utils::bimap::IdBimap;
pub trait DavHierarchy: Sync + Send {
fn fetch_dav_resources(
&self,
access_token: &AccessToken,
account_id: u32,
collection: Collection,
) -> impl Future<Output = trc::Result<Arc<DavResources>>> + Send;
fn create_default_addressbook(
&self,
access_token: &AccessToken,
account_id: u32,
) -> impl Future<Output = trc::Result<()>> + Send;
fn create_default_calendar(
&self,
access_token: &AccessToken,
account_id: u32,
) -> impl Future<Output = trc::Result<()>> + Send;
fn cached_dav_resources(
&self,
account_id: u32,
collection: Collection,
) -> Option<Arc<DavResources>>;
}
impl DavHierarchy for Server {
async fn fetch_dav_resources(
&self,
access_token: &AccessToken,
account_id: u32,
collection: Collection,
) -> trc::Result<Arc<DavResources>> {
let todo = "fix";
let is_files = collection == Collection::FileNode;
let mut change_id = self
.store()
.get_last_change_id(account_id, collection)
.await
.caused_by(trc::location!())?;
if !is_files {
let child_change_id = self
.store()
.get_last_change_id(account_id, collection.child_collection().unwrap())
.await
.caused_by(trc::location!())?;
change_id = change_id.max(child_change_id);
}
let resource_id = DavResourceId {
account_id,
collection: collection.into(),
};
if let Some(files) = self
.inner
.cache
.dav
.get(&resource_id)
.filter(|x| x.modseq == change_id)
{
Ok(files)
} else {
let mut files = if !is_files {
let files = build_hierarchy(self, account_id, collection).await?;
if files.paths.is_empty() {
match collection {
Collection::Calendar => {
self.create_default_calendar(access_token, account_id)
.await?
}
Collection::AddressBook => {
self.create_default_addressbook(access_token, account_id)
.await?
}
_ => unreachable!(),
}
build_hierarchy(self, account_id, collection).await?
} else {
files
}
} else {
build_file_hierarchy(self, account_id).await?
};
files.modseq = change_id;
let files = Arc::new(files);
self.inner.cache.dav.insert(resource_id, files.clone());
Ok(files)
}
}
async fn create_default_addressbook(
&self,
access_token: &AccessToken,
account_id: u32,
) -> trc::Result<()> {
if let Some(name) = &self.core.groupware.default_addressbook_name {
let mut batch = BatchBuilder::new();
let document_id = self
.store()
.assign_document_ids(account_id, Collection::AddressBook, 1)
.await?;
AddressBook {
name: name.clone(),
display_name: self.core.groupware.default_addressbook_display_name.clone(),
is_default: true,
..Default::default()
}
.insert(access_token, account_id, document_id, &mut batch)?;
self.commit_batch(batch).await?;
}
Ok(())
}
async fn create_default_calendar(
&self,
access_token: &AccessToken,
account_id: u32,
) -> trc::Result<()> {
if let Some(name) = &self.core.groupware.default_calendar_name {
let mut batch = BatchBuilder::new();
let document_id = self
.store()
.assign_document_ids(account_id, Collection::Calendar, 1)
.await?;
Calendar {
name: name.clone(),
preferences: vec![CalendarPreferences {
account_id,
name: name.clone(),
description: self.core.groupware.default_calendar_display_name.clone(),
..Default::default()
}],
..Default::default()
}
.insert(access_token, account_id, document_id, &mut batch)?;
self.commit_batch(batch).await?;
}
Ok(())
}
fn cached_dav_resources(
&self,
account_id: u32,
collection: Collection,
) -> Option<Arc<DavResources>> {
self.inner
.cache
.dav
.get(&DavResourceId {
account_id,
collection: collection.into(),
})
.clone()
}
}
async fn build_hierarchy(
server: &Server,
account_id: u32,
collection_: Collection,
) -> trc::Result<DavResources> {
let base_path = DavResourceName::from(collection_).base_path();
let collection = u8::from(collection_);
let mut containers: AHashMap<u32, String> = AHashMap::with_capacity(16);
let mut resources: AHashMap<u32, Vec<DavName>> = AHashMap::with_capacity(16);
let mut time_ranges: AHashMap<u32, (i64, u32)> = AHashMap::new();
let mut time_zones: AHashMap<u32, Tz> = AHashMap::new();
server
.store()
.iterate(
IterateParams::new(
IndexKey {
account_id,
collection,
document_id: 0,
field: IDX_NAME,
key: 0u32.serialize(),
},
IndexKey {
account_id,
collection: collection + 1,
document_id: u32::MAX,
field: IDX_TIME,
key: u32::MAX.serialize(),
},
)
.no_values()
.ascending(),
|key, _| {
let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?;
let value = key
.get(IndexKeyPrefix::len()..key.len() - U32_LEN)
.ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?;
let key_collection = key
.get(U32_LEN)
.copied()
.ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?;
let field = key
.get(U32_LEN + 1)
.copied()
.ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?;
if key_collection == collection {
if field == IDX_NAME {
containers.insert(
document_id,
std::str::from_utf8(value)
.map_err(|_| {
trc::Error::corrupted_key(key, None, trc::location!())
})?
.to_string(),
);
} else if field == IDX_TIME {
let tz = Tz::from_id(key.deserialize_be_u16(IndexKeyPrefix::len())?)
.ok_or_else(|| {
trc::Error::corrupted_key(key, None, trc::location!())
})?;
time_zones.insert(document_id, tz);
}
} else if field == IDX_NAME {
resources.entry(document_id).or_default().push(
DavName::deserialize(value)
.map_err(|_| trc::Error::corrupted_key(key, None, trc::location!()))?,
);
} else if field == IDX_TIME {
let start_time = key.deserialize_be_u64(IndexKeyPrefix::len())?;
let duration = key.deserialize_be_u32(IndexKeyPrefix::len() + U64_LEN)?;
time_ranges.insert(document_id, (start_time as i64, duration));
}
Ok(true)
},
)
.await
.caused_by(trc::location!())?;
let name = server
.store()
.get_principal_name(account_id)
.await
.caused_by(trc::location!())?
.unwrap_or_else(|| format!("_{account_id}"));
let mut files = DavResources {
paths: IdBimap::with_capacity(containers.len() + resources.len()),
size: std::mem::size_of::<DavResources>() as u64,
modseq: None,
base_path: format!(
"{}/{}/",
base_path,
percent_encoding::utf8_percent_encode(&name, NON_ALPHANUMERIC),
),
};
for (document_id, dav_names) in resources {
for dav_name in dav_names {
if let Some(container) = containers.get(&dav_name.parent_id) {
let name = format!("{}/{}", container, dav_name.name);
files.size += (std::mem::size_of::<u32>()
+ std::mem::size_of::<String>()
+ name.len()) as u64;
files.paths.insert(DavResource {
document_id,
parent_id: dav_name.parent_id.into(),
name,
data: time_ranges
.get(&document_id)
.map(|(start, duration)| DavResourceMetadata::CalendarEvent {
start: *start,
duration: *duration,
})
.unwrap_or(DavResourceMetadata::None),
});
}
}
}
for (document_id, name) in containers {
files.size +=
(std::mem::size_of::<u32>() + std::mem::size_of::<String>() + name.len()) as u64;
files.paths.insert(DavResource {
document_id,
parent_id: None,
name,
data: time_zones
.get(&document_id)
.map(|tz| DavResourceMetadata::Calendar { tz: *tz })
.unwrap_or(DavResourceMetadata::None),
});
}
Ok(files)
}
async fn build_file_hierarchy(server: &Server, account_id: u32) -> trc::Result<DavResources> {
let list = server
.fetch_folders::<FileNode>(account_id, Collection::FileNode)
.await
.caused_by(trc::location!())?;
let name = server
.store()
.get_principal_name(account_id)
.await
.caused_by(trc::location!())?
.unwrap_or_else(|| format!("_{account_id}"));
let mut files = DavResources {
base_path: format!(
"{}/{}/",
DavResourceName::File.base_path(),
percent_encoding::utf8_percent_encode(&name, NON_ALPHANUMERIC),
),
paths: IdBimap::with_capacity(list.len()),
size: std::mem::size_of::<DavResources>() as u64,
modseq: None,
};
for expanded in list.into_iterator() {
files.size += (std::mem::size_of::<u32>()
+ std::mem::size_of::<String>()
+ expanded.name.len()) as u64;
files.paths.insert(DavResource {
document_id: expanded.document_id,
parent_id: expanded.parent_id,
name: expanded.name,
data: DavResourceMetadata::File {
size: expanded.size,
hierarchy_sequence: expanded.hierarchy_sequence,
is_container: expanded.is_container,
},
});
}
Ok(files)
}

76
crates/groupware/src/lib.rs
View File

@@ -7,17 +7,13 @@
use calcard::common::timezone::Tz;
use common::DavResources;
use jmap_proto::types::collection::Collection;
use store::{Deserialize, SerializeInfallible, write::key::KeySerializer};
use utils::codec::leb128::Leb128Reader;
pub mod cache;
pub mod calendar;
pub mod contact;
pub mod file;
pub mod hierarchy;
pub const IDX_NAME: u8 = 0;
pub const IDX_TIME: u8 = 1;
pub const IDX_UID: u8 = 2;
pub const IDX_UID: u8 = 0;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DavResourceName {
@@ -29,61 +25,6 @@ pub enum DavResourceName {
pub struct DestroyArchive<T>(pub T);
#[derive(
rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq,
)]
#[rkyv(derive(Debug))]
pub struct DavName {
pub name: String,
pub parent_id: u32,
}
impl SerializeInfallible for DavName {
fn serialize(&self) -> Vec<u8> {
KeySerializer::new(self.name.len() + std::mem::size_of::<u32>())
.write_leb128(self.parent_id)
.write(self.name.as_bytes())
.finalize()
}
}
impl SerializeInfallible for ArchivedDavName {
fn serialize(&self) -> Vec<u8> {
KeySerializer::new(self.name.len() + std::mem::size_of::<u32>())
.write_leb128(self.parent_id.to_native())
.write(self.name.as_bytes())
.finalize()
}
}
impl DavName {
pub fn new(name: String, parent_id: u32) -> Self {
Self { name, parent_id }
}
}
impl Deserialize for DavName {
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
let (parent_id, bytes_read) = bytes.read_leb128::<u32>().ok_or_else(|| {
trc::StoreEvent::DataCorruption
.caused_by(trc::location!())
.ctx(trc::Key::Value, bytes)
})?;
let name = bytes
.get(bytes_read..)
.and_then(|bytes| std::str::from_utf8(bytes).ok())
.ok_or_else(|| {
trc::StoreEvent::DataCorruption
.caused_by(trc::location!())
.ctx(trc::Key::Value, bytes)
})?
.into();
Ok(DavName { name, parent_id })
}
}
impl DavResourceName {
pub fn parse(service: &str) -> Option<Self> {
hashify::tiny_map!(service.as_bytes(),
@@ -138,22 +79,11 @@ impl From<Collection> for DavResourceName {
pub trait DavCalendarResource {
fn calendar_default_tz(&self, calendar_id: u32) -> Option<Tz>;
fn event_default_tz(&self, event_id: u32) -> Option<Tz>;
}
impl DavCalendarResource for DavResources {
fn calendar_default_tz(&self, calendar_id: u32) -> Option<Tz> {
self.paths
.iter()
.find(|c| c.is_container() && c.document_id == calendar_id)
self.container_resource_by_id(calendar_id)
.and_then(|c| c.timezone())
}
fn event_default_tz(&self, event_id: u32) -> Option<Tz> {
self.paths
.iter()
.find(|c| !c.is_container() && c.document_id == event_id)
.and_then(|c| c.parent_id)
.and_then(|parent_id| self.calendar_default_tz(parent_id))
}
}