filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Database schema optimization - part 13 (fixes #1882 fixes #2415)

Browse Source
This commit is contained in:
mdecimus
2025-11-25 12:14:51 +01:00
parent c7fc16d9a2
commit 2b614aa536
62 changed files with 1661 additions and 2264 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
crates/http/src/form/mod.rs
View File

@@ -25,13 +25,8 @@ use mail_builder::{
};
use serde_json::json;
use std::{borrow::Cow, fmt::Write, future::Future};
use store::{
SerializeInfallible,
write::{BatchBuilder, BlobOp, now},
};
use store::write::BatchBuilder;
use trc::AddContext;
use types::blob_hash::BlobHash;
use x509_parser::nom::AsBytes;
pub trait FormHandler: Sync + Send {
fn handle_contact_form(
@@ -175,22 +170,8 @@ impl FormHandler for Server {
.unwrap_or_default();
// Reserve and write blob
let message_blob = BlobHash::generate(message.as_bytes());
let message_size = message.len() as u64;
let mut batch = BatchBuilder::new();
batch.set(
BlobOp::Reserve {
hash: message_blob.clone(),
until: now() + 120,
},
0u32.serialize(),
);
self.store()
.write(batch.build_all())
.await
.caused_by(trc::location!())?;
self.blob_store()
.put_blob(message_blob.as_slice(), message.as_ref())
let (message_blob, blob_hold) = self
.put_temporary_blob(u32::MAX, &message, 60)
.await
.caused_by(trc::location!())?;
@@ -200,7 +181,7 @@ impl FormHandler for Server {
sender_authenticated: false,
recipients: form.rcpt_to.clone(),
message_blob,
message_size,
message_size: message.len() as u64,
session_id: session.session_id,
})
.await
@@ -217,6 +198,14 @@ impl FormHandler for Server {
}
}
// Remove blob hold
let mut batch = BatchBuilder::new();
batch.clear(blob_hold);
self.store()
.write(batch.build_all())
.await
.caused_by(trc::location!())?;
// Suppress errors if there is at least one success
if has_success {
failure = None;

112
crates/http/src/management/enterprise/undelete.rs
View File

@@ -9,25 +9,24 @@
*/
use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
use common::{Server, enterprise::undelete::DeletedBlob};
use common::{Server, enterprise::undelete::DeletedItemType};
use directory::backend::internal::manage::ManageDirectory;
use email::{
mailbox::INBOX_ID,
message::ingest::{EmailIngest, IngestEmail, IngestSource},
};
use http_proto::{request::decode_path_element, *};
use hyper::Method;
use mail_parser::{DateTime, MessageParser};
use serde_json::json;
use std::future::Future;
use std::str::FromStr;
use store::write::{BatchBuilder, BlobOp, ValueClass};
use store::write::{BatchBuilder, BlobLink, BlobOp};
use trc::AddContext;
use types::{blob_hash::BlobHash, collection::Collection};
use utils::url_params::UrlParams;
use http_proto::{request::decode_path_element, *};
#[derive(serde::Deserialize, serde::Serialize)]
#[derive(serde::Deserialize, serde::Serialize, Debug)]
pub struct UndeleteRequest<H, C, T> {
pub hash: H,
pub collection: C,
@@ -47,6 +46,41 @@ pub enum UndeleteResponse {
Error { reason: String },
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
pub struct DeletedBlobResponse {
pub hash: String,
pub size: u32,
#[serde(rename = "deletedAt")]
pub deleted_at: String,
#[serde(rename = "expiresAt")]
pub expires_at: String,
pub item: DeletedItemResponse,
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
#[serde(tag = "type")]
#[serde(rename_all = "camelCase")]
pub enum DeletedItemResponse {
Email {
from: Box<str>,
subject: Box<str>,
received_at: String,
},
FileNode {
name: Box<str>,
},
CalendarEvent {
title: Box<str>,
start_time: String,
},
ContactCard {
name: Box<str>,
},
SieveScript {
name: Box<str>,
},
}
pub trait UndeleteApi: Sync + Send {
fn handle_undelete_api_request(
&self,
@@ -87,19 +121,46 @@ impl UndeleteApi for Server {
// Sort ascending by deleted_at
let total = deleted.len();
deleted.sort_by(|a, b| a.deleted_at.cmp(&b.deleted_at));
deleted.sort_by(|a, b| a.item.deleted_at.cmp(&b.item.deleted_at));
let mut results = Vec::with_capacity(if limit > 0 { limit } else { total });
for blob in deleted {
if offset == 0 {
results.push(DeletedBlob {
results.push(DeletedBlobResponse {
hash: URL_SAFE_NO_PAD.encode(blob.hash.as_slice()),
size: blob.size,
deleted_at: DateTime::from_timestamp(blob.deleted_at as i64)
size: blob.item.size,
deleted_at: DateTime::from_timestamp(blob.item.deleted_at as i64)
.to_rfc3339(),
expires_at: DateTime::from_timestamp(blob.expires_at as i64)
.to_rfc3339(),
collection: Collection::from(blob.collection).to_string(),
item: match blob.item.typ {
DeletedItemType::Email {
from,
subject,
received_at,
} => DeletedItemResponse::Email {
from,
subject,
received_at: DateTime::from_timestamp(received_at as i64)
.to_rfc3339(),
},
DeletedItemType::FileNode { name } => {
DeletedItemResponse::FileNode { name }
}
DeletedItemType::CalendarEvent { title, start_time } => {
DeletedItemResponse::CalendarEvent {
title,
start_time: DateTime::from_timestamp(start_time as i64)
.to_rfc3339(),
}
}
DeletedItemType::ContactCard { name } => {
DeletedItemResponse::ContactCard { name }
}
DeletedItemType::SieveScript { name } => {
DeletedItemResponse::SieveScript { name }
}
},
});
if results.len() == limit {
break;
@@ -169,8 +230,20 @@ impl UndeleteApi for Server {
for blob in deleted {
results.push(UndeleteRequest {
hash: blob.hash,
collection: Collection::from(blob.collection),
time: blob.deleted_at,
collection: match blob.item.typ {
DeletedItemType::Email { .. } => Collection::Email,
DeletedItemType::FileNode { .. } => Collection::FileNode,
DeletedItemType::CalendarEvent { .. } => {
Collection::CalendarEvent
}
DeletedItemType::ContactCard { .. } => {
Collection::ContactCard
}
DeletedItemType::SieveScript { .. } => {
Collection::SieveScript
}
},
time: blob.item.deleted_at,
cancel_deletion: blob.expires_at.into(),
});
}
@@ -216,10 +289,17 @@ impl UndeleteApi for Server {
Ok(_) => {
results.push(UndeleteResponse::Success);
if let Some(cancel_deletion) = request.cancel_deletion {
batch.clear(ValueClass::Blob(BlobOp::Reserve {
hash: request.hash,
until: cancel_deletion,
}));
batch
.clear(BlobOp::Link {
hash: request.hash.clone(),
to: BlobLink::Temporary {
until: cancel_deletion,
},
})
.clear(BlobOp::Undelete {
hash: request.hash,
until: cancel_deletion,
});
}
}
Err(mut err)

97
crates/http/src/management/principal.rs
View File

@@ -4,7 +4,8 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use common::{KV_BAYES_MODEL_USER, Server, auth::AccessToken};
use crate::management::stores::destroy_account_data;
use common::{Server, auth::AccessToken};
use directory::{
DirectoryInner, Permission, PrincipalData, QueryBy, QueryParams, Type,
backend::internal::{
@@ -20,7 +21,6 @@ use hyper::{Method, header};
use serde_json::json;
use std::future::Future;
use std::sync::Arc;
use store::{search::SearchQuery, write::SearchIndex};
use trc::AddContext;
use utils::url_params::UrlParams;
@@ -376,12 +376,6 @@ impl PrincipalManager for Server {
if found {
let server = self.clone();
tokio::spawn(async move {
let has_bayes = server
.core
.spam
.bayes
.as_ref()
.is_some_and(|c| c.account_classify);
for principal in principals.items {
// Delete account
match server
@@ -399,41 +393,14 @@ impl PrincipalManager for Server {
}
}
if matches!(typ, Type::Individual | Type::Group) {
// Remove search index
for index in [
SearchIndex::Email,
SearchIndex::Contacts,
SearchIndex::Calendar,
] {
if let Err(err) = server
.core
.storage
.fts
.unindex(
SearchQuery::new(index).with_account_id(principal.id()),
)
.await
{
trc::error!(err.details("Failed to delete FTS index"));
}
}
// Delete bayes model
if has_bayes {
let mut key =
Vec::with_capacity(std::mem::size_of::<u32>() + 1);
key.push(KV_BAYES_MODEL_USER);
key.extend_from_slice(&principal.id().to_be_bytes());
if let Err(err) =
server.in_memory_store().key_delete_prefix(&key).await
{
trc::error!(
err.details("Failed to delete user bayes model")
);
}
}
if let Err(err) = destroy_account_data(
&server,
principal.id(),
matches!(typ, Type::Individual | Type::Group),
)
.await
{
trc::error!(err.details("Failed to delete principal"));
}
}
});
@@ -527,42 +494,14 @@ impl PrincipalManager for Server {
.delete_principal(QueryBy::Id(account_id))
.await?;
if matches!(typ, Type::Individual | Type::Group) {
// Remove FTS index
for index in [
SearchIndex::Email,
SearchIndex::Contacts,
SearchIndex::Calendar,
] {
if let Err(err) = self
.core
.storage
.fts
.unindex(SearchQuery::new(index).with_account_id(account_id))
.await
{
trc::error!(err.details("Failed to delete FTS index"));
}
}
// Delete bayes model
if self
.core
.spam
.bayes
.as_ref()
.is_some_and(|c| c.account_classify)
{
let mut key = Vec::with_capacity(std::mem::size_of::<u32>() + 1);
key.push(KV_BAYES_MODEL_USER);
key.extend_from_slice(&account_id.to_be_bytes());
if let Err(err) =
self.in_memory_store().key_delete_prefix(&key).await
{
trc::error!(err.details("Failed to delete user bayes model"));
}
}
if let Err(err) = destroy_account_data(
self,
account_id,
matches!(typ, Type::Individual | Type::Group),
)
.await
{
trc::error!(err.details("Failed to delete principal"));
}
// Increment revision

160
crates/http/src/management/stores.rs
View File

@@ -18,7 +18,11 @@ use directory::{
};
use email::{
cache::MessageCacheFetch,
message::{ingest::EmailIngest, metadata::MessageData},
message::{
ingest::EmailIngest,
metadata::{MessageData, MessageMetadata},
},
sieve::SieveScript,
};
use groupware::{
calendar::{Calendar, CalendarEvent, CalendarEventNotification},
@@ -32,12 +36,14 @@ use services::task_manager::index::ReindexIndexTask;
use std::future::Future;
use store::{
Serialize, rand,
write::{Archiver, BatchBuilder, DirectoryClass, SearchIndex, ValueClass},
search::SearchQuery,
write::{Archiver, BatchBuilder, BlobLink, BlobOp, DirectoryClass, SearchIndex, ValueClass},
};
use trc::AddContext;
use types::{
blob_hash::BlobHash,
collection::Collection,
field::{EmailField, MailboxField},
field::{EmailField, Field, MailboxField},
};
use utils::url_params::UrlParams;
@@ -398,6 +404,138 @@ pub async fn recalculate_quota(server: &Server, account_id: u32) -> trc::Result<
.map(|_| ())
}
pub async fn destroy_account_blobs(server: &Server, account_id: u32) -> trc::Result<()> {
let mut delete_keys = Vec::new();
for (collection, field) in [
(Collection::Email, u8::from(EmailField::Metadata)),
(Collection::FileNode, u8::from(Field::ARCHIVE)),
(Collection::SieveScript, u8::from(Field::ARCHIVE)),
] {
server
.all_archives(account_id, collection, field, |document_id, archive| {
match collection {
Collection::Email => {
let message = archive.unarchive::<MessageMetadata>()?;
delete_keys.push((
collection,
document_id,
BlobHash::from(&message.blob_hash),
));
}
Collection::FileNode => {
if let Some(file) = archive.unarchive::<FileNode>()?.file.as_ref() {
delete_keys.push((
collection,
document_id,
BlobHash::from(&file.blob_hash),
));
}
}
Collection::SieveScript => {
let sieve = archive.unarchive::<SieveScript>()?;
delete_keys.push((
collection,
document_id,
BlobHash::from(&sieve.blob_hash),
));
}
_ => {}
}
Ok(())
})
.await
.caused_by(trc::location!())?;
}
let mut batch = BatchBuilder::new();
batch.with_account_id(account_id);
for (collection, document_id, hash) in delete_keys {
if batch.is_large_batch() {
server
.store()
.write(batch.build_all())
.await
.caused_by(trc::location!())?;
batch = BatchBuilder::new();
batch.with_account_id(account_id);
}
batch
.with_collection(collection)
.with_document(document_id)
.clear(ValueClass::Blob(BlobOp::Link {
hash,
to: BlobLink::Document,
}));
}
if !batch.is_empty() {
server
.store()
.write(batch.build_all())
.await
.caused_by(trc::location!())?;
}
Ok(())
}
pub async fn destroy_account_data(
server: &Server,
account_id: u32,
has_data: bool,
) -> trc::Result<()> {
// Unlink all accounts's blobs
if has_data {
destroy_account_blobs(server, account_id).await?;
}
// Destroy account data
server
.store()
.danger_destroy_account(account_id)
.await
.caused_by(trc::location!())?;
if has_data {
// Remove search index
for index in [
SearchIndex::Email,
SearchIndex::Contacts,
SearchIndex::Calendar,
] {
if let Err(err) = server
.core
.storage
.fts
.unindex(SearchQuery::new(index).with_account_id(account_id))
.await
{
trc::error!(err.details("Failed to delete FTS index"));
}
}
// Delete bayes model
if server
.core
.spam
.bayes
.as_ref()
.is_some_and(|c| c.account_classify)
{
let mut key = Vec::with_capacity(std::mem::size_of::<u32>() + 1);
key.push(KV_BAYES_MODEL_USER);
key.extend_from_slice(&account_id.to_be_bytes());
if let Err(err) = server.in_memory_store().key_delete_prefix(&key).await {
trc::error!(err.details("Failed to delete user bayes model"));
}
}
}
Ok(())
}
pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u32, u32)> {
let mut mailbox_count = 0;
let mut email_count = 0;
@@ -455,11 +593,17 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u
.deserialize::<MessageData>()
.caused_by(trc::location!())?;
for uid_mailbox in &mut new_data.mailboxes {
uid_mailbox.uid = server
.assign_imap_uid(account_id, uid_mailbox.mailbox_id)
.await
.caused_by(trc::location!())?;
let ids = server
.assign_email_ids(
account_id,
new_data.mailboxes.iter().map(|m| m.mailbox_id),
false,
)
.await
.caused_by(trc::location!())?;
for (uid_mailbox, uid) in new_data.mailboxes.iter_mut().zip(ids) {
uid_mailbox.uid = uid;
}
// Prepare write batch