filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Store incoming reports in the data store

Browse Source
This commit is contained in:
mdecimus
2024-03-02 12:21:03 +01:00
parent 417bc38288
commit 875b1fa744
40 changed files with 1992 additions and 1541 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crates/store/src/backend/foundationdb/write.rs
View File

@@ -390,7 +390,7 @@ impl FdbStore {
}
}
pub(crate) async fn purge_bitmaps(&self) -> crate::Result<()> {
pub(crate) async fn purge_store(&self) -> crate::Result<()> {
// Obtain all empty bitmaps
let trx = self.db.create_trx()?;
let mut iter = trx.get_ranges(

2
crates/store/src/backend/mysql/write.rs
View File

@@ -270,7 +270,7 @@ impl MysqlStore {
trx.commit().await.map(|_| true)
}
pub(crate) async fn purge_bitmaps(&self) -> crate::Result<()> {
pub(crate) async fn purge_store(&self) -> crate::Result<()> {
let mut conn = self.conn_pool.get_conn().await?;
let s = conn

2
crates/store/src/backend/postgres/write.rs
View File

@@ -287,7 +287,7 @@ impl PostgresStore {
trx.commit().await.map(|_| true)
}
pub(crate) async fn purge_bitmaps(&self) -> crate::Result<()> {
pub(crate) async fn purge_store(&self) -> crate::Result<()> {
let conn = self.conn_pool.get().await?;
let s = conn

2
crates/store/src/backend/rocksdb/write.rs
View File

@@ -120,7 +120,7 @@ impl RocksDbStore {
.await
}
pub(crate) async fn purge_bitmaps(&self) -> crate::Result<()> {
pub(crate) async fn purge_store(&self) -> crate::Result<()> {
let db = self.db.clone();
self.spawn_worker(move || {
let cf = db

2
crates/store/src/backend/sqlite/write.rs
View File

@@ -203,7 +203,7 @@ impl SqliteStore {
.await
}
pub(crate) async fn purge_bitmaps(&self) -> crate::Result<()> {
pub(crate) async fn purge_store(&self) -> crate::Result<()> {
let conn = self.conn_pool.get()?;
self.spawn_worker(move || {
conn.prepare_cached(&format!(

2
crates/store/src/config.rs
View File

@@ -228,7 +228,7 @@ impl ConfigStore for Config {
schedules.push(PurgeSchedule {
cron,
store_id: store_id.to_string(),
store: PurgeStore::Bitmaps(store.clone()),
store: PurgeStore::Data(store.clone()),
});
}

2
crates/store/src/dispatch/lookup.rs
View File

@@ -294,7 +294,7 @@ impl LookupStore {
}
}
pub async fn purge_expired(&self) -> crate::Result<()> {
pub async fn purge_lookup_store(&self) -> crate::Result<()> {
match self {
LookupStore::Store(store) => {
// Delete expired keys

43
crates/store/src/dispatch/store.rs
View File

@@ -26,7 +26,7 @@ use std::ops::{BitAndAssign, Range};
use roaring::RoaringBitmap;
use crate::{
write::{key::KeySerializer, AnyKey, Batch, BitmapClass, ValueClass},
write::{key::KeySerializer, now, AnyKey, Batch, BitmapClass, ReportClass, ValueClass},
BitmapKey, Deserialize, IterateParams, Key, Store, ValueKey, SUBSPACE_BITMAPS,
SUBSPACE_INDEXES, SUBSPACE_LOGS, U32_LEN,
};
@@ -241,18 +241,45 @@ impl Store {
}
}
pub async fn purge_bitmaps(&self) -> crate::Result<()> {
pub async fn purge_store(&self) -> crate::Result<()> {
// Delete expired reports
let now = now();
self.delete_range(
ValueKey::from(ValueClass::Report(ReportClass::Dmarc { id: 0, expires: 0 })),
ValueKey::from(ValueClass::Report(ReportClass::Dmarc {
id: u64::MAX,
expires: now,
})),
)
.await?;
self.delete_range(
ValueKey::from(ValueClass::Report(ReportClass::Tls { id: 0, expires: 0 })),
ValueKey::from(ValueClass::Report(ReportClass::Tls {
id: u64::MAX,
expires: now,
})),
)
.await?;
self.delete_range(
ValueKey::from(ValueClass::Report(ReportClass::Arf { id: 0, expires: 0 })),
ValueKey::from(ValueClass::Report(ReportClass::Arf {
id: u64::MAX,
expires: now,
})),
)
.await?;
match self {
#[cfg(feature = "sqlite")]
Self::SQLite(store) => store.purge_bitmaps().await,
Self::SQLite(store) => store.purge_store().await,
#[cfg(feature = "foundation")]
Self::FoundationDb(store) => store.purge_bitmaps().await,
Self::FoundationDb(store) => store.purge_store().await,
#[cfg(feature = "postgres")]
Self::PostgreSQL(store) => store.purge_bitmaps().await,
Self::PostgreSQL(store) => store.purge_store().await,
#[cfg(feature = "mysql")]
Self::MySQL(store) => store.purge_bitmaps().await,
Self::MySQL(store) => store.purge_store().await,
#[cfg(feature = "rocks")]
Self::RocksDb(store) => store.purge_bitmaps().await,
Self::RocksDb(store) => store.purge_store().await,
}
}
@@ -462,7 +489,7 @@ impl Store {
self.blob_expire_all().await;
self.purge_blobs(blob_store).await.unwrap();
self.purge_bitmaps().await.unwrap();
self.purge_store().await.unwrap();
let store = self.clone();
let mut failed = false;

16
crates/store/src/write/key.rs
View File

@@ -31,8 +31,8 @@ use crate::{
};
use super::{
AnyKey, BitmapClass, BlobOp, DirectoryClass, LookupClass, QueueClass, ReportEvent, TagValue,
ValueClass,
AnyKey, BitmapClass, BlobOp, DirectoryClass, LookupClass, QueueClass, ReportClass, ReportEvent,
TagValue, ValueClass,
};
pub struct KeySerializer {
@@ -348,6 +348,17 @@ impl<T: AsRef<ValueClass> + Sync + Send> Key for ValueKey<T> {
QueueClass::QuotaCount(key) => serializer.write(55u8).write(key.as_slice()),
QueueClass::QuotaSize(key) => serializer.write(56u8).write(key.as_slice()),
},
ValueClass::Report(report) => match report {
ReportClass::Tls { id, expires } => {
serializer.write(60u8).write(*expires).write(*id)
}
ReportClass::Dmarc { id, expires } => {
serializer.write(61u8).write(*expires).write(*id)
}
ReportClass::Arf { id, expires } => {
serializer.write(62u8).write(*expires).write(*id)
}
},
}
.finalize()
}
@@ -503,6 +514,7 @@ impl ValueClass {
}
QueueClass::QuotaCount(v) | QueueClass::QuotaSize(v) => v.len(),
},
ValueClass::Report(_) => U64_LEN * 2 + 1,
}
}
}

8
crates/store/src/write/mod.rs
View File

@@ -140,6 +140,7 @@ pub enum ValueClass {
IndexEmail(u64),
Config(Vec<u8>),
Queue(QueueClass),
Report(ReportClass),
}
#[derive(Debug, PartialEq, Clone, Eq, Hash)]
@@ -172,6 +173,13 @@ pub enum QueueClass {
QuotaSize(Vec<u8>),
}
#[derive(Debug, PartialEq, Clone, Eq, Hash)]
pub enum ReportClass {
Tls { id: u64, expires: u64 },
Dmarc { id: u64, expires: u64 },
Arf { id: u64, expires: u64 },
}
#[derive(Debug, PartialEq, Clone, Eq, Hash)]
pub struct QueueEvent {
pub due: u64,

8
crates/store/src/write/purge.rs
View File

@@ -29,7 +29,7 @@ use utils::config::cron::SimpleCron;
use crate::{BlobStore, LookupStore, Store};
pub enum PurgeStore {
Bitmaps(Store),
Data(Store),
Blobs { store: Store, blob_store: BlobStore },
Lookup(LookupStore),
}
@@ -62,11 +62,11 @@ impl PurgeSchedule {
}
let result = match &self.store {
PurgeStore::Bitmaps(store) => store.purge_bitmaps().await,
PurgeStore::Data(store) => store.purge_store().await,
PurgeStore::Blobs { store, blob_store } => {
store.purge_blobs(blob_store.clone()).await
}
PurgeStore::Lookup(store) => store.purge_expired().await,
PurgeStore::Lookup(store) => store.purge_lookup_store().await,
};
if let Err(err) = result {
@@ -85,7 +85,7 @@ impl PurgeSchedule {
impl Display for PurgeStore {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
PurgeStore::Bitmaps(_) => write!(f, "bitmaps"),
PurgeStore::Data(_) => write!(f, "bitmaps"),
PurgeStore::Blobs { .. } => write!(f, "blobs"),
PurgeStore::Lookup(_) => write!(f, "expired keys"),
}