filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Metric history + Live metrics

Browse Source
This commit is contained in:
mdecimus
2024-08-23 16:21:35 +02:00
parent 0aaf493f94
commit dcc31e8b3e
34 changed files with 1256 additions and 512 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crates/common/src/config/network.rs
View File

@@ -32,6 +32,7 @@ impl Default for Network {
Self {
blocked_ips: Default::default(),
allowed_ips: Default::default(),
node_id: 0,
http_response_url: IfBlock::new::<()>(
"server.http.url",
[],
@@ -45,6 +46,7 @@ impl Default for Network {
impl Network {
pub fn parse(config: &mut Config) -> Self {
let mut network = Network {
node_id: config.property("cluster.node-id").unwrap_or_default(),
blocked_ips: BlockedIps::parse(config),
allowed_ips: AllowedIps::parse(config),
..Default::default()

34
crates/common/src/config/telemetry.rs
View File

@@ -137,28 +137,18 @@ impl Telemetry {
};
// Parse metrics
if config
.property_or_default("metrics.prometheus.enable", "false")
.unwrap_or(false)
|| ["http", "grpc"].contains(
&config
.value("metrics.open-telemetry.transport")
.unwrap_or("disabled"),
)
{
apply_events(
config
.properties::<EventOrMany>("metrics.disabled-events")
.into_iter()
.map(|(_, e)| e),
false,
|event_type| {
if event_type.is_metric() {
telemetry.metrics.set(event_type);
}
},
);
}
apply_events(
config
.properties::<EventOrMany>("metrics.disabled-events")
.into_iter()
.map(|(_, e)| e),
false,
|event_type| {
if event_type.is_metric() {
telemetry.metrics.set(event_type);
}
},
);
telemetry
}

68
crates/common/src/enterprise/config.rs
View File

@@ -12,9 +12,9 @@ use std::time::Duration;
use jmap_proto::types::collection::Collection;
use store::{BitmapKey, Store, Stores};
use utils::config::Config;
use utils::config::{cron::SimpleCron, utils::ParseValue, Config};
use super::{license::LicenseValidator, Enterprise};
use super::{license::LicenseValidator, Enterprise, MetricsStore, TraceStore, Undelete};
impl Enterprise {
pub async fn parse(config: &mut Config, stores: &Stores, data: &Store) -> Option<Self> {
@@ -54,18 +54,62 @@ impl Enterprise {
_ => (),
}
Some(Enterprise {
license,
undelete_period: config
.property_or_default::<Option<Duration>>("storage.undelete.retention", "false")
.unwrap_or_default(),
trace_hold_period: config
.property_or_default::<Option<Duration>>("tracing.history.retention", "30d")
.unwrap_or(Some(Duration::from_secs(30 * 24 * 60 * 60))),
trace_store: config
let trace_store = if config
.property_or_default("tracing.history.enable", "false")
.unwrap_or(false)
{
if let Some(store) = config
.value("tracing.history.store")
.and_then(|name| stores.stores.get(name))
.cloned(),
.cloned()
{
TraceStore {
retention: config
.property_or_default::<Option<Duration>>("tracing.history.retention", "30d")
.unwrap_or(Some(Duration::from_secs(30 * 24 * 60 * 60))),
store,
}
.into()
} else {
None
}
} else {
None
};
let metrics_store = if config
.property_or_default("metrics.history.enable", "false")
.unwrap_or(false)
{
if let Some(store) = config
.value("metrics.history.store")
.and_then(|name| stores.stores.get(name))
.cloned()
{
MetricsStore {
retention: config
.property_or_default::<Option<Duration>>("metrics.history.retention", "30d")
.unwrap_or(Some(Duration::from_secs(30 * 24 * 60 * 60))),
store,
interval: config
.property_or_default::<SimpleCron>("metrics.history.interval", "0 * *")
.unwrap_or_else(|| SimpleCron::parse_value("0 * *").unwrap()),
}
.into()
} else {
None
}
} else {
None
};
Some(Enterprise {
license,
undelete: config
.property_or_default::<Option<Duration>>("storage.undelete.retention", "false")
.unwrap_or_default()
.map(|retention| Undelete { retention }),
trace_store,
metrics_store,
})
}
}

25
crates/common/src/enterprise/mod.rs
View File

@@ -17,15 +17,34 @@ use std::time::Duration;
use license::LicenseKey;
use mail_parser::DateTime;
use store::Store;
use utils::config::cron::SimpleCron;
use crate::Core;
#[derive(Clone)]
pub struct Enterprise {
pub license: LicenseKey,
pub undelete_period: Option<Duration>,
pub trace_hold_period: Option<Duration>,
pub trace_store: Option<Store>,
pub undelete: Option<Undelete>,
pub trace_store: Option<TraceStore>,
pub metrics_store: Option<MetricsStore>,
}
#[derive(Clone)]
pub struct Undelete {
pub retention: Duration,
}
#[derive(Clone)]
pub struct TraceStore {
pub retention: Option<Duration>,
pub store: Store,
}
#[derive(Clone)]
pub struct MetricsStore {
pub retention: Option<Duration>,
pub store: Store,
pub interval: SimpleCron,
}
impl Core {

4
crates/common/src/enterprise/undelete.rs
View File

@@ -40,13 +40,13 @@ impl Core {
blob_hash: &BlobHash,
blob_size: usize,
) {
if let Some(hold_period) = self.enterprise.as_ref().and_then(|e| e.undelete_period) {
if let Some(undelete) = self.enterprise.as_ref().and_then(|e| e.undelete.as_ref()) {
let now = now();
batch.set(
BlobOp::Reserve {
hash: blob_hash.clone(),
until: now + hold_period.as_secs(),
until: now + undelete.retention.as_secs(),
},
KeySerializer::new(U64_LEN + U64_LEN)
.write(blob_size as u32)

26
crates/common/src/lib.rs
View File

@@ -28,7 +28,10 @@ use listener::{
use mail_send::Credentials;
use sieve::Sieve;
use store::LookupStore;
use store::{
write::{QueueClass, ValueClass},
IterateParams, LookupStore, ValueKey,
};
use tokio::sync::{mpsc, oneshot};
use utils::BlobHash;
@@ -65,6 +68,7 @@ pub struct Core {
#[derive(Clone)]
pub struct Network {
pub node_id: u64,
pub blocked_ips: BlockedIps,
pub allowed_ips: AllowedIps,
pub http_response_url: IfBlock,
@@ -303,6 +307,26 @@ impl Core {
.ctx(trc::Key::AccountName, credentials.login().to_string()))
}
}
pub async fn message_queue_size(&self) -> trc::Result<u64> {
let mut total = 0;
self.storage
.data
.iterate(
IterateParams::new(
ValueKey::from(ValueClass::Queue(QueueClass::Message(0))),
ValueKey::from(ValueClass::Queue(QueueClass::Message(u64::MAX))),
)
.no_values(),
|_, _| {
total += 1;
Ok(true)
},
)
.await
.map(|_| total)
}
}
trait CredentialsUsername {

3
crates/common/src/telemetry/metrics/mod.rs
View File

@@ -6,3 +6,6 @@
pub mod otel;
pub mod prometheus;
#[cfg(feature = "enterprise")]
pub mod store;

36
crates/common/src/telemetry/metrics/otel.rs
View File

@@ -29,28 +29,8 @@ impl OtelMetrics {
// Add counters
for counter in Collector::collect_counters(is_enterprise) {
metrics.push(Metric {
name: counter.id().into(),
description: counter.description().into(),
unit: counter.unit().into(),
data: Box::new(Sum {
data_points: vec![DataPoint {
attributes: vec![],
start_time: start_time.into(),
time: now.into(),
value: counter.get(),
exemplars: vec![],
}],
temporality: Temporality::Cumulative,
is_monotonic: true,
}),
});
}
// Add event counters
for counter in Collector::collect_event_counters(is_enterprise) {
metrics.push(Metric {
name: counter.id().into(),
description: counter.description().into(),
name: counter.id().name().into(),
description: counter.id().description().into(),
unit: "events".into(),
data: Box::new(Sum {
data_points: vec![DataPoint {
@@ -69,9 +49,9 @@ impl OtelMetrics {
// Add gauges
for gauge in Collector::collect_gauges(is_enterprise) {
metrics.push(Metric {
name: gauge.id().into(),
description: gauge.description().into(),
unit: gauge.unit().into(),
name: gauge.id().name().into(),
description: gauge.id().description().into(),
unit: gauge.id().unit().into(),
data: Box::new(Gauge {
data_points: vec![DataPoint {
attributes: vec![],
@@ -87,9 +67,9 @@ impl OtelMetrics {
// Add histograms
for histogram in Collector::collect_histograms(is_enterprise) {
metrics.push(Metric {
name: histogram.id().into(),
description: histogram.description().into(),
unit: histogram.unit().into(),
name: histogram.id().name().into(),
description: histogram.id().description().into(),
unit: histogram.id().unit().into(),
data: Box::new(Histogram {
data_points: vec![HistogramDataPoint {
attributes: vec![],

22
crates/common/src/telemetry/metrics/prometheus.rs
View File

@@ -25,18 +25,8 @@ impl Core {
// Add counters
for counter in Collector::collect_counters(is_enterprise) {
let mut metric = MetricFamily::default();
metric.set_name(metric_name(counter.id()));
metric.set_help(counter.description().into());
metric.set_field_type(MetricType::COUNTER);
metric.set_metric(vec![new_counter(counter.get())]);
metrics.push(metric);
}
// Add event counters
for counter in Collector::collect_event_counters(is_enterprise) {
let mut metric = MetricFamily::default();
metric.set_name(metric_name(counter.id()));
metric.set_help(counter.description().into());
metric.set_name(metric_name(counter.id().name()));
metric.set_help(counter.id().description().into());
metric.set_field_type(MetricType::COUNTER);
metric.set_metric(vec![new_counter(counter.value())]);
metrics.push(metric);
@@ -45,8 +35,8 @@ impl Core {
// Add gauges
for gauge in Collector::collect_gauges(is_enterprise) {
let mut metric = MetricFamily::default();
metric.set_name(metric_name(gauge.id()));
metric.set_help(gauge.description().into());
metric.set_name(metric_name(gauge.id().name()));
metric.set_help(gauge.id().description().into());
metric.set_field_type(MetricType::GAUGE);
metric.set_metric(vec![new_gauge(gauge.get())]);
metrics.push(metric);
@@ -55,8 +45,8 @@ impl Core {
// Add histograms
for histogram in Collector::collect_histograms(is_enterprise) {
let mut metric = MetricFamily::default();
metric.set_name(metric_name(histogram.id()));
metric.set_help(histogram.description().into());
metric.set_name(metric_name(histogram.id().name()));
metric.set_help(histogram.id().description().into());
metric.set_field_type(MetricType::HISTOGRAM);
metric.set_metric(vec![new_histogram(histogram)]);
metrics.push(metric);

257
crates/common/src/telemetry/metrics/store.rs Normal file
View File

@@ -0,0 +1,257 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
*
* SPDX-License-Identifier: LicenseRef-SEL
*
* This file is subject to the Stalwart Enterprise License Agreement (SEL) and
* is NOT open source software.
*
*/
use std::{future::Future, sync::Arc, time::Duration};
use ahash::AHashMap;
use parking_lot::Mutex;
use serde::{Deserialize, Serialize};
use store::{
write::{
key::{DeserializeBigEndian, KeySerializer},
now, BatchBuilder, TelemetryClass, ValueClass,
},
IterateParams, Store, ValueKey, U32_LEN, U64_LEN,
};
use trc::*;
use utils::codec::leb128::Leb128Reader;
use crate::Core;
pub trait MetricsStore: Sync + Send {
fn write_metrics(
&self,
core: Arc<Core>,
history: SharedMetricHistory,
) -> impl Future<Output = trc::Result<()>> + Send;
fn query_metrics(
&self,
from_timestamp: u64,
to_timestamp: u64,
) -> impl Future<Output = trc::Result<Vec<Metric<EventType, MetricType, u64>>>> + Send;
fn purge_metrics(&self, period: Duration) -> impl Future<Output = trc::Result<()>> + Send;
}
#[derive(Default)]
pub struct MetricsHistory {
events: AHashMap<EventType, u32>,
histograms: AHashMap<MetricType, HistogramHistory>,
}
#[derive(Default)]
struct HistogramHistory {
sum: u64,
count: u64,
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(tag = "type")]
#[serde(rename_all = "snake_case")]
pub enum Metric<CI, MI, T> {
Counter {
id: CI,
timestamp: T,
value: u32,
},
Histogram {
id: MI,
timestamp: T,
count: u64,
sum: u64,
},
}
pub type SharedMetricHistory = Arc<Mutex<MetricsHistory>>;
const TYPE_COUNTER: u64 = 0x00;
const TYPE_HISTOGRAM: u64 = 0x01;
impl MetricsStore for Store {
async fn write_metrics(
&self,
core: Arc<Core>,
history_: SharedMetricHistory,
) -> trc::Result<()> {
let mut batch = BatchBuilder::new();
{
let timestamp = now();
let node_id = core.network.node_id;
let mut history = history_.lock();
for event in [
EventType::Smtp(SmtpEvent::ConnectionStart),
EventType::Imap(ImapEvent::ConnectionStart),
EventType::Pop3(Pop3Event::ConnectionStart),
EventType::ManageSieve(ManageSieveEvent::ConnectionStart),
EventType::Http(HttpEvent::ConnectionStart),
EventType::Delivery(DeliveryEvent::AttemptStart),
EventType::Delivery(DeliveryEvent::Completed),
EventType::MessageIngest(MessageIngestEvent::Ham),
EventType::MessageIngest(MessageIngestEvent::Spam),
EventType::Network(NetworkEvent::DropBlocked),
EventType::IncomingReport(IncomingReportEvent::DmarcReport),
EventType::IncomingReport(IncomingReportEvent::DmarcReportWithWarnings),
EventType::IncomingReport(IncomingReportEvent::TlsReport),
EventType::IncomingReport(IncomingReportEvent::TlsReportWithWarnings),
] {
let reading = Collector::read_event_metric(event.id());
if reading > 0 {
let history = history.events.entry(event).or_insert(0);
let diff = reading - *history;
if diff > 0 {
batch.set(
ValueClass::Telemetry(TelemetryClass::Metric {
timestamp,
metric_id: (event.code() << 2) | TYPE_COUNTER,
node_id,
}),
KeySerializer::new(U32_LEN).write_leb128(diff).finalize(),
);
}
*history = reading;
}
}
for histogram in Collector::collect_histograms(true) {
let histogram_id = histogram.id();
if matches!(
histogram_id,
MetricType::MessageIngestionTime
| MetricType::MessageFtsIndexTime
| MetricType::DeliveryTotalTime
| MetricType::DeliveryTime
| MetricType::DnsLookupTime
) {
let history = history.histograms.entry(histogram_id).or_default();
let sum = histogram.sum();
let count = histogram.count();
let diff_sum = sum - history.sum;
let diff_count = count - history.count;
if diff_sum > 0 || diff_count > 0 {
batch.set(
ValueClass::Telemetry(TelemetryClass::Metric {
timestamp,
metric_id: (histogram_id.code() << 2) | TYPE_HISTOGRAM,
node_id,
}),
KeySerializer::new(U32_LEN)
.write_leb128(diff_count)
.write_leb128(diff_sum)
.finalize(),
);
}
history.sum = sum;
history.count = count;
}
}
}
if !batch.is_empty() {
self.write(batch.build())
.await
.caused_by(trc::location!())?;
}
Ok(())
}
async fn query_metrics(
&self,
from_timestamp: u64,
to_timestamp: u64,
) -> trc::Result<Vec<Metric<EventType, MetricType, u64>>> {
let mut metrics = Vec::new();
self.iterate(
IterateParams::new(
ValueKey::from(ValueClass::Telemetry(TelemetryClass::Metric {
timestamp: from_timestamp,
metric_id: 0,
node_id: 0,
})),
ValueKey::from(ValueClass::Telemetry(TelemetryClass::Metric {
timestamp: to_timestamp,
metric_id: 0,
node_id: 0,
})),
),
|key, value| {
let timestamp = key.deserialize_be_u64(0).caused_by(trc::location!())?;
let (metric_type, _) = key
.get(U64_LEN..)
.and_then(|bytes| bytes.read_leb128::<u64>())
.ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?;
match metric_type & 0x03 {
TYPE_COUNTER => {
let id = EventType::from_code(metric_type >> 2).ok_or_else(|| {
trc::Error::corrupted_key(key, None, trc::location!())
})?;
let (value, _) = value.read_leb128::<u32>().ok_or_else(|| {
trc::Error::corrupted_key(key, value.into(), trc::location!())
})?;
metrics.push(Metric::Counter {
id,
timestamp,
value,
});
}
TYPE_HISTOGRAM => {
let id = MetricType::from_code(metric_type >> 2).ok_or_else(|| {
trc::Error::corrupted_key(key, None, trc::location!())
})?;
let (count, bytes_read) = value.read_leb128::<u64>().ok_or_else(|| {
trc::Error::corrupted_key(key, value.into(), trc::location!())
})?;
let (sum, _) = value
.get(bytes_read..)
.and_then(|bytes| bytes.read_leb128::<u64>())
.ok_or_else(|| {
trc::Error::corrupted_key(key, value.into(), trc::location!())
})?;
metrics.push(Metric::Histogram {
id,
timestamp,
count,
sum,
});
}
_ => return Err(trc::Error::corrupted_key(key, None, trc::location!())),
}
Ok(true)
},
)
.await
.caused_by(trc::location!())?;
Ok(metrics)
}
async fn purge_metrics(&self, period: Duration) -> trc::Result<()> {
self.delete_range(
ValueKey::from(ValueClass::Telemetry(TelemetryClass::Metric {
timestamp: 0,
metric_id: 0,
node_id: 0,
})),
ValueKey::from(ValueClass::Telemetry(TelemetryClass::Metric {
timestamp: now() - period.as_secs(),
metric_id: 0,
node_id: 0,
})),
)
.await
.caused_by(trc::location!())
}
}
impl MetricsHistory {
pub fn init() -> SharedMetricHistory {
Arc::new(Mutex::new(Self::default()))
}
}

36
crates/common/src/telemetry/tracers/store.rs
View File

@@ -12,7 +12,7 @@ use std::{future::Future, time::Duration};
use ahash::{AHashMap, AHashSet};
use store::{
write::{key::DeserializeBigEndian, BatchBuilder, MaybeDynamicId, TraceClass, ValueClass},
write::{key::DeserializeBigEndian, BatchBuilder, MaybeDynamicId, TelemetryClass, ValueClass},
Deserialize, IterateParams, Store, ValueKey, U64_LEN,
};
use trc::{
@@ -81,7 +81,7 @@ pub(crate) fn spawn_store_tracer(builder: SubscriberBuilder, settings: StoreTrac
if !queue_ids.is_empty() {
// Serialize events
batch.set(
ValueClass::Trace(TraceClass::Span { span_id }),
ValueClass::Telemetry(TelemetryClass::Span { span_id }),
serialize_events(
[span.as_ref()]
.into_iter()
@@ -93,7 +93,7 @@ pub(crate) fn spawn_store_tracer(builder: SubscriberBuilder, settings: StoreTrac
// Build index
batch.set(
ValueClass::Trace(TraceClass::Index {
ValueClass::Telemetry(TelemetryClass::Index {
span_id,
value: (span.inner.typ.code() as u16).to_be_bytes().to_vec(),
}),
@@ -101,7 +101,7 @@ pub(crate) fn spawn_store_tracer(builder: SubscriberBuilder, settings: StoreTrac
);
for queue_id in queue_ids {
batch.set(
ValueClass::Trace(TraceClass::Index {
ValueClass::Telemetry(TelemetryClass::Index {
span_id,
value: queue_id.to_be_bytes().to_vec(),
}),
@@ -110,7 +110,7 @@ pub(crate) fn spawn_store_tracer(builder: SubscriberBuilder, settings: StoreTrac
}
for value in values {
batch.set(
ValueClass::Trace(TraceClass::Index {
ValueClass::Telemetry(TelemetryClass::Index {
span_id,
value: value.into_bytes(),
}),
@@ -158,18 +158,18 @@ pub trait TracingStore: Sync + Send {
impl TracingStore for Store {
async fn get_span(&self, span_id: u64) -> trc::Result<Vec<Event<EventDetails>>> {
self.get_value::<Span>(ValueKey::from(ValueClass::Trace(TraceClass::Span {
span_id,
})))
self.get_value::<Span>(ValueKey::from(ValueClass::Telemetry(
TelemetryClass::Span { span_id },
)))
.await
.caused_by(trc::location!())
.map(|span| span.map(|span| span.0).unwrap_or_default())
}
async fn get_raw_span(&self, span_id: u64) -> trc::Result<Option<Vec<u8>>> {
self.get_value::<RawSpan>(ValueKey::from(ValueClass::Trace(TraceClass::Span {
span_id,
})))
self.get_value::<RawSpan>(ValueKey::from(ValueClass::Telemetry(
TelemetryClass::Span { span_id },
)))
.await
.caused_by(trc::location!())
.map(|span| span.map(|span| span.0))
@@ -206,11 +206,11 @@ impl TracingStore for Store {
let mut param_spans = SpanCollector::new(num_params);
self.iterate(
IterateParams::new(
ValueKey::from(ValueClass::Trace(TraceClass::Index {
ValueKey::from(ValueClass::Telemetry(TelemetryClass::Index {
span_id: 0,
value: value.clone(),
})),
ValueKey::from(ValueClass::Trace(TraceClass::Index {
ValueKey::from(ValueClass::Telemetry(TelemetryClass::Index {
span_id: u64::MAX,
value,
})),
@@ -253,8 +253,8 @@ impl TracingStore for Store {
})?;
self.delete_range(
ValueKey::from(ValueClass::Trace(TraceClass::Span { span_id: 0 })),
ValueKey::from(ValueClass::Trace(TraceClass::Span {
ValueKey::from(ValueClass::Telemetry(TelemetryClass::Span { span_id: 0 })),
ValueKey::from(ValueClass::Telemetry(TelemetryClass::Span {
span_id: until_span_id,
})),
)
@@ -264,11 +264,11 @@ impl TracingStore for Store {
let mut delete_keys: Vec<ValueClass<MaybeDynamicId>> = Vec::new();
self.iterate(
IterateParams::new(
ValueKey::from(ValueClass::Trace(TraceClass::Index {
ValueKey::from(ValueClass::Telemetry(TelemetryClass::Index {
span_id: 0,
value: vec![],
})),
ValueKey::from(ValueClass::Trace(TraceClass::Index {
ValueKey::from(ValueClass::Telemetry(TelemetryClass::Index {
span_id: u64::MAX,
value: vec![u8::MAX; 16],
})),
@@ -279,7 +279,7 @@ impl TracingStore for Store {
.deserialize_be_u64(key.len() - U64_LEN)
.caused_by(trc::location!())?;
if span_id < until_span_id {
delete_keys.push(ValueClass::Trace(TraceClass::Index {
delete_keys.push(ValueClass::Telemetry(TelemetryClass::Index {
span_id,
value: key[0..key.len() - U64_LEN].to_vec(),
}));