filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Metric history + Live metrics

Browse Source
This commit is contained in:
mdecimus
2024-08-23 16:21:35 +02:00
parent 0aaf493f94
commit dcc31e8b3e
34 changed files with 1256 additions and 512 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crates/jmap/Cargo.toml
View File

@@ -57,7 +57,7 @@ lz4_flex = { version = "0.11", default-features = false }
rev_lines = "0.3.0"
x509-parser = "0.16.0"
quick-xml = "0.36"
memory-stats = "1.2.0"
[features]
test_mode = []

20
crates/jmap/src/api/http.rs
View File

@@ -297,16 +297,26 @@ impl JMAP {
if err.matches(trc::EventType::Auth(trc::AuthEvent::Failed))
&& self.core.is_enterprise_edition()
{
if let Some(token) =
req.uri().path().strip_prefix("/api/tracing/live/")
if let Some((live_path, token)) = req
.uri()
.path()
.strip_prefix("/api/telemetry/")
.and_then(|p| {
p.strip_suffix("traces/live/")
.map(|t| ("traces", t))
.or_else(|| {
p.strip_suffix("metrics/live/")
.map(|t| ("metrics", t))
})
})
{
let (account_id, _, _) =
self.validate_access_token("live_tracing", token).await?;
self.validate_access_token("live_telemetry", token).await?;
return self
.handle_tracing_api_request(
.handle_telemetry_api_request(
&req,
vec!["", "live"],
vec!["", live_path, "live"],
account_id,
)
.await;

2
crates/jmap/src/api/management/enterprise/mod.rs
View File

@@ -8,5 +8,5 @@
*
*/
pub mod tracing;
pub mod telemetry;
pub mod undelete;

252
crates/jmap/src/api/management/enterprise/tracing.rs → crates/jmap/src/api/management/enterprise/telemetry.rs
View File

@@ -8,9 +8,15 @@
*
*/
use std::time::{Duration, Instant};
use std::{
fmt::Write,
time::{Duration, Instant},
};
use common::telemetry::tracers::store::{TracingQuery, TracingStore};
use common::telemetry::{
metrics::store::{Metric, MetricsStore},
tracers::store::{TracingQuery, TracingStore},
};
use directory::backend::internal::manage;
use http_body_util::{combinators::BoxBody, StreamBody};
use hyper::{
@@ -23,7 +29,7 @@ use store::ahash::{AHashMap, AHashSet};
use trc::{
ipc::{bitset::Bitset, subscriber::SubscriberBuilder},
serializers::json::JsonEventSerializer,
DeliveryEvent, EventType, Key, QueueEvent, Value,
Collector, DeliveryEvent, EventType, Key, MetricType, QueueEvent, Value,
};
use utils::{snowflake::SnowflakeIdGenerator, url_params::UrlParams};
@@ -36,7 +42,7 @@ use crate::{
};
impl JMAP {
pub async fn handle_tracing_api_request(
pub async fn handle_telemetry_api_request(
&self,
req: &HttpRequest,
path: Vec<&str>,
@@ -49,7 +55,7 @@ impl JMAP {
path.get(2).copied(),
req.method(),
) {
("spans", None, &Method::GET) => {
("traces", None, &Method::GET) => {
let page: usize = params.parse("page").unwrap_or(0);
let limit: usize = params.parse("limit").unwrap_or(0);
let mut tracing_query = Vec::new();
@@ -100,12 +106,13 @@ impl JMAP {
.and_then(SnowflakeIdGenerator::from_timestamp)
.unwrap_or(0);
let values = params.get("values").is_some();
let store = self
let store = &self
.core
.enterprise
.as_ref()
.and_then(|e| e.trace_store.as_ref())
.ok_or_else(|| manage::unsupported("No tracing store has been configured"))?;
.ok_or_else(|| manage::unsupported("No tracing store has been configured"))?
.store;
let span_ids = store.query_spans(&tracing_query, after, before).await?;
let (total, span_ids) = if limit > 0 {
@@ -154,52 +161,7 @@ impl JMAP {
.into_http_response())
}
}
("span", id, &Method::GET) => {
let store = self
.core
.enterprise
.as_ref()
.and_then(|e| e.trace_store.as_ref())
.ok_or_else(|| manage::unsupported("No tracing store has been configured"))?;
let mut events = Vec::new();
for span_id in id
.or_else(|| params.get("id"))
.unwrap_or_default()
.split(',')
{
if let Ok(span_id) = span_id.parse::<u64>() {
events.push(
JsonEventSerializer::new(store.get_span(span_id).await?)
.with_description()
.with_explanation(),
);
} else {
events.push(JsonEventSerializer::new(Vec::new()));
}
}
if events.len() == 1 && id.is_some() {
Ok(JsonResponse::new(json!({
"data": events.into_iter().next().unwrap(),
}))
.into_http_response())
} else {
Ok(JsonResponse::new(json!({
"data": events,
}))
.into_http_response())
}
}
("live", Some("token"), &Method::GET) => {
// Issue a live tracing token valid for 60 seconds
Ok(JsonResponse::new(json!({
"data": self.issue_custom_token(account_id, "live_tracing", "web", 60).await?,
}))
.into_http_response())
}
("live", _, &Method::GET) => {
("traces", Some("live"), &Method::GET) => {
let mut key_filters = AHashMap::new();
let mut filter = None;
@@ -327,6 +289,190 @@ impl JMAP {
))),
})
}
("trace", id, &Method::GET) => {
let store = &self
.core
.enterprise
.as_ref()
.and_then(|e| e.trace_store.as_ref())
.ok_or_else(|| manage::unsupported("No tracing store has been configured"))?
.store;
let mut events = Vec::new();
for span_id in id
.or_else(|| params.get("id"))
.unwrap_or_default()
.split(',')
{
if let Ok(span_id) = span_id.parse::<u64>() {
events.push(
JsonEventSerializer::new(store.get_span(span_id).await?)
.with_description()
.with_explanation(),
);
} else {
events.push(JsonEventSerializer::new(Vec::new()));
}
}
if events.len() == 1 && id.is_some() {
Ok(JsonResponse::new(json!({
"data": events.into_iter().next().unwrap(),
}))
.into_http_response())
} else {
Ok(JsonResponse::new(json!({
"data": events,
}))
.into_http_response())
}
}
("live", Some("token"), &Method::GET) => {
// Issue a live telemetry token valid for 60 seconds
Ok(JsonResponse::new(json!({
"data": self.issue_custom_token(account_id, "live_telemetry", "web", 60).await?,
}))
.into_http_response())
}
("metrics", None, &Method::GET) => {
let before = params
.parse::<Timestamp>("before")
.map(|t| t.into_inner())
.unwrap_or(0);
let after = params
.parse::<Timestamp>("after")
.map(|t| t.into_inner())
.unwrap_or(0);
let results = self
.core
.enterprise
.as_ref()
.and_then(|e| e.metrics_store.as_ref())
.ok_or_else(|| manage::unsupported("No metrics store has been configured"))?
.store
.query_metrics(after, before)
.await?;
let mut metrics = Vec::with_capacity(results.len());
for metric in results {
metrics.push(match metric {
Metric::Counter {
id,
timestamp,
value,
} => Metric::Counter {
id: id.name(),
timestamp: DateTime::from_timestamp(timestamp as i64).to_rfc3339(),
value,
},
Metric::Histogram {
id,
timestamp,
count,
sum,
} => Metric::Histogram {
id: id.name(),
timestamp: DateTime::from_timestamp(timestamp as i64).to_rfc3339(),
count,
sum,
},
});
}
Ok(JsonResponse::new(json!({
"data": metrics,
}))
.into_http_response())
}
("metrics", Some("live"), &Method::GET) => {
let interval = Duration::from_secs(
params
.parse::<u64>("interval")
.filter(|interval| *interval >= 1)
.unwrap_or(30),
);
let mut event_types = AHashSet::new();
let mut metric_types = AHashSet::new();
for metric_name in params.get("metrics").unwrap_or_default().split(',') {
let metric_name = metric_name.trim();
if !metric_name.is_empty() {
if let Some(event_type) = EventType::try_parse(metric_name) {
event_types.insert(event_type);
} else if let Some(metric_type) = MetricType::try_parse(metric_name) {
metric_types.insert(metric_type);
}
}
}
Ok(HttpResponse {
status: StatusCode::OK,
content_type: "text/event-stream".into(),
content_disposition: "".into(),
cache_control: "no-store".into(),
body: HttpResponseBody::Stream(BoxBody::new(StreamBody::new(
async_stream::stream! {
loop {
let mut metrics = String::with_capacity(512);
metrics.push_str("event: metrics\ndata: [");
let mut is_first = true;
for counter in Collector::collect_counters(true) {
if event_types.is_empty() || event_types.contains(&counter.id()) {
if !is_first {
metrics.push(',');
} else {
is_first = false;
}
let _ = write!(
&mut metrics,
"{{\"id\":\"{}\",\"type\":\"counter\",\"value\":{}}}",
counter.id().name(),
counter.value()
);
}
}
for gauge in Collector::collect_gauges(true) {
if metric_types.is_empty() || metric_types.contains(&gauge.id()) {
if !is_first {
metrics.push(',');
} else {
is_first = false;
}
let _ = write!(
&mut metrics,
"{{\"id\":\"{}\",\"type\":\"gauge\",\"value\":{}}}",
gauge.id().name(),
gauge.get()
);
}
}
for histogram in Collector::collect_histograms(true) {
if metric_types.is_empty() || metric_types.contains(&histogram.id()) {
if !is_first {
metrics.push(',');
} else {
is_first = false;
}
let _ = write!(
&mut metrics,
"{{\"id\":\"{}\",\"type\":\"histogram\",\"count\":{},\"sum\":{}}}",
histogram.id().name(),
histogram.count(),
histogram.sum()
);
}
}
metrics.push_str("]\n\n");
yield Ok(Frame::data(Bytes::from(metrics)));
tokio::time::sleep(interval).await;
}
},
))),
})
}
_ => Err(trc::ResourceEvent::NotFound.into_err()),
}
}

4
crates/jmap/src/api/management/mod.rs
View File

@@ -83,7 +83,7 @@ impl JMAP {
// SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
// SPDX-License-Identifier: LicenseRef-SEL
#[cfg(feature = "enterprise")]
"tracing" if is_superuser => {
"telemetry" if is_superuser => {
// WARNING: TAMPERING WITH THIS FUNCTION IS STRICTLY PROHIBITED
// Any attempt to modify, bypass, or disable this license validation mechanism
// constitutes a severe violation of the Stalwart Enterprise License Agreement.
@@ -94,7 +94,7 @@ impl JMAP {
// for copyright infringement, breach of contract, and fraud.
if self.core.is_enterprise_edition() {
self.handle_tracing_api_request(req, path, access_token.primary_id())
self.handle_telemetry_api_request(req, path, access_token.primary_id())
.await
} else {
Err(manage::enterprise())

121
crates/jmap/src/services/housekeeper.rs
View File

@@ -12,14 +12,17 @@ use std::{
use common::{config::telemetry::OtelMetrics, IPC_CHANNEL_BUFFER};
#[cfg(feature = "enterprise")]
use common::telemetry::tracers::store::TracingStore;
use common::telemetry::{
metrics::store::{MetricsStore, SharedMetricHistory},
tracers::store::TracingStore,
};
use store::{
write::{now, purge::PurgeStore},
BlobStore, LookupStore, Store,
};
use tokio::sync::mpsc;
use trc::HousekeeperEvent;
use trc::{Collector, HousekeeperEvent, MetricType};
use utils::map::ttl_dashmap::TtlMap;
use crate::{Inner, JmapInstance, JMAP, LONG_SLUMBER};
@@ -55,6 +58,9 @@ enum ActionClass {
Acme(String),
OtelMetrics,
#[cfg(feature = "enterprise")]
InternalMetrics,
CalculateMetrics,
#[cfg(feature = "enterprise")]
ReloadSettings,
}
@@ -99,6 +105,9 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) {
queue.schedule(Instant::now() + otel.interval, ActionClass::OtelMetrics);
}
// Calculate expensive metrics
queue.schedule(Instant::now(), ActionClass::CalculateMetrics);
// Add all ACME renewals to heap
for provider in core_.tls.acme_providers.values() {
match core_.init_acme(provider).await {
@@ -125,10 +134,21 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) {
Instant::now() + enterprise.license.expires_in(),
ActionClass::ReloadSettings,
);
if let Some(metrics_store) = enterprise.metrics_store.as_ref() {
queue.schedule(
Instant::now() + metrics_store.interval.time_to_next(),
ActionClass::InternalMetrics,
);
}
}
// SPDX-SnippetEnd
}
// Metrics history
#[cfg(feature = "enterprise")]
let metrics_history = SharedMetricHistory::default();
loop {
match tokio::time::timeout(queue.wake_up_time(), rx.recv()).await {
Ok(Some(event)) => match event {
@@ -185,12 +205,21 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) {
// SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
// SPDX-License-Identifier: LicenseRef-SEL
#[cfg(feature = "enterprise")]
let trace_hold_period = core
let trace_retention = core
.core
.load()
.enterprise
.as_ref()
.and_then(|e| e.trace_hold_period);
.and_then(|e| e.trace_store.as_ref())
.and_then(|t| t.retention);
#[cfg(feature = "enterprise")]
let metrics_retention = core
.core
.load()
.enterprise
.as_ref()
.and_then(|e| e.metrics_store.as_ref())
.and_then(|m| m.retention);
// SPDX-SnippetEnd
tokio::spawn(async move {
@@ -206,11 +235,18 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) {
// SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
// SPDX-License-Identifier: LicenseRef-SEL
#[cfg(feature = "enterprise")]
if let Some(trace_hold_period) = trace_hold_period {
if let Err(err) = store.purge_spans(trace_hold_period).await {
if let Some(trace_retention) = trace_retention {
if let Err(err) = store.purge_spans(trace_retention).await {
trc::error!(err.details("Failed to purge tracing spans"));
}
}
#[cfg(feature = "enterprise")]
if let Some(metrics_retention) = metrics_retention {
if let Err(err) = store.purge_metrics(metrics_retention).await {
trc::error!(err.details("Failed to purge metrics"));
}
}
// SPDX-SnippetEnd
});
}
@@ -382,10 +418,83 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) {
});
}
}
ActionClass::CalculateMetrics => {
// Calculate expensive metrics every 5 minutes
queue.schedule(
Instant::now() + Duration::from_secs(5 * 60),
ActionClass::OtelMetrics,
);
let core = core_.clone();
tokio::spawn(async move {
#[cfg(feature = "enterprise")]
if core.is_enterprise_edition() {
// Obtain queue size
match core.message_queue_size().await {
Ok(total) => {
Collector::update_gauge(
MetricType::QueueCount,
total,
);
}
Err(err) => {
trc::error!(
err.details("Failed to obtain queue size")
);
}
}
}
match tokio::task::spawn_blocking(memory_stats::memory_stats)
.await
{
Ok(Some(stats)) => {
Collector::update_gauge(
MetricType::ServerMemory,
stats.physical_mem as u64,
);
}
Ok(None) => {}
Err(err) => {
trc::error!(trc::EventType::Server(
trc::ServerEvent::ThreadError,
)
.reason(err)
.caused_by(trc::location!())
.details("Join Error"));
}
}
});
}
// SPDX-SnippetBegin
// SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
// SPDX-License-Identifier: LicenseRef-SEL
#[cfg(feature = "enterprise")]
ActionClass::InternalMetrics => {
if let Some(metrics_store) = &core_
.enterprise
.as_ref()
.and_then(|e| e.metrics_store.as_ref())
{
queue.schedule(
Instant::now() + metrics_store.interval.time_to_next(),
ActionClass::InternalMetrics,
);
let metrics_store = metrics_store.store.clone();
let metrics_history = metrics_history.clone();
let core = core_.clone();
tokio::spawn(async move {
if let Err(err) =
metrics_store.write_metrics(core, metrics_history).await
{
trc::error!(err.details("Failed to write metrics"));
}
});
}
}
#[cfg(feature = "enterprise")]
ActionClass::ReloadSettings => {
match core_.reload().await {