filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Improved tracing (part 4)

Browse Source
This commit is contained in:
mdecimus
2024-07-28 15:21:22 +02:00
parent 1e76792d03
commit a45eb50231
102 changed files with 2634 additions and 1539 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
crates/trc/src/atomic.rs Normal file
View File

@@ -0,0 +1,118 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use std::sync::atomic::{AtomicUsize, Ordering};
pub struct AtomicBitset<const N: usize>([AtomicUsize; N]);
const USIZE_BITS: usize = std::mem::size_of::<usize>() * 8;
const USIZE_BITS_MASK: usize = USIZE_BITS - 1;
impl<const N: usize> AtomicBitset<N> {
#[allow(clippy::new_without_default)]
#[allow(clippy::declare_interior_mutable_const)]
pub const fn new() -> Self {
Self({
const INIT: AtomicUsize = AtomicUsize::new(0);
let mut array = [INIT; N];
let mut i = 0;
while i < N {
array[i] = AtomicUsize::new(0);
i += 1;
}
array
})
}
#[inline(always)]
pub fn set(&self, index: impl Into<usize>) {
let index = index.into();
self.0[index / USIZE_BITS].fetch_or(1 << (index & USIZE_BITS_MASK), Ordering::Relaxed);
}
#[inline(always)]
pub fn clear(&self, index: impl Into<usize>) {
let index = index.into();
self.0[index / USIZE_BITS].fetch_and(!(1 << (index & USIZE_BITS_MASK)), Ordering::Relaxed);
}
#[inline(always)]
pub fn get(&self, index: impl Into<usize>) -> bool {
let index = index.into();
self.0[index / USIZE_BITS].load(Ordering::Relaxed) & (1 << (index & USIZE_BITS_MASK)) != 0
}
pub fn clear_all(&self) {
for i in 0..N {
self.0[i].store(0, Ordering::Relaxed);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
const TEST_SIZE: usize = 1000;
type TestBitset = AtomicBitset<{ (TEST_SIZE + USIZE_BITS - 1) / USIZE_BITS }>;
static BITSET: TestBitset = TestBitset::new();
#[test]
fn test_atomic_bitset() {
for i in 0..TEST_SIZE {
assert!(!BITSET.get(i), "Bit {} should be unset in new BITSET", i);
}
for i in 0..TEST_SIZE {
assert!(!BITSET.get(i), "Bit {} should be initially unset", i);
BITSET.set(i);
assert!(BITSET.get(i), "Bit {} should be set after setting", i);
}
BITSET.clear_all();
for i in 0..TEST_SIZE {
BITSET.set(i);
assert!(BITSET.get(i), "Bit {} should be set before clearing", i);
BITSET.clear(i);
assert!(!BITSET.get(i), "Bit {} should be unset after clearing", i);
}
BITSET.clear_all();
// Set even bits
for i in (0..TEST_SIZE).step_by(2) {
BITSET.set(i);
}
// Check all bits
for i in 0..TEST_SIZE {
if i % 2 == 0 {
assert!(BITSET.get(i), "Even bit {} should be set", i);
} else {
assert!(!BITSET.get(i), "Odd bit {} should be unset", i);
}
}
// Clear even bits and set odd bits
for i in 0..TEST_SIZE {
if i % 2 == 0 {
BITSET.clear(i);
} else {
BITSET.set(i);
}
}
// Check all bits again
for i in 0..TEST_SIZE {
if i % 2 == 0 {
assert!(!BITSET.get(i), "Even bit {} should now be unset", i);
} else {
assert!(BITSET.get(i), "Odd bit {} should now be set", i);
}
}
}
}

14
crates/trc/src/conv.rs
View File

@@ -6,6 +6,8 @@
use std::{borrow::Cow, fmt::Debug, time::Duration};
use mail_auth::common::headers::HeaderWriter;
use crate::*;
impl AsRef<EventType> for Error {
@@ -333,7 +335,17 @@ impl From<&mail_auth::DmarcResult> for Event {
impl From<&mail_auth::DkimOutput<'_>> for Event {
fn from(value: &mail_auth::DkimOutput<'_>) -> Self {
Event::from(value.result()).ctx_opt(Key::Contents, value.signature().map(|s| s.to_string()))
Event::from(value.result()).ctx_opt(
Key::Contents,
value.signature().map(|s| {
let mut buf = Vec::new();
s.write_header(&mut buf);
String::from_utf8(buf)
.map(Value::String)
.unwrap_or_else(|err| Value::Bytes(err.into_bytes()))
}),
)
}
}

167
crates/trc/src/imple.rs
View File

@@ -574,6 +574,14 @@ impl NetworkEvent {
}
impl Value {
pub fn from_maybe_string(value: &[u8]) -> Self {
if let Ok(value) = std::str::from_utf8(value) {
Self::String(value.to_string())
} else {
Self::Bytes(value.to_vec())
}
}
pub fn to_uint(&self) -> Option<u64> {
match self {
Self::UInt(value) => Some(*value),
@@ -748,26 +756,141 @@ impl EventType {
pub fn level(&self) -> Level {
match self {
EventType::Store(event) => match event {
StoreEvent::SqlQuery | StoreEvent::LdapQuery => Level::Trace,
StoreEvent::SqlQuery | StoreEvent::LdapQuery | StoreEvent::LdapBind => Level::Trace,
StoreEvent::NotFound => Level::Debug,
StoreEvent::Ingest => Level::Info,
_ => Level::Error,
StoreEvent::Ingest | StoreEvent::IngestDuplicate => Level::Info,
StoreEvent::IngestError
| StoreEvent::AssertValueFailed
| StoreEvent::FoundationDBError
| StoreEvent::MySQLError
| StoreEvent::PostgreSQLError
| StoreEvent::RocksDBError
| StoreEvent::SQLiteError
| StoreEvent::LdapError
| StoreEvent::ElasticSearchError
| StoreEvent::RedisError
| StoreEvent::S3Error
| StoreEvent::FilesystemError
| StoreEvent::PoolError
| StoreEvent::DataCorruption
| StoreEvent::DecompressError
| StoreEvent::DeserializeError
| StoreEvent::NotConfigured
| StoreEvent::NotSupported
| StoreEvent::UnexpectedError
| StoreEvent::CryptoError => Level::Error,
StoreEvent::BlobMissingMarker => Level::Warn,
},
EventType::Jmap(_) => Level::Debug,
EventType::Imap(event) => match event {
ImapEvent::Error | ImapEvent::IdleStart | ImapEvent::IdleStop => Level::Debug,
ImapEvent::GetAcl
| ImapEvent::SetAcl
| ImapEvent::MyRights
| ImapEvent::ListRights
| ImapEvent::Append
| ImapEvent::Capabilities
| ImapEvent::Id
| ImapEvent::Close
| ImapEvent::Copy
| ImapEvent::Move
| ImapEvent::CreateMailbox
| ImapEvent::DeleteMailbox
| ImapEvent::RenameMailbox
| ImapEvent::Enable
| ImapEvent::Expunge
| ImapEvent::Fetch
| ImapEvent::List
| ImapEvent::Lsub
| ImapEvent::Logout
| ImapEvent::Namespace
| ImapEvent::Noop
| ImapEvent::Search
| ImapEvent::Sort
| ImapEvent::Select
| ImapEvent::Status
| ImapEvent::Store
| ImapEvent::Subscribe
| ImapEvent::Unsubscribe
| ImapEvent::Thread
| ImapEvent::Error
| ImapEvent::IdleStart
| ImapEvent::IdleStop => Level::Debug,
ImapEvent::RawInput | ImapEvent::RawOutput => Level::Trace,
},
EventType::ManageSieve(event) => match event {
ManageSieveEvent::Error => Level::Debug,
ManageSieveEvent::CreateScript
| ManageSieveEvent::UpdateScript
| ManageSieveEvent::GetScript
| ManageSieveEvent::DeleteScript
| ManageSieveEvent::RenameScript
| ManageSieveEvent::CheckScript
| ManageSieveEvent::HaveSpace
| ManageSieveEvent::ListScripts
| ManageSieveEvent::SetActive
| ManageSieveEvent::Capabilities
| ManageSieveEvent::StartTls
| ManageSieveEvent::Unauthenticate
| ManageSieveEvent::Logout
| ManageSieveEvent::Noop
| ManageSieveEvent::Error => Level::Debug,
ManageSieveEvent::RawInput | ManageSieveEvent::RawOutput => Level::Trace,
},
EventType::Pop3(event) => match event {
Pop3Event::Error => Level::Debug,
Pop3Event::Delete
| Pop3Event::Reset
| Pop3Event::Quit
| Pop3Event::Fetch
| Pop3Event::List
| Pop3Event::ListMessage
| Pop3Event::Uidl
| Pop3Event::UidlMessage
| Pop3Event::Stat
| Pop3Event::Noop
| Pop3Event::Capabilities
| Pop3Event::StartTls
| Pop3Event::Utf8
| Pop3Event::Error => Level::Debug,
Pop3Event::RawInput | Pop3Event::RawOutput => Level::Trace,
},
EventType::Smtp(event) => match event {
SmtpEvent::PipeSuccess | SmtpEvent::PipeError | SmtpEvent::Error => Level::Debug,
SmtpEvent::DidNotSayEhlo
| SmtpEvent::EhloExpected
| SmtpEvent::LhloExpected
| SmtpEvent::MailFromUnauthenticated
| SmtpEvent::MailFromUnauthorized
| SmtpEvent::MailFromRewritten
| SmtpEvent::MailFromMissing
| SmtpEvent::MultipleMailFrom
| SmtpEvent::RcptToDuplicate
| SmtpEvent::RcptToRewritten
| SmtpEvent::RcptToMissing
| SmtpEvent::RequireTlsDisabled
| SmtpEvent::DeliverByDisabled
| SmtpEvent::DeliverByInvalid
| SmtpEvent::FutureReleaseDisabled
| SmtpEvent::FutureReleaseInvalid
| SmtpEvent::MtPriorityDisabled
| SmtpEvent::MtPriorityInvalid
| SmtpEvent::DsnDisabled
| SmtpEvent::AuthExchangeTooLong
| SmtpEvent::AlreadyAuthenticated
| SmtpEvent::Noop
| SmtpEvent::StartTls
| SmtpEvent::StartTlsUnavailable
| SmtpEvent::StartTlsAlready
| SmtpEvent::Rset
| SmtpEvent::Quit
| SmtpEvent::Help
| SmtpEvent::CommandNotImplemented
| SmtpEvent::InvalidCommand
| SmtpEvent::InvalidSenderAddress
| SmtpEvent::InvalidRecipientAddress
| SmtpEvent::InvalidParameter
| SmtpEvent::UnsupportedParameter
| SmtpEvent::SyntaxError
| SmtpEvent::PipeSuccess
| SmtpEvent::PipeError
| SmtpEvent::Error => Level::Debug,
SmtpEvent::MissingLocalHostname | SmtpEvent::RemoteIdNotFound => Level::Warn,
SmtpEvent::ConcurrencyLimitExceeded
| SmtpEvent::TransferLimitExceeded
@@ -789,7 +912,6 @@ impl EventType {
| SmtpEvent::DmarcFail
| SmtpEvent::IprevPass
| SmtpEvent::IprevFail
| SmtpEvent::QuotaExceeded
| SmtpEvent::TooManyMessages
| SmtpEvent::Ehlo
| SmtpEvent::InvalidEhlo
@@ -803,7 +925,11 @@ impl EventType {
| SmtpEvent::VrfyDisabled
| SmtpEvent::Expn
| SmtpEvent::ExpnNotFound
| SmtpEvent::ExpnDisabled => Level::Info,
| SmtpEvent::AuthNotAllowed
| SmtpEvent::AuthMechanismNotSupported
| SmtpEvent::ExpnDisabled
| SmtpEvent::RequestTooLarge
| SmtpEvent::TooManyRecipients => Level::Info,
SmtpEvent::RawInput | SmtpEvent::RawOutput => Level::Trace,
},
EventType::Network(event) => match event {
@@ -813,7 +939,7 @@ impl EventType {
| NetworkEvent::Closed => Level::Trace,
NetworkEvent::Timeout | NetworkEvent::AcceptError => Level::Debug,
NetworkEvent::ConnectionStart
| NetworkEvent::ConnectionStop
| NetworkEvent::ConnectionEnd
| NetworkEvent::ListenStart
| NetworkEvent::ListenStop
| NetworkEvent::DropBlocked => Level::Info,
@@ -1063,15 +1189,27 @@ impl EventType {
| DeliveryEvent::StartTlsError
| DeliveryEvent::StartTlsDisabled
| DeliveryEvent::ImplicitTlsError
| DeliveryEvent::TooManyConcurrent
| DeliveryEvent::DoubleBounce => Level::Info,
DeliveryEvent::MissingOutboundHostname => Level::Warn,
DeliveryEvent::ConcurrencyLimitExceeded
| DeliveryEvent::RateLimitExceeded
| DeliveryEvent::MissingOutboundHostname => Level::Warn,
DeliveryEvent::DsnSuccess
| DeliveryEvent::DsnTempFail
| DeliveryEvent::DsnPermFail => Level::Info,
DeliveryEvent::MxLookup
| DeliveryEvent::IpLookup
| DeliveryEvent::Ehlo
| DeliveryEvent::Auth
| DeliveryEvent::MailFrom
| DeliveryEvent::RcptTo => Level::Debug,
DeliveryEvent::RawInput | DeliveryEvent::RawOutput => Level::Trace,
},
EventType::Queue(event) => match event {
QueueEvent::RateLimitExceeded
| QueueEvent::ConcurrencyLimitExceeded
| QueueEvent::Scheduled
| QueueEvent::Rescheduled => Level::Info,
| QueueEvent::Rescheduled
| QueueEvent::QuotaExceeded => Level::Info,
QueueEvent::LockBusy | QueueEvent::Locked | QueueEvent::BlobNotFound => {
Level::Debug
}
@@ -1084,7 +1222,8 @@ impl EventType {
| MtaStsEvent::PolicyNotFound
| MtaStsEvent::PolicyFetchError
| MtaStsEvent::InvalidPolicy
| MtaStsEvent::NotAuthorized => Level::Info,
| MtaStsEvent::NotAuthorized
| MtaStsEvent::Authorized => Level::Info,
},
EventType::IncomingReport(event) => match event {
IncomingReportEvent::DmarcReportWithWarnings

152
crates/trc/src/lib.rs
View File

@@ -4,6 +4,7 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
pub mod atomic;
pub mod channel;
pub mod collector;
pub mod conv;
@@ -77,7 +78,9 @@ pub enum Key {
Property,
Path,
Url,
Used,
Name,
OldName,
DocumentId,
Collection,
AccountId,
@@ -96,6 +99,7 @@ pub enum Key {
Renewal,
Attempt,
NextRetry,
NextDsn,
LocalIp,
LocalPort,
RemoteIp,
@@ -138,6 +142,12 @@ pub enum Key {
TotalSuccesses,
TotalFailures,
Date,
Uid,
UidValidity,
UidNext,
SourceAccountId,
SourceMailboxId,
SourceUid,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -231,23 +241,94 @@ pub enum FtsIndexEvent {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ImapEvent {
Error,
RawInput,
RawOutput,
// Commands
GetAcl,
SetAcl,
MyRights,
ListRights,
Append,
Capabilities,
Id,
Close,
Copy,
Move,
CreateMailbox,
DeleteMailbox,
RenameMailbox,
Enable,
Expunge,
Fetch,
IdleStart,
IdleStop,
List,
Lsub,
Logout,
Namespace,
Noop,
Search,
Sort,
Select,
Status,
Store,
Subscribe,
Unsubscribe,
Thread,
// Errors
Error,
// Debugging
RawInput,
RawOutput,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Pop3Event {
// Commands
Delete,
Reset,
Quit,
Fetch,
List,
ListMessage,
Uidl,
UidlMessage,
Stat,
Noop,
Capabilities,
StartTls,
Utf8,
// Errors
Error,
// Debugging
RawInput,
RawOutput,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ManageSieveEvent {
// Commands
CreateScript,
UpdateScript,
GetScript,
DeleteScript,
RenameScript,
CheckScript,
HaveSpace,
ListScripts,
SetActive,
Capabilities,
StartTls,
Unauthenticate,
Logout,
Noop,
// Errors
Error,
// Debugging
RawInput,
RawOutput,
}
@@ -278,14 +359,25 @@ pub enum SmtpEvent {
DmarcFail,
IprevPass,
IprevFail,
QuotaExceeded,
TooManyMessages,
Ehlo,
InvalidEhlo,
DidNotSayEhlo,
EhloExpected,
LhloExpected,
MailFromUnauthenticated,
MailFromUnauthorized,
MailFromRewritten,
MailFromMissing,
MailFrom,
MultipleMailFrom,
MailboxDoesNotExist,
RelayNotAllowed,
RcptTo,
RcptToDuplicate,
RcptToRewritten,
RcptToMissing,
TooManyRecipients,
TooManyInvalidRcpt,
RawInput,
RawOutput,
@@ -296,6 +388,33 @@ pub enum SmtpEvent {
Expn,
ExpnNotFound,
ExpnDisabled,
RequireTlsDisabled,
DeliverByDisabled,
DeliverByInvalid,
FutureReleaseDisabled,
FutureReleaseInvalid,
MtPriorityDisabled,
MtPriorityInvalid,
DsnDisabled,
AuthNotAllowed,
AuthMechanismNotSupported,
AuthExchangeTooLong,
AlreadyAuthenticated,
Noop,
StartTls,
StartTlsUnavailable,
StartTlsAlready,
Rset,
Quit,
Help,
CommandNotImplemented,
InvalidCommand,
InvalidSenderAddress,
InvalidRecipientAddress,
InvalidParameter,
UnsupportedParameter,
SyntaxError,
RequestTooLarge,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -305,17 +424,23 @@ pub enum DeliveryEvent {
Completed,
Failed,
AttemptCount,
MxLookup,
MxLookupFailed,
IpLookup,
IpLookupFailed,
NullMX,
Connect,
ConnectError,
MissingOutboundHostname,
GreetingFailed,
Ehlo,
EhloRejected,
Auth,
AuthFailed,
MailFrom,
MailFromRejected,
Delivered,
RcptTo,
RcptToRejected,
RcptToFailed,
MessageRejected,
@@ -324,8 +449,14 @@ pub enum DeliveryEvent {
StartTlsError,
StartTlsDisabled,
ImplicitTlsError,
TooManyConcurrent,
ConcurrencyLimitExceeded,
RateLimitExceeded,
DoubleBounce,
DsnSuccess,
DsnTempFail,
DsnPermFail,
RawInput,
RawOutput,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -337,6 +468,7 @@ pub enum QueueEvent {
BlobNotFound,
RateLimitExceeded,
ConcurrencyLimitExceeded,
QuotaExceeded,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -381,11 +513,12 @@ pub enum OutgoingReportEvent {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum MtaStsEvent {
Authorized,
NotAuthorized,
PolicyFetch,
PolicyNotFound,
PolicyFetchError,
InvalidPolicy,
NotAuthorized,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -486,7 +619,7 @@ pub enum TlsEvent {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum NetworkEvent {
ConnectionStart,
ConnectionStop,
ConnectionEnd,
ListenStart,
ListenStop,
ListenError,
@@ -685,13 +818,18 @@ pub enum StoreEvent {
// Traces
SqlQuery,
LdapQuery,
LdapBind,
// Events
Ingest,
IngestDuplicate,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum JmapEvent {
// Calls
MethodCall,
// Method errors
InvalidArguments,
RequestTooLarge,