diff --git a/Cargo.lock b/Cargo.lock index 1727856a..64c0d32d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1666,6 +1666,10 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "575f75dfd25738df5b91b8e43e14d44bda14637a58fae779fd2b064f8bf3e010" +[[package]] +name = "dav" +version = "0.11.5" + [[package]] name = "dbl" version = "0.3.2" @@ -2976,6 +2980,48 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "0.11.5" +dependencies = [ + "async-stream", + "base64 0.22.1", + "chrono", + "common", + "directory", + "email", + "form-data", + "http-body-util", + "http_proto", + "hyper 1.6.0", + "hyper-util", + "jmap", + "jmap_proto", + "mail-auth", + "mail-builder", + "mail-parser", + "mail-send", + "mime", + "pkcs8", + "quick-xml 0.37.2", + "rev_lines", + "rkyv 0.8.10", + "rsa", + "serde", + "serde_json", + "services", + "sha1", + "sha2 0.10.8", + "smtp", + "smtp-proto", + "spam-filter", + "store", + "tokio", + "trc", + "utils", + "x509-parser 0.16.0", +] + [[package]] name = "http" version = "1.2.0" @@ -3041,6 +3087,20 @@ dependencies = [ "url", ] +[[package]] +name = "http_proto" +version = "0.11.5" +dependencies = [ + "common", + "form_urlencoded", + "http-body-util", + "hyper 1.6.0", + "hyper-util", + "serde", + "serde_json", + "trc", +] + [[package]] name = "httparse" version = "1.10.0" @@ -3623,11 +3683,10 @@ dependencies = [ "common", "directory", "email", - "form-data", - "form_urlencoded", "futures-util", "hkdf", "http-body-util", + "http_proto", "hyper 1.6.0", "hyper-util", "jmap_proto", @@ -3636,19 +3695,16 @@ dependencies = [ "mail-builder", "mail-parser", "mail-send", - "memory-stats", - "mime", "nlp", "p256", "pkcs8", - "quick-xml 0.37.2", "rand 0.9.0", "reqwest 0.12.12", - "rev_lines", "rkyv 0.8.10", "rsa", "serde", "serde_json", + "services", "sha1", "sha2 0.10.8", "sieve-rs", @@ -3661,7 +3717,6 @@ dependencies = [ "trc", "tungstenite 0.26.2", "utils", - "x509-parser 0.16.0", ] [[package]] @@ -4074,14 +4129,18 @@ name = "mail-server" version = "0.11.8" dependencies = [ "common", + "dav", "directory", "email", + "groupware", + "http 0.11.5", "imap", "jemallocator", "jmap", "jmap_proto", "managesieve", "pop3", + "services", "smtp", "spam-filter", "store", @@ -6530,6 +6589,33 @@ dependencies = [ "syn 2.0.98", ] +[[package]] +name = "services" +version = "0.11.5" +dependencies = [ + "aes-gcm", + "aes-gcm-siv", + "base64 0.22.1", + "common", + "directory", + "email", + "hkdf", + "jmap_proto", + "mail-parser", + "memory-stats", + "p256", + "reqwest 0.12.12", + "rsa", + "serde", + "serde_json", + "sha2 0.10.8", + "smtp", + "store", + "tokio", + "trc", + "utils", +] + [[package]] name = "sha-1" version = "0.9.8" @@ -7071,7 +7157,9 @@ dependencies = [ "flate2", "form_urlencoded", "futures", + "http 0.11.5", "http-body-util", + "http_proto", "hyper 1.6.0", "hyper-util", "imap", @@ -7097,6 +7185,7 @@ dependencies = [ "serde", "serde_json", "serial_test", + "services", "sieve-rs", "smtp", "smtp-proto", diff --git a/Cargo.toml b/Cargo.toml index e5619fc1..b6477019 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,8 @@ resolver = "2" members = [ "crates/main", + "crates/http", + "crates/http-proto", "crates/jmap", "crates/jmap-proto", "crates/email", @@ -10,11 +12,13 @@ members = [ "crates/smtp", "crates/managesieve", "crates/pop3", + "crates/dav", "crates/groupware", "crates/spam-filter", "crates/nlp", "crates/store", "crates/directory", + "crates/services", "crates/utils", "crates/common", "crates/trc", diff --git a/crates/common/src/auth/mod.rs b/crates/common/src/auth/mod.rs index 5778e293..43c9d0ea 100644 --- a/crates/common/src/auth/mod.rs +++ b/crates/common/src/auth/mod.rs @@ -7,7 +7,7 @@ use std::{net::IpAddr, sync::Arc}; use directory::{ - core::secret::verify_secret_hash, Directory, Permission, Permissions, Principal, QueryBy, + Directory, Permission, Permissions, Principal, QueryBy, core::secret::verify_secret_hash, }; use jmap_proto::types::collection::Collection; use mail_send::Credentials; @@ -17,10 +17,11 @@ use utils::{ map::{bitmap::Bitmap, vec_map::VecMap}, }; -use crate::{listener::limiter::ConcurrencyLimiter, Server}; +use crate::{Server, listener::limiter::ConcurrencyLimiter}; pub mod access_token; pub mod oauth; +pub mod rate_limit; pub mod roles; pub mod sasl; diff --git a/crates/jmap/src/auth/rate_limit.rs b/crates/common/src/auth/rate_limit.rs similarity index 76% rename from crates/jmap/src/auth/rate_limit.rs rename to crates/common/src/auth/rate_limit.rs index c49b49a1..770db12e 100644 --- a/crates/jmap/src/auth/rate_limit.rs +++ b/crates/common/src/auth/rate_limit.rs @@ -6,31 +6,17 @@ use std::net::IpAddr; -use common::{ - ip_to_bytes, +use crate::{ + KV_RATE_LIMIT_HTTP_ANONYMOUS, KV_RATE_LIMIT_HTTP_AUTHENTICATED, Server, ip_to_bytes, listener::limiter::{InFlight, LimiterResult}, - Server, KV_RATE_LIMIT_HTTP_ANONYMOUS, KV_RATE_LIMIT_HTTP_AUTHENTICATED, }; use directory::Permission; use trc::AddContext; -use common::auth::AccessToken; -use std::future::Future; +use crate::auth::AccessToken; -pub trait RateLimiter: Sync + Send { - fn is_http_authenticated_request_allowed( - &self, - access_token: &AccessToken, - ) -> impl Future>> + Send; - fn is_http_anonymous_request_allowed( - &self, - addr: &IpAddr, - ) -> impl Future> + Send; - fn is_upload_allowed(&self, access_token: &AccessToken) -> trc::Result>; -} - -impl RateLimiter for Server { - async fn is_http_authenticated_request_allowed( +impl Server { + pub async fn is_http_authenticated_request_allowed( &self, access_token: &AccessToken, ) -> trc::Result> { @@ -70,7 +56,7 @@ impl RateLimiter for Server { } } - async fn is_http_anonymous_request_allowed(&self, addr: &IpAddr) -> trc::Result<()> { + pub async fn is_http_anonymous_request_allowed(&self, addr: &IpAddr) -> trc::Result<()> { if let Some(rate) = &self.core.jmap.rate_anonymous { if !self.is_ip_allowed(addr) && self @@ -93,7 +79,7 @@ impl RateLimiter for Server { Ok(()) } - fn is_upload_allowed(&self, access_token: &AccessToken) -> trc::Result> { + pub fn is_upload_allowed(&self, access_token: &AccessToken) -> trc::Result> { match access_token.is_upload_allowed() { LimiterResult::Allowed(in_flight) => Ok(Some(in_flight)), LimiterResult::Forbidden => { diff --git a/crates/common/src/core.rs b/crates/common/src/core.rs index 1a13f486..83b61d5c 100644 --- a/crates/common/src/core.rs +++ b/crates/common/src/core.rs @@ -667,6 +667,13 @@ impl Server { }) } + pub fn increment_config_version(&self) { + self.inner + .data + .config_version + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + pub async fn total_accounts(&self) -> trc::Result { self.store() .count_principals(None, Type::Individual.into(), None) diff --git a/crates/common/src/expr/functions/asynch.rs b/crates/common/src/expr/functions/asynch.rs index ccf05d61..b745961d 100644 --- a/crates/common/src/expr/functions/asynch.rs +++ b/crates/common/src/expr/functions/asynch.rs @@ -1,4 +1,10 @@ -use std::{cmp::Ordering, net::IpAddr, vec::IntoIter}; +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + + use std::{cmp::Ordering, net::IpAddr, vec::IntoIter}; use directory::backend::RcptType; use mail_auth::IpLookupStrategy; diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 32e6ab32..b3cad7dc 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -12,6 +12,7 @@ use std::{ Arc, atomic::{AtomicBool, AtomicU8}, }, + time::Duration, }; use ahash::{AHashMap, AHashSet}; @@ -68,6 +69,9 @@ pub use psl; pub static USER_AGENT: &str = concat!("Stalwart/", env!("CARGO_PKG_VERSION"),); pub static DAEMON_NAME: &str = concat!("Stalwart Mail Server v", env!("CARGO_PKG_VERSION"),); +pub const LONG_1D_SLUMBER: Duration = Duration::from_secs(60 * 60 * 24); +pub const LONG_1Y_SLUMBER: Duration = Duration::from_secs(60 * 60 * 24 * 365); + pub const IPC_CHANNEL_BUFFER: usize = 1024; pub const KV_ACME: u8 = 0; diff --git a/crates/common/src/telemetry/mod.rs b/crates/common/src/telemetry/mod.rs index 81fde864..4a0cd71d 100644 --- a/crates/common/src/telemetry/mod.rs +++ b/crates/common/src/telemetry/mod.rs @@ -8,18 +8,14 @@ pub mod metrics; pub mod tracers; pub mod webhooks; -use std::time::Duration; - use tracers::log::spawn_log_tracer; use tracers::otel::spawn_otel_tracer; use tracers::stdout::spawn_console_tracer; -use trc::{ipc::subscriber::SubscriberBuilder, Collector}; +use trc::{Collector, ipc::subscriber::SubscriberBuilder}; use webhooks::spawn_webhook_tracer; use crate::config::telemetry::{Telemetry, TelemetrySubscriberType}; -pub const LONG_SLUMBER: Duration = Duration::from_secs(60 * 60 * 24 * 365); - impl Telemetry { pub fn enable(self, is_enterprise: bool) { // Spawn tracers diff --git a/crates/common/src/telemetry/tracers/otel.rs b/crates/common/src/telemetry/tracers/otel.rs index 8dad9d3f..60dec650 100644 --- a/crates/common/src/telemetry/tracers/otel.rs +++ b/crates/common/src/telemetry/tracers/otel.rs @@ -12,19 +12,19 @@ use std::{ use ahash::AHashMap; use mail_parser::DateTime; use opentelemetry::{ + InstrumentationLibrary, Key, KeyValue, Value, logs::{AnyValue, Severity}, trace::{SpanContext, SpanKind, Status, TraceFlags, TraceState}, - InstrumentationLibrary, Key, KeyValue, Value, }; use opentelemetry_sdk::{ + Resource, export::{logs::LogBatch, trace::SpanData}, trace::{SpanEvents, SpanLinks}, - Resource, }; use opentelemetry_semantic_conventions::resource::{SERVICE_NAME, SERVICE_VERSION}; -use trc::{ipc::subscriber::SubscriberBuilder, Event, EventDetails, Level, TelemetryEvent}; +use trc::{Event, EventDetails, Level, TelemetryEvent, ipc::subscriber::SubscriberBuilder}; -use crate::{config::telemetry::OtelTracer, telemetry::LONG_SLUMBER}; +use crate::{LONG_1Y_SLUMBER, config::telemetry::OtelTracer}; const MAX_EVENTS: usize = 2048; @@ -42,7 +42,7 @@ pub(crate) fn spawn_otel_tracer(builder: SubscriberBuilder, mut otel: OtelTracer otel.log_exporter.set_resource(&resource); otel.span_exporter.set_resource(&resource); - let mut wakeup_time = LONG_SLUMBER; + let mut wakeup_time = LONG_1Y_SLUMBER; let mut next_delivery = Instant::now(); let mut pending_logs = Vec::new(); @@ -134,7 +134,7 @@ pub(crate) fn spawn_otel_tracer(builder: SubscriberBuilder, mut otel: OtelTracer } } } - wakeup_time = next_retry.unwrap_or(LONG_SLUMBER); + wakeup_time = next_retry.unwrap_or(LONG_1Y_SLUMBER); } }); } diff --git a/crates/common/src/telemetry/webhooks/mod.rs b/crates/common/src/telemetry/webhooks/mod.rs index 6207fd6c..a7000539 100644 --- a/crates/common/src/telemetry/webhooks/mod.rs +++ b/crates/common/src/telemetry/webhooks/mod.rs @@ -6,31 +6,29 @@ use std::{ sync::{ - atomic::{AtomicBool, Ordering}, Arc, + atomic::{AtomicBool, Ordering}, }, time::Instant, }; -use crate::config::telemetry::WebhookTracer; -use base64::{engine::general_purpose::STANDARD, Engine}; +use crate::{LONG_1Y_SLUMBER, config::telemetry::WebhookTracer}; +use base64::{Engine, engine::general_purpose::STANDARD}; use ring::hmac; use serde::Serialize; use store::write::now; use tokio::sync::mpsc; use trc::{ + Event, EventDetails, ServerEvent, TelemetryEvent, ipc::subscriber::{EventBatch, SubscriberBuilder}, serializers::json::JsonEventSerializer, - Event, EventDetails, ServerEvent, TelemetryEvent, }; -use super::LONG_SLUMBER; - pub(crate) fn spawn_webhook_tracer(builder: SubscriberBuilder, settings: WebhookTracer) { let (tx, mut rx) = builder.register(); tokio::spawn(async move { let settings = Arc::new(settings); - let mut wakeup_time = LONG_SLUMBER; + let mut wakeup_time = LONG_1Y_SLUMBER; let discard_after = settings.discard_after.as_secs(); let mut pending_events = Vec::new(); let mut next_delivery = Instant::now(); @@ -91,7 +89,7 @@ pub(crate) fn spawn_webhook_tracer(builder: SubscriberBuilder, settings: Webhook } } } - wakeup_time = next_retry.unwrap_or(LONG_SLUMBER); + wakeup_time = next_retry.unwrap_or(LONG_1Y_SLUMBER); } }); } diff --git a/crates/dav/Cargo.toml b/crates/dav/Cargo.toml new file mode 100644 index 00000000..58d14eed --- /dev/null +++ b/crates/dav/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "dav" +version = "0.11.5" +edition = "2024" +resolver = "2" + +[dependencies] + +[dev-dependencies] + +[features] +test_mode = [] +enterprise = [] diff --git a/crates/jmap/src/services/mod.rs b/crates/dav/src/lib.rs similarity index 66% rename from crates/jmap/src/services/mod.rs rename to crates/dav/src/lib.rs index 276c9749..c8a832f8 100644 --- a/crates/jmap/src/services/mod.rs +++ b/crates/dav/src/lib.rs @@ -3,8 +3,3 @@ * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ - -pub mod gossip; -pub mod housekeeper; -pub mod index; -pub mod state; diff --git a/crates/groupware/src/calendar/mod.rs b/crates/groupware/src/calendar/mod.rs index 15fa31d5..25bdcac3 100644 --- a/crates/groupware/src/calendar/mod.rs +++ b/crates/groupware/src/calendar/mod.rs @@ -1,4 +1,10 @@ -use calcard::icalendar::ICalendar; +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + + use calcard::icalendar::ICalendar; use jmap_proto::types::{acl::Acl, value::AclGrant}; use utils::map::vec_map::VecMap; diff --git a/crates/groupware/src/contact/mod.rs b/crates/groupware/src/contact/mod.rs index 15a57f2c..6a7526ec 100644 --- a/crates/groupware/src/contact/mod.rs +++ b/crates/groupware/src/contact/mod.rs @@ -1,4 +1,10 @@ -use calcard::vcard::VCard; +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + + use calcard::vcard::VCard; use jmap_proto::types::{acl::Acl, value::AclGrant}; pub struct AddressBook { diff --git a/crates/groupware/src/file/mod.rs b/crates/groupware/src/file/mod.rs index 64ceff91..4197199d 100644 --- a/crates/groupware/src/file/mod.rs +++ b/crates/groupware/src/file/mod.rs @@ -1,4 +1,10 @@ -use jmap_proto::types::{blob::BlobId, value::AclGrant}; +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + + use jmap_proto::types::{blob::BlobId, value::AclGrant}; pub struct FileNode { pub parent_id: Option, diff --git a/crates/groupware/src/lib.rs b/crates/groupware/src/lib.rs index 7f48a0ca..80eaacad 100644 --- a/crates/groupware/src/lib.rs +++ b/crates/groupware/src/lib.rs @@ -1,3 +1,9 @@ -pub mod calendar; +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + + pub mod calendar; pub mod contact; pub mod file; diff --git a/crates/http-proto/Cargo.toml b/crates/http-proto/Cargo.toml new file mode 100644 index 00000000..a6cbd56a --- /dev/null +++ b/crates/http-proto/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "http_proto" +version = "0.11.5" +edition = "2024" +resolver = "2" + +[dependencies] +common = { path = "../common" } +trc = { path = "../trc" } +serde = { version = "1.0", features = ["derive"]} +serde_json = "1.0" +hyper = { version = "1.0.1", features = ["server", "http1", "http2"] } +hyper-util = { version = "0.1.1", features = ["tokio"] } +http-body-util = "0.1.0" +form_urlencoded = "1.1.0" + +[dev-dependencies] + +[features] +test_mode = [] +enterprise = [] diff --git a/crates/http-proto/src/context.rs b/crates/http-proto/src/context.rs new file mode 100644 index 00000000..6dd242df --- /dev/null +++ b/crates/http-proto/src/context.rs @@ -0,0 +1,81 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use common::{ + Server, + expr::{functions::ResolveVariable, *}, +}; +use hyper::StatusCode; + +use crate::{HttpContext, HttpRequest, HttpSessionData}; + +impl<'x> HttpContext<'x> { + pub fn new(session: &'x HttpSessionData, req: &'x HttpRequest) -> Self { + Self { session, req } + } + + pub async fn resolve_response_url(&self, server: &Server) -> String { + server + .eval_if( + &server.core.network.http_response_url, + self, + self.session.session_id, + ) + .await + .unwrap_or_else(|| { + format!( + "http{}://{}:{}", + if self.session.is_tls { "s" } else { "" }, + self.session.local_ip, + self.session.local_port + ) + }) + } + + pub async fn has_endpoint_access(&self, server: &Server) -> StatusCode { + server + .eval_if( + &server.core.network.http_allowed_endpoint, + self, + self.session.session_id, + ) + .await + .unwrap_or(StatusCode::OK) + } +} + +impl ResolveVariable for HttpContext<'_> { + fn resolve_variable(&self, variable: u32) -> Variable<'_> { + match variable { + V_REMOTE_IP => self.session.remote_ip.to_string().into(), + V_REMOTE_PORT => self.session.remote_port.into(), + V_LOCAL_IP => self.session.local_ip.to_string().into(), + V_LOCAL_PORT => self.session.local_port.into(), + V_TLS => self.session.is_tls.into(), + V_PROTOCOL => if self.session.is_tls { "https" } else { "http" }.into(), + V_LISTENER => self.session.instance.id.as_str().into(), + V_URL => self.req.uri().to_string().into(), + V_URL_PATH => self.req.uri().path().into(), + V_METHOD => self.req.method().as_str().into(), + V_HEADERS => self + .req + .headers() + .iter() + .map(|(h, v)| { + Variable::String( + format!("{}: {}", h.as_str(), v.to_str().unwrap_or_default()).into(), + ) + }) + .collect::>() + .into(), + _ => Variable::default(), + } + } + + fn resolve_global(&self, _: &str) -> Variable<'_> { + Variable::Integer(0) + } +} diff --git a/crates/http-proto/src/lib.rs b/crates/http-proto/src/lib.rs new file mode 100644 index 00000000..dfd05999 --- /dev/null +++ b/crates/http-proto/src/lib.rs @@ -0,0 +1,108 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod context; +pub mod request; +pub mod response; + +pub use form_urlencoded; + +use std::{borrow::Cow, net::IpAddr, sync::Arc}; + +use common::listener::ServerInstance; +use hyper::StatusCode; + +pub type HttpRequest = hyper::Request; + +pub struct JsonResponse { + status: StatusCode, + inner: T, + no_cache: bool, +} + +pub struct HtmlResponse { + status: StatusCode, + body: String, +} + +pub enum HttpResponseBody { + Text(String), + Binary(Vec), + Stream(http_body_util::combinators::BoxBody), + WebsocketUpgrade(String), + Empty, +} + +pub struct HttpResponse { + pub status: StatusCode, + pub content_type: Cow<'static, str>, + pub content_disposition: Cow<'static, str>, + pub cache_control: Cow<'static, str>, + pub body: HttpResponseBody, +} + +pub struct HttpContext<'x> { + pub session: &'x HttpSessionData, + pub req: &'x HttpRequest, +} + +pub struct HttpSessionData { + pub instance: Arc, + pub local_ip: IpAddr, + pub local_port: u16, + pub remote_ip: IpAddr, + pub remote_port: u16, + pub is_tls: bool, + pub session_id: u64, +} + +pub struct DownloadResponse { + pub filename: String, + pub content_type: String, + pub blob: Vec, +} + +pub struct JsonProblemResponse(pub StatusCode); + +impl JsonResponse { + pub fn new(inner: T) -> Self { + JsonResponse { + inner, + status: StatusCode::OK, + no_cache: false, + } + } + + pub fn with_status(status: StatusCode, inner: T) -> Self { + JsonResponse { + inner, + status, + no_cache: false, + } + } + + pub fn no_cache(mut self) -> Self { + self.no_cache = true; + self + } +} + +impl HtmlResponse { + pub fn new(body: String) -> Self { + HtmlResponse { + body, + status: StatusCode::OK, + } + } + + pub fn with_status(status: StatusCode, body: String) -> Self { + HtmlResponse { body, status } + } +} + +pub trait ToHttpResponse { + fn into_http_response(self) -> HttpResponse; +} diff --git a/crates/http-proto/src/request.rs b/crates/http-proto/src/request.rs new file mode 100644 index 00000000..1a18536b --- /dev/null +++ b/crates/http-proto/src/request.rs @@ -0,0 +1,58 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::borrow::Cow; + +use http_body_util::BodyExt; + +use crate::HttpRequest; + +pub fn decode_path_element(item: &str) -> Cow<'_, str> { + // Bit hackish but avoids an extra dependency + form_urlencoded::parse(item.as_bytes()) + .into_iter() + .next() + .map(|(k, _)| k) + .unwrap_or_else(|| item.into()) +} + +pub async fn fetch_body( + req: &mut HttpRequest, + max_size: usize, + session_id: u64, +) -> Option> { + let mut bytes = Vec::with_capacity(1024); + while let Some(Ok(frame)) = req.frame().await { + if let Some(data) = frame.data_ref() { + if bytes.len() + data.len() <= max_size || max_size == 0 { + bytes.extend_from_slice(data); + } else { + trc::event!( + Http(trc::HttpEvent::RequestBody), + SpanId = session_id, + Contents = std::str::from_utf8(&bytes) + .unwrap_or("[binary data]") + .to_string(), + Size = bytes.len(), + Limit = max_size, + ); + + return None; + } + } + } + + trc::event!( + Http(trc::HttpEvent::RequestBody), + SpanId = session_id, + Contents = std::str::from_utf8(&bytes) + .unwrap_or("[binary data]") + .to_string(), + Size = bytes.len(), + ); + + bytes.into() +} diff --git a/crates/http-proto/src/response.rs b/crates/http-proto/src/response.rs new file mode 100644 index 00000000..2bc77a32 --- /dev/null +++ b/crates/http-proto/src/response.rs @@ -0,0 +1,185 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + + use std::borrow::Cow; + +use common::manager::webadmin::Resource; +use http_body_util::{BodyExt, Full}; +use hyper::{StatusCode, body::Bytes, header}; +use serde_json::json; + +use crate::{ + DownloadResponse, HtmlResponse, HttpResponse, HttpResponseBody, JsonProblemResponse, + JsonResponse, ToHttpResponse, +}; + +impl HttpResponse { + pub fn new_empty(status: StatusCode) -> Self { + HttpResponse { + status, + content_type: "".into(), + content_disposition: "".into(), + cache_control: "".into(), + body: HttpResponseBody::Empty, + } + } + + pub fn new_text( + status: StatusCode, + content_type: impl Into>, + body: impl Into, + ) -> Self { + HttpResponse { + status, + content_type: content_type.into(), + content_disposition: "".into(), + cache_control: "".into(), + body: HttpResponseBody::Text(body.into()), + } + } + + pub fn new_binary( + status: StatusCode, + content_type: impl Into>, + body: impl Into>, + ) -> Self { + HttpResponse { + status, + content_type: content_type.into(), + content_disposition: "".into(), + cache_control: "".into(), + body: HttpResponseBody::Binary(body.into()), + } + } + + pub fn size(&self) -> usize { + match &self.body { + HttpResponseBody::Text(value) => value.len(), + HttpResponseBody::Binary(value) => value.len(), + _ => 0, + } + } + + pub fn build( + self, + ) -> hyper::Response> + { + let builder = hyper::Response::builder().status(self.status); + + match self.body { + HttpResponseBody::Text(body) => builder + .header(header::CONTENT_TYPE, self.content_type.as_ref()) + .body( + Full::new(Bytes::from(body)) + .map_err(|never| match never {}) + .boxed(), + ), + HttpResponseBody::Binary(body) => { + let mut builder = builder.header(header::CONTENT_TYPE, self.content_type.as_ref()); + + if !self.content_disposition.is_empty() { + builder = builder.header( + header::CONTENT_DISPOSITION, + self.content_disposition.as_ref(), + ); + } + + if !self.cache_control.is_empty() { + builder = builder.header(header::CACHE_CONTROL, self.cache_control.as_ref()); + } + + builder.body( + Full::new(Bytes::from(body)) + .map_err(|never| match never {}) + .boxed(), + ) + } + HttpResponseBody::Empty => builder.body( + Full::new(Bytes::new()) + .map_err(|never| match never {}) + .boxed(), + ), + HttpResponseBody::Stream(stream) => builder + .header(header::CONTENT_TYPE, self.content_type.as_ref()) + .header(header::CACHE_CONTROL, self.cache_control.as_ref()) + .body(stream), + HttpResponseBody::WebsocketUpgrade(derived_key) => builder + .header(header::CONNECTION, "upgrade") + .header(header::UPGRADE, "websocket") + .header("Sec-WebSocket-Accept", &derived_key) + .header("Sec-WebSocket-Protocol", "jmap") + .body( + Full::new(Bytes::from("Switching to WebSocket protocol")) + .map_err(|never| match never {}) + .boxed(), + ), + } + .unwrap() + } +} + +impl ToHttpResponse for JsonResponse { + fn into_http_response(self) -> HttpResponse { + HttpResponse { + status: self.status, + content_type: "application/json; charset=utf-8".into(), + content_disposition: "".into(), + cache_control: if !self.no_cache { + "" + } else { + "no-store, no-cache, must-revalidate" + } + .into(), + body: HttpResponseBody::Text(serde_json::to_string(&self.inner).unwrap_or_default()), + } + } +} + +impl ToHttpResponse for DownloadResponse { + fn into_http_response(self) -> HttpResponse { + HttpResponse { + status: StatusCode::OK, + content_type: self.content_type.into(), + content_disposition: format!( + "attachment; filename=\"{}\"", + self.filename.replace('\"', "\\\"") + ) + .into(), + cache_control: "private, immutable, max-age=31536000".into(), + body: HttpResponseBody::Binary(self.blob), + } + } +} + +impl ToHttpResponse for Resource> { + fn into_http_response(self) -> HttpResponse { + HttpResponse::new_binary(StatusCode::OK, self.content_type, self.contents) + } +} + +impl ToHttpResponse for HtmlResponse { + fn into_http_response(self) -> HttpResponse { + HttpResponse::new_text(self.status, "text/html; charset=utf-8", self.body) + } +} + +impl ToHttpResponse for JsonProblemResponse { + fn into_http_response(self) -> HttpResponse { + HttpResponse::new_text( + self.0, + "application/problem+json", + serde_json::to_string(&json!( + { + "type": "about:blank", + "title": self.0.canonical_reason().unwrap_or_default(), + "status": self.0.as_u16(), + "detail": self.0.canonical_reason().unwrap_or_default(), + } + )) + .unwrap_or_default(), + ) + } +} diff --git a/crates/http/Cargo.toml b/crates/http/Cargo.toml new file mode 100644 index 00000000..7855521c --- /dev/null +++ b/crates/http/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "http" +version = "0.11.5" +edition = "2024" +resolver = "2" + +[dependencies] +store = { path = "../store" } +common = { path = "../common" } +utils = { path = "../utils" } +trc = { path = "../trc" } +email = { path = "../email" } +smtp = { path = "../smtp" } +jmap = { path = "../jmap" } +spam-filter = { path = "../spam-filter" } +http_proto = { path = "../http-proto" } +jmap_proto = { path = "../jmap-proto" } +directory = { path = "../directory" } +services = { path = "../services" } +smtp-proto = { version = "0.1" } +mail-parser = { version = "0.10", features = ["full_encoding", "serde_support"] } +mail-builder = { version = "0.4" } +mail-auth = { version = "0.6", features = ["generate"] } +mail-send = { version = "0.5", default-features = false, features = ["cram-md5", "ring", "tls12"] } +tokio = { version = "1.23", features = ["rt"] } +hyper = { version = "1.0.1", features = ["server", "http1", "http2"] } +hyper-util = { version = "0.1.1", features = ["tokio"] } +http-body-util = "0.1.0" +async-stream = "0.3.5" +quick-xml = "0.37" +serde = { version = "1.0", features = ["derive"]} +serde_json = "1.0" +x509-parser = "0.16.0" +chrono = "0.4" +base64 = "0.22" +pkcs8 = { version = "0.10.2", features = ["alloc", "std"] } +rsa = "0.9.2" +sha1 = "0.10" +sha2 = "0.10" +rev_lines = "0.3.0" +rkyv = { version = "0.8.10", features = ["little_endian"] } +form-data = { version = "0.6.0", features = ["sync"], default-features = false } +mime = "0.3.17" + +[dev-dependencies] + +[features] +test_mode = [] +enterprise = [] diff --git a/crates/jmap/src/auth/authenticate.rs b/crates/http/src/auth/authenticate.rs similarity index 96% rename from crates/jmap/src/auth/authenticate.rs rename to crates/http/src/auth/authenticate.rs index 58d6232f..f354c2b5 100644 --- a/crates/jmap/src/auth/authenticate.rs +++ b/crates/http/src/auth/authenticate.rs @@ -6,18 +6,15 @@ use std::sync::Arc; -use common::{auth::AuthRequest, listener::limiter::InFlight, HttpAuthCache, Server}; +use common::{HttpAuthCache, Server, auth::AuthRequest, listener::limiter::InFlight}; +use http_proto::{HttpRequest, HttpSessionData}; use hyper::header; use mail_parser::decoders::base64::base64_decode; use mail_send::Credentials; -use crate::api::{http::HttpSessionData, HttpRequest}; - use common::auth::AccessToken; use std::future::Future; -use super::rate_limit::RateLimiter; - pub trait Authenticator: Sync + Send { fn authenticate_headers( &self, diff --git a/crates/jmap/src/auth/mod.rs b/crates/http/src/auth/mod.rs similarity index 89% rename from crates/jmap/src/auth/mod.rs rename to crates/http/src/auth/mod.rs index 62df1495..afe12509 100644 --- a/crates/jmap/src/auth/mod.rs +++ b/crates/http/src/auth/mod.rs @@ -6,4 +6,3 @@ pub mod authenticate; pub mod oauth; -pub mod rate_limit; diff --git a/crates/jmap/src/auth/oauth/auth.rs b/crates/http/src/auth/oauth/auth.rs similarity index 98% rename from crates/jmap/src/auth/oauth/auth.rs rename to crates/http/src/auth/oauth/auth.rs index 2cc6b1e0..2bf84b9a 100644 --- a/crates/jmap/src/auth/oauth/auth.rs +++ b/crates/http/src/auth/oauth/auth.rs @@ -13,14 +13,14 @@ use common::{ oauth::{CLIENT_ID_MAX_LEN, DEVICE_CODE_LEN, USER_CODE_ALPHABET, USER_CODE_LEN}, }, }; -use rand::{ +use serde::Deserialize; +use serde_json::json; +use std::future::Future; +use store::rand::{ Rng, distr::{Alphanumeric, StandardUniform}, rng, }; -use serde::Deserialize; -use serde_json::json; -use std::future::Future; use store::{ Serialize, dispatch::lookup::KeyValue, @@ -28,13 +28,8 @@ use store::{ }; use trc::AddContext; -use crate::{ - api::{ - HttpRequest, HttpResponse, JsonResponse, - http::{HttpContext, HttpSessionData, ToHttpResponse}, - }, - auth::oauth::OAuthStatus, -}; +use crate::auth::oauth::OAuthStatus; +use http_proto::*; use super::{DeviceAuthResponse, FormData, MAX_POST_LEN, OAuthCode, OAuthCodeRequest}; diff --git a/crates/jmap/src/auth/oauth/mod.rs b/crates/http/src/auth/oauth/mod.rs similarity index 97% rename from crates/jmap/src/auth/oauth/mod.rs rename to crates/http/src/auth/oauth/mod.rs index 212b51be..a9d676a4 100644 --- a/crates/jmap/src/auth/oauth/mod.rs +++ b/crates/http/src/auth/oauth/mod.rs @@ -4,12 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use http_proto::{HttpRequest, request::fetch_body}; use hyper::header::CONTENT_TYPE; use serde::{Deserialize, Serialize}; use utils::map::vec_map::VecMap; -use crate::api::{HttpRequest, http::fetch_body}; - pub mod auth; pub mod openid; pub mod registration; @@ -207,7 +206,7 @@ impl FormData { fields.append(field.name, value); } } else { - for (key, value) in form_urlencoded::parse(&body) { + for (key, value) in http_proto::form_urlencoded::parse(&body) { fields.append(key.into_owned(), value.into_owned()); } } diff --git a/crates/jmap/src/auth/oauth/openid.rs b/crates/http/src/auth/oauth/openid.rs similarity index 95% rename from crates/jmap/src/auth/oauth/openid.rs rename to crates/http/src/auth/oauth/openid.rs index aa414197..8ed363fb 100644 --- a/crates/jmap/src/auth/oauth/openid.rs +++ b/crates/http/src/auth/oauth/openid.rs @@ -7,15 +7,12 @@ use std::future::Future; use common::{ - auth::{oauth::oidc::Userinfo, AccessToken}, Server, + auth::{AccessToken, oauth::oidc::Userinfo}, }; use serde::{Deserialize, Serialize}; -use crate::api::{ - http::{HttpContext, HttpSessionData, ToHttpResponse}, - HttpRequest, HttpResponse, JsonResponse, -}; +use http_proto::*; #[derive(Debug, Serialize, Deserialize)] pub struct OpenIdMetadata { diff --git a/crates/jmap/src/auth/oauth/registration.rs b/crates/http/src/auth/oauth/registration.rs similarity index 94% rename from crates/jmap/src/auth/oauth/registration.rs rename to crates/http/src/auth/oauth/registration.rs index f84bd291..c6d4c021 100644 --- a/crates/jmap/src/auth/oauth/registration.rs +++ b/crates/http/src/auth/oauth/registration.rs @@ -14,16 +14,11 @@ use directory::{ Permission, Principal, QueryBy, Type, backend::internal::{PrincipalField, lookup::DirectoryStore, manage::ManageDirectory}, }; -use rand::{Rng, distr::Alphanumeric, rng}; +use store::rand::{Rng, distr::Alphanumeric, rng}; use trc::{AddContext, AuthEvent}; -use crate::{ - api::{ - HttpRequest, HttpResponse, JsonResponse, - http::{HttpSessionData, ToHttpResponse, fetch_body}, - }, - auth::{authenticate::Authenticator, rate_limit::RateLimiter}, -}; +use crate::auth::authenticate::Authenticator; +use http_proto::{request::fetch_body, *}; use super::ErrorType; diff --git a/crates/jmap/src/auth/oauth/token.rs b/crates/http/src/auth/oauth/token.rs similarity index 99% rename from crates/jmap/src/auth/oauth/token.rs rename to crates/http/src/auth/oauth/token.rs index a8b8e3e3..599c55a1 100644 --- a/crates/jmap/src/auth/oauth/token.rs +++ b/crates/http/src/auth/oauth/token.rs @@ -16,10 +16,7 @@ use std::future::Future; use store::{dispatch::lookup::KeyValue, write::Archive}; use trc::AddContext; -use crate::api::{ - HttpRequest, HttpResponse, JsonResponse, - http::{HttpContext, HttpSessionData, ToHttpResponse}, -}; +use http_proto::*; use super::{ ArchivedOAuthStatus, ErrorType, FormData, MAX_POST_LEN, OAuthCode, OAuthResponse, OAuthStatus, diff --git a/crates/jmap/src/api/autoconfig.rs b/crates/http/src/autoconfig/mod.rs similarity index 95% rename from crates/jmap/src/api/autoconfig.rs rename to crates/http/src/autoconfig/mod.rs index 101f3b85..80b7301a 100644 --- a/crates/jmap/src/api/autoconfig.rs +++ b/crates/http/src/autoconfig/mod.rs @@ -6,16 +6,14 @@ use std::fmt::Write; -use common::{manager::webadmin::Resource, Server}; -use directory::{backend::internal::PrincipalField, QueryBy}; -use quick_xml::events::Event; +use common::{Server, manager::webadmin::Resource}; +use directory::{QueryBy, backend::internal::PrincipalField}; use quick_xml::Reader; +use quick_xml::events::Event; use trc::AddContext; use utils::url_params::UrlParams; -use crate::api::http::ToHttpResponse; - -use super::{HttpRequest, HttpResponse}; +use http_proto::*; use std::future::Future; pub trait Autoconfig: Sync + Send { @@ -108,8 +106,14 @@ impl Autoconfig for Server { // Build XML response let mut config = String::with_capacity(1024); let _ = writeln!(&mut config, ""); - let _ = writeln!(&mut config, ""); - let _ = writeln!(&mut config, "\t"); + let _ = writeln!( + &mut config, + "" + ); + let _ = writeln!( + &mut config, + "\t" + ); let _ = writeln!(&mut config, "\t\t"); let _ = writeln!( &mut config, @@ -251,7 +255,7 @@ fn parse_autodiscover_request(bytes: &[u8]) -> Result { return Err(format!( "Expected value, found unexpected EOF at position {}.", reader.buffer_position() - )) + )); } _ => (), } @@ -271,14 +275,14 @@ fn parse_autodiscover_request(bytes: &[u8]) -> Result { "Error at position {}: {:?}", reader.buffer_position(), e - )) + )); } Ok(event) => { return Err(format!( "Expected tag {}, found unexpected event {event:?} at position {}.", tag_name, reader.buffer_position() - )) + )); } } } diff --git a/crates/jmap/src/api/form.rs b/crates/http/src/form/mod.rs similarity index 98% rename from crates/jmap/src/api/form.rs rename to crates/http/src/form/mod.rs index 6b7cd154..ba01dace 100644 --- a/crates/jmap/src/api/form.rs +++ b/crates/http/src/form/mod.rs @@ -34,10 +34,7 @@ use x509_parser::nom::AsBytes; use crate::auth::oauth::FormData; -use super::{ - HttpResponse, JsonResponse, - http::{HttpSessionData, ToHttpResponse}, -}; +use http_proto::*; pub trait FormHandler: Sync + Send { fn handle_contact_form( diff --git a/crates/http/src/lib.rs b/crates/http/src/lib.rs new file mode 100644 index 00000000..6ebce6fc --- /dev/null +++ b/crates/http/src/lib.rs @@ -0,0 +1,26 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + + pub mod auth; +pub mod autoconfig; +pub mod form; +pub mod management; +pub mod request; + +use std::sync::Arc; + +use common::Inner; + +#[derive(Clone)] +pub struct HttpSessionManager { + pub inner: Arc, +} + +impl HttpSessionManager { + pub fn new(inner: Arc) -> Self { + Self { inner } + } +} diff --git a/crates/jmap/src/email/crypto.rs b/crates/http/src/management/crypto.rs similarity index 98% rename from crates/jmap/src/email/crypto.rs rename to crates/http/src/management/crypto.rs index 4e86f61f..4d059957 100644 --- a/crates/jmap/src/email/crypto.rs +++ b/crates/http/src/management/crypto.rs @@ -6,13 +6,13 @@ use std::{future::Future, sync::Arc}; -use crate::api::{HttpResponse, JsonResponse, http::ToHttpResponse}; use common::{Server, auth::AccessToken}; use directory::backend::internal::manage; use email::message::crypto::{ Algorithm, ArchivedAlgorithm, ArchivedEncryptionMethod, EncryptMessage, EncryptMessageError, EncryptionMethod, EncryptionParams, EncryptionType, try_parse_certs, }; +use http_proto::*; use jmap_proto::types::{collection::Collection, property::Property}; use mail_builder::encoders::base64::base64_encode_mime; use mail_parser::MessageParser; diff --git a/crates/jmap/src/api/management/dkim.rs b/crates/http/src/management/dkim.rs similarity index 85% rename from crates/jmap/src/api/management/dkim.rs rename to crates/http/src/management/dkim.rs index 2b4a140b..f9390fc7 100644 --- a/crates/jmap/src/api/management/dkim.rs +++ b/crates/http/src/management/dkim.rs @@ -6,8 +6,8 @@ use std::str::FromStr; -use common::{auth::AccessToken, config::smtp::auth::simple_pem_parse, Server}; -use directory::{backend::internal::manage, Permission}; +use common::{Server, auth::AccessToken, config::smtp::auth::simple_pem_parse}; +use directory::{Permission, backend::internal::manage}; use hyper::Method; use mail_auth::{ common::crypto::{Ed25519Key, RsaKey, Sha256}, @@ -21,9 +21,7 @@ use serde::{Deserialize, Serialize}; use serde_json::json; use store::write::now; -use crate::api::{http::ToHttpResponse, HttpRequest, HttpResponse, JsonResponse}; - -use super::decode_path_element; +use http_proto::{request::decode_path_element, *}; use std::future::Future; #[derive(Debug, Serialize, Deserialize, Copy, Clone, PartialEq, Eq)] @@ -132,7 +130,7 @@ impl DkimManagement for Server { Err(err) => { return Err( trc::EventType::Resource(trc::ResourceEvent::BadParameters).reason(err) - ) + ); } }; @@ -223,28 +221,31 @@ impl DkimManagement for Server { self.core .storage .config - .set([ - ( - format!("signature.{id}.private-key"), - String::from_utf8(pk).unwrap(), - ), - (format!("signature.{id}.domain"), domain.into()), - (format!("signature.{id}.selector"), selector.into()), - (format!("signature.{id}.algorithm"), algorithm.to_string()), - ( - format!("signature.{id}.canonicalization"), - "relaxed/relaxed".to_string(), - ), - (format!("signature.{id}.headers.0"), "From".to_string()), - (format!("signature.{id}.headers.1"), "To".to_string()), - (format!("signature.{id}.headers.2"), "Date".to_string()), - (format!("signature.{id}.headers.3"), "Subject".to_string()), - ( - format!("signature.{id}.headers.4"), - "Message-ID".to_string(), - ), - (format!("signature.{id}.report"), "false".to_string()), - ], true) + .set( + [ + ( + format!("signature.{id}.private-key"), + String::from_utf8(pk).unwrap(), + ), + (format!("signature.{id}.domain"), domain.into()), + (format!("signature.{id}.selector"), selector.into()), + (format!("signature.{id}.algorithm"), algorithm.to_string()), + ( + format!("signature.{id}.canonicalization"), + "relaxed/relaxed".to_string(), + ), + (format!("signature.{id}.headers.0"), "From".to_string()), + (format!("signature.{id}.headers.1"), "To".to_string()), + (format!("signature.{id}.headers.2"), "Date".to_string()), + (format!("signature.{id}.headers.3"), "Subject".to_string()), + ( + format!("signature.{id}.headers.4"), + "Message-ID".to_string(), + ), + (format!("signature.{id}.report"), "false".to_string()), + ], + true, + ) .await } } diff --git a/crates/jmap/src/api/management/dns.rs b/crates/http/src/management/dns.rs similarity index 97% rename from crates/jmap/src/api/management/dns.rs rename to crates/http/src/management/dns.rs index 5703e35b..26c6f155 100644 --- a/crates/jmap/src/api/management/dns.rs +++ b/crates/http/src/management/dns.rs @@ -4,10 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{auth::AccessToken, Server}; +use common::{Server, auth::AccessToken}; use directory::{ - backend::internal::manage::{self}, Permission, + backend::internal::manage::{self}, }; use hyper::Method; @@ -17,13 +17,8 @@ use sha1::Digest; use utils::config::Config; use x509_parser::parse_x509_certificate; -use crate::api::{ - http::ToHttpResponse, - management::dkim::{obtain_dkim_public_key, Algorithm}, - HttpRequest, HttpResponse, JsonResponse, -}; - -use super::decode_path_element; +use crate::management::dkim::{Algorithm, obtain_dkim_public_key}; +use http_proto::{request::decode_path_element, *}; use std::future::Future; #[derive(Debug, Serialize, Deserialize)] diff --git a/crates/jmap/src/api/management/enterprise/mod.rs b/crates/http/src/management/enterprise/mod.rs similarity index 100% rename from crates/jmap/src/api/management/enterprise/mod.rs rename to crates/http/src/management/enterprise/mod.rs diff --git a/crates/jmap/src/api/management/enterprise/telemetry.rs b/crates/http/src/management/enterprise/telemetry.rs similarity index 98% rename from crates/jmap/src/api/management/enterprise/telemetry.rs rename to crates/http/src/management/enterprise/telemetry.rs index 7354f774..c054ee20 100644 --- a/crates/jmap/src/api/management/enterprise/telemetry.rs +++ b/crates/http/src/management/enterprise/telemetry.rs @@ -14,34 +14,32 @@ use std::{ }; use common::{ - auth::{oauth::GrantType, AccessToken}, + Server, + auth::{AccessToken, oauth::GrantType}, telemetry::{ metrics::store::{Metric, MetricsStore}, tracers::store::{TracingQuery, TracingStore}, }, - Server, }; -use directory::{backend::internal::manage, Permission}; -use http_body_util::{combinators::BoxBody, StreamBody}; +use directory::{Permission, backend::internal::manage}; +use http_body_util::{StreamBody, combinators::BoxBody}; +use http_proto::*; use hyper::{ - body::{Bytes, Frame}, Method, StatusCode, + body::{Bytes, Frame}, }; use mail_parser::DateTime; use serde_json::json; use std::future::Future; use store::ahash::{AHashMap, AHashSet}; use trc::{ + Collector, DeliveryEvent, EventType, Key, MetricType, QueueEvent, Value, ipc::{bitset::Bitset, subscriber::SubscriberBuilder}, serializers::json::JsonEventSerializer, - Collector, DeliveryEvent, EventType, Key, MetricType, QueueEvent, Value, }; use utils::{snowflake::SnowflakeIdGenerator, url_params::UrlParams}; -use crate::api::{ - http::ToHttpResponse, management::Timestamp, HttpRequest, HttpResponse, HttpResponseBody, - JsonResponse, -}; +use crate::management::Timestamp; pub trait TelemetryApi: Sync + Send { fn handle_telemetry_api_request( diff --git a/crates/jmap/src/api/management/enterprise/undelete.rs b/crates/http/src/management/enterprise/undelete.rs similarity index 97% rename from crates/jmap/src/api/management/enterprise/undelete.rs rename to crates/http/src/management/enterprise/undelete.rs index 88686a3e..e1b4b2f3 100644 --- a/crates/jmap/src/api/management/enterprise/undelete.rs +++ b/crates/http/src/management/enterprise/undelete.rs @@ -26,14 +26,7 @@ use store::write::{BatchBuilder, BlobOp, ValueClass}; use trc::AddContext; use utils::{BlobHash, url_params::UrlParams}; -use crate::{ - api::{ - HttpRequest, HttpResponse, JsonResponse, - http::{HttpSessionData, ToHttpResponse}, - management::decode_path_element, - }, - blob::download::BlobDownload, -}; +use http_proto::{request::decode_path_element, *}; #[derive(serde::Deserialize, serde::Serialize)] pub struct UndeleteRequest { @@ -195,7 +188,11 @@ impl UndeleteApi for Server { for request in requests { match request.collection { Collection::Email => { - match self.get_blob(&request.hash, 0..usize::MAX).await? { + match self + .blob_store() + .get_blob(request.hash.as_slice(), 0..usize::MAX) + .await? + { Some(bytes) => { match self .email_ingest(IngestEmail { diff --git a/crates/jmap/src/api/management/log.rs b/crates/http/src/management/log.rs similarity index 94% rename from crates/jmap/src/api/management/log.rs rename to crates/http/src/management/log.rs index ef4191e9..4da11f1c 100644 --- a/crates/jmap/src/api/management/log.rs +++ b/crates/http/src/management/log.rs @@ -1,12 +1,18 @@ -use std::{ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + + use std::{ fs::{self, File}, io, path::Path, }; use chrono::DateTime; -use common::{auth::AccessToken, Server}; -use directory::{backend::internal::manage, Permission}; +use common::{Server, auth::AccessToken}; +use directory::{Permission, backend::internal::manage}; use rev_lines::RevLines; use serde::Serialize; use serde_json::json; @@ -14,7 +20,7 @@ use std::future::Future; use tokio::sync::oneshot; use utils::url_params::UrlParams; -use crate::api::{http::ToHttpResponse, HttpRequest, HttpResponse, JsonResponse}; +use http_proto::*; #[derive(Serialize)] struct LogEntry { diff --git a/crates/jmap/src/api/management/mod.rs b/crates/http/src/management/mod.rs similarity index 76% rename from crates/jmap/src/api/management/mod.rs rename to crates/http/src/management/mod.rs index 2c288d50..ccac3b46 100644 --- a/crates/jmap/src/api/management/mod.rs +++ b/crates/http/src/management/mod.rs @@ -4,6 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +pub mod crypto; pub mod dkim; pub mod dns; #[cfg(feature = "enterprise")] @@ -18,15 +19,17 @@ pub mod spam; pub mod stores; pub mod troubleshoot; -use std::{borrow::Cow, str::FromStr, sync::Arc}; +use std::{str::FromStr, sync::Arc}; -use common::{auth::AccessToken, Server}; -use directory::{backend::internal::manage, Permission}; +use common::{Server, auth::AccessToken}; +use crypto::CryptoHandler; +use directory::{Permission, backend::internal::manage}; use dkim::DkimManagement; use dns::DnsManagement; #[cfg(feature = "enterprise")] use enterprise::telemetry::TelemetryApi; use hyper::Method; +use jmap::api::{ToJmapHttpResponse, ToRequestError}; use log::LogManagement; use mail_parser::DateTime; use principal::PrincipalManager; @@ -40,12 +43,9 @@ use store::write::now; use stores::ManageStore; use troubleshoot::TroubleshootApi; -use crate::{auth::oauth::auth::OAuthApiHandler, email::crypto::CryptoHandler}; +use crate::auth::oauth::auth::OAuthApiHandler; -use super::{ - http::{fetch_body, HttpSessionData}, - HttpRequest, HttpResponse, -}; +use http_proto::{request::fetch_body, *}; use std::future::Future; #[derive(Serialize)] @@ -194,15 +194,6 @@ impl ManagementApi for Server { } } -pub fn decode_path_element(item: &str) -> Cow<'_, str> { - // Bit hackish but avoids an extra dependency - form_urlencoded::parse(item.as_bytes()) - .into_iter() - .next() - .map(|(k, _)| k) - .unwrap_or_else(|| item.into()) -} - pub(super) struct FutureTimestamp(u64); pub(super) struct Timestamp(u64); @@ -244,3 +235,51 @@ impl Timestamp { self.0 } } + +pub trait ToManageHttpResponse { + fn into_http_response(self) -> HttpResponse; +} + +impl ToManageHttpResponse for &trc::Error { + fn into_http_response(self) -> HttpResponse { + match self.as_ref() { + trc::EventType::Manage(cause) => { + match cause { + trc::ManageEvent::MissingParameter => ManagementApiError::FieldMissing { + field: self.value_as_str(trc::Key::Key).unwrap_or_default(), + }, + trc::ManageEvent::AlreadyExists => ManagementApiError::FieldAlreadyExists { + field: self.value_as_str(trc::Key::Key).unwrap_or_default(), + value: self.value_as_str(trc::Key::Value).unwrap_or_default(), + }, + trc::ManageEvent::NotFound => ManagementApiError::NotFound { + item: self.value_as_str(trc::Key::Key).unwrap_or_default(), + }, + trc::ManageEvent::NotSupported => ManagementApiError::Unsupported { + details: self + .value(trc::Key::Details) + .or_else(|| self.value(trc::Key::Reason)) + .and_then(|v| v.as_str()) + .unwrap_or("Requested action is unsupported"), + }, + trc::ManageEvent::AssertFailed => ManagementApiError::AssertFailed, + trc::ManageEvent::Error => ManagementApiError::Other { + reason: self.value_as_str(trc::Key::Reason), + details: self + .value_as_str(trc::Key::Details) + .unwrap_or("Unknown error"), + }, + } + } + .into_http_response(), + + _ => self.to_request_error().into_http_response(), + } + } +} + +impl ManagementApiError<'_> { + fn into_http_response(self) -> HttpResponse { + JsonResponse::new(self).into_http_response() + } +} diff --git a/crates/jmap/src/api/management/principal.rs b/crates/http/src/management/principal.rs similarity index 97% rename from crates/jmap/src/api/management/principal.rs rename to crates/http/src/management/principal.rs index 44281879..784638cb 100644 --- a/crates/jmap/src/api/management/principal.rs +++ b/crates/http/src/management/principal.rs @@ -6,24 +6,22 @@ use std::sync::Arc; -use common::{auth::AccessToken, Server, KV_BAYES_MODEL_USER}; +use common::{KV_BAYES_MODEL_USER, Server, auth::AccessToken}; use directory::{ - backend::internal::{ - lookup::DirectoryStore, - manage::{self, not_found, ChangedPrincipals, ManageDirectory, UpdatePrincipal}, - PrincipalAction, PrincipalField, PrincipalUpdate, PrincipalValue, SpecialSecrets, - }, DirectoryInner, Permission, Principal, QueryBy, Type, + backend::internal::{ + PrincipalAction, PrincipalField, PrincipalUpdate, PrincipalValue, SpecialSecrets, + lookup::DirectoryStore, + manage::{self, ChangedPrincipals, ManageDirectory, UpdatePrincipal, not_found}, + }, }; -use hyper::{header, Method}; +use hyper::{Method, header}; use serde_json::json; use trc::AddContext; use utils::url_params::UrlParams; -use crate::api::{http::ToHttpResponse, HttpRequest, HttpResponse, JsonResponse}; - -use super::decode_path_element; +use http_proto::{request::decode_path_element, *}; use std::future::Future; #[derive(Debug, serde::Serialize, serde::Deserialize)] @@ -575,13 +573,15 @@ impl PrincipalManager for Server { PrincipalField::Tenant => { // Tenants are not allowed to change their tenantId if access_token.tenant.is_some() { - trc::bail!(trc::SecurityEvent::Unauthorized - .into_err() - .details(permission_needed.name()) - .ctx( - trc::Key::Reason, - "Tenants cannot change their tenantId" - )); + trc::bail!( + trc::SecurityEvent::Unauthorized + .into_err() + .details(permission_needed.name()) + .ctx( + trc::Key::Reason, + "Tenants cannot change their tenantId" + ) + ); } } PrincipalField::Roles @@ -732,7 +732,7 @@ impl PrincipalManager for Server { .headers() .get(header::AUTHORIZATION) .and_then(|h| h.to_str().ok()) - .is_none_or( |header| !header.to_lowercase().starts_with("basic ")) + .is_none_or(|header| !header.to_lowercase().starts_with("basic ")) { return Err(manage::error( "Password changes only allowed using Basic auth", diff --git a/crates/jmap/src/api/management/queue.rs b/crates/http/src/management/queue.rs similarity index 99% rename from crates/jmap/src/api/management/queue.rs rename to crates/http/src/management/queue.rs index 449f0c92..e935a710 100644 --- a/crates/jmap/src/api/management/queue.rs +++ b/crates/http/src/management/queue.rs @@ -35,9 +35,8 @@ use store::{ use trc::AddContext; use utils::url_params::UrlParams; -use crate::api::{HttpRequest, HttpResponse, JsonResponse, http::ToHttpResponse}; - -use super::{FutureTimestamp, decode_path_element}; +use super::FutureTimestamp; +use http_proto::{request::decode_path_element, *}; #[derive(Debug, serde::Serialize, serde::Deserialize, PartialEq, Eq)] pub struct Message { diff --git a/crates/jmap/src/api/management/reload.rs b/crates/http/src/management/reload.rs similarity index 96% rename from crates/jmap/src/api/management/reload.rs rename to crates/http/src/management/reload.rs index b8d00469..aec61963 100644 --- a/crates/jmap/src/api/management/reload.rs +++ b/crates/http/src/management/reload.rs @@ -4,17 +4,14 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{auth::AccessToken, ipc::HousekeeperEvent, Server}; +use common::{Server, auth::AccessToken, ipc::HousekeeperEvent}; use directory::Permission; use hyper::Method; use serde_json::json; use std::future::Future; use utils::url_params::UrlParams; -use crate::{ - api::{http::ToHttpResponse, HttpRequest, HttpResponse, JsonResponse}, - JmapMethods, -}; +use http_proto::*; pub trait ManageReload: Sync + Send { fn handle_manage_reload( diff --git a/crates/jmap/src/api/management/report.rs b/crates/http/src/management/report.rs similarity index 99% rename from crates/jmap/src/api/management/report.rs rename to crates/http/src/management/report.rs index 4cc485f0..4e7ef79a 100644 --- a/crates/jmap/src/api/management/report.rs +++ b/crates/http/src/management/report.rs @@ -25,9 +25,7 @@ use store::{ use trc::AddContext; use utils::url_params::UrlParams; -use crate::api::{HttpRequest, HttpResponse, JsonResponse, http::ToHttpResponse}; - -use super::decode_path_element; +use http_proto::{request::decode_path_element, *}; enum ReportType { Dmarc, diff --git a/crates/jmap/src/api/management/settings.rs b/crates/http/src/management/settings.rs similarity index 98% rename from crates/jmap/src/api/management/settings.rs rename to crates/http/src/management/settings.rs index fe69140a..4aacfff1 100644 --- a/crates/jmap/src/api/management/settings.rs +++ b/crates/http/src/management/settings.rs @@ -4,16 +4,14 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{auth::AccessToken, Server}; +use common::{Server, auth::AccessToken}; use directory::Permission; use hyper::Method; use serde_json::json; use store::ahash::AHashMap; use utils::{config::ConfigKey, map::vec_map::VecMap, url_params::UrlParams}; -use crate::api::{http::ToHttpResponse, HttpRequest, HttpResponse, JsonResponse}; - -use super::decode_path_element; +use http_proto::{request::decode_path_element, *}; use std::future::Future; #[derive(Debug, serde::Serialize, serde::Deserialize)] @@ -119,7 +117,7 @@ impl ManageSettings for Server { record.insert("_id".to_string(), id.to_string()); for (k, v) in &settings { if let Some(k) = k.strip_prefix(&prefix) { - if field.is_none_or( |field| field == k) { + if field.is_none_or(|field| field == k) { record.insert(k.to_string(), v.to_string()); } } else if record.len() > 1 { diff --git a/crates/jmap/src/api/management/spam.rs b/crates/http/src/management/spam.rs similarity index 97% rename from crates/jmap/src/api/management/spam.rs rename to crates/http/src/management/spam.rs index cc2c999f..cdfa649a 100644 --- a/crates/jmap/src/api/management/spam.rs +++ b/crates/http/src/management/spam.rs @@ -6,32 +6,27 @@ use std::net::IpAddr; -use common::{auth::AccessToken, config::spamfilter::SpamFilterAction, psl, Server}; +use common::{Server, auth::AccessToken, config::spamfilter::SpamFilterAction, psl}; use directory::{ - backend::internal::manage::{self, ManageDirectory}, Permission, + backend::internal::manage::{self, ManageDirectory}, }; use hyper::Method; use mail_auth::{ - dmarc::verify::DmarcParameters, spf::verify::SpfParameters, AuthenticatedMessage, DmarcResult, + AuthenticatedMessage, DmarcResult, dmarc::verify::DmarcParameters, spf::verify::SpfParameters, }; use mail_parser::{Message, MessageParser}; use serde::{Deserialize, Serialize}; use serde_json::json; use spam_filter::{ + SpamFilterInput, analysis::{init::SpamFilterInit, score::SpamFilterAnalyzeScore}, modules::bayes::BayesClassifier, - SpamFilterInput, }; use std::future::Future; use store::ahash::AHashMap; -use crate::api::{ - http::{HttpSessionData, ToHttpResponse}, - HttpRequest, HttpResponse, JsonResponse, -}; - -use super::decode_path_element; +use http_proto::{request::decode_path_element, *}; pub trait ManageSpamHandler: Sync + Send { fn handle_manage_spam( diff --git a/crates/jmap/src/api/management/stores.rs b/crates/http/src/management/stores.rs similarity index 98% rename from crates/jmap/src/api/management/stores.rs rename to crates/http/src/management/stores.rs index 9d97e620..a30e4d00 100644 --- a/crates/jmap/src/api/management/stores.rs +++ b/crates/http/src/management/stores.rs @@ -20,22 +20,16 @@ use email::{mailbox::UidMailbox, message::ingest::EmailIngest}; use hyper::Method; use jmap_proto::types::{collection::Collection, property::Property}; use serde_json::json; +use services::index::Indexer; use store::{ - Serialize, + Serialize, rand, write::{Archive, Archiver, BatchBuilder, ValueClass, assert::HashedValue}, }; use trc::AddContext; use utils::url_params::UrlParams; -use crate::{ - api::{ - HttpRequest, HttpResponse, JsonResponse, - http::{HttpSessionData, ToHttpResponse}, - }, - services::index::Indexer, -}; +use http_proto::{request::decode_path_element, *}; -use super::decode_path_element; #[cfg(feature = "enterprise")] use super::enterprise::undelete::UndeleteApi; use std::future::Future; diff --git a/crates/jmap/src/api/management/troubleshoot.rs b/crates/http/src/management/troubleshoot.rs similarity index 99% rename from crates/jmap/src/api/management/troubleshoot.rs rename to crates/http/src/management/troubleshoot.rs index b9efa7cf..b0f7fe58 100644 --- a/crates/jmap/src/api/management/troubleshoot.rs +++ b/crates/http/src/management/troubleshoot.rs @@ -11,22 +11,23 @@ use std::{ }; use common::{ - auth::{oauth::GrantType, AccessToken}, + Server, + auth::{AccessToken, oauth::GrantType}, config::smtp::resolver::{Policy, Tlsa}, - psl, Server, + psl, }; use directory::backend::internal::manage; -use http_body_util::{combinators::BoxBody, StreamBody}; +use http_body_util::{StreamBody, combinators::BoxBody}; use hyper::{ - body::{Bytes, Frame}, Method, StatusCode, + body::{Bytes, Frame}, }; use mail_auth::{ + AuthenticatedMessage, DkimResult, DmarcResult, IpLookupStrategy, IprevOutput, IprevResult, + SpfOutput, SpfResult, dmarc::{self, verify::DmarcParameters}, mta_sts::TlsRpt, spf::verify::SpfParameters, - AuthenticatedMessage, DkimResult, DmarcResult, IpLookupStrategy, IprevOutput, IprevResult, - SpfOutput, SpfResult, }; use serde::{Deserialize, Serialize}; use serde_json::json; @@ -39,10 +40,7 @@ use smtp::outbound::{ use tokio::{io::AsyncWriteExt, sync::mpsc}; use utils::url_params::UrlParams; -use crate::api::{ - http::ToHttpResponse, management::decode_path_element, HttpRequest, HttpResponse, - HttpResponseBody, JsonResponse, -}; +use http_proto::{request::decode_path_element, *}; pub trait TroubleshootApi: Sync + Send { fn handle_troubleshoot_api_request( diff --git a/crates/jmap/src/api/http.rs b/crates/http/src/request.rs similarity index 62% rename from crates/jmap/src/api/http.rs rename to crates/http/src/request.rs index 68f05de3..290def79 100644 --- a/crates/jmap/src/api/http.rs +++ b/crates/http/src/request.rs @@ -1,77 +1,55 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use std::{borrow::Cow, net::IpAddr, sync::Arc}; +use std::{net::IpAddr, sync::Arc}; use common::{ Inner, KV_ACME, Server, auth::{AccessToken, oauth::GrantType}, core::BuildServer, - expr::{functions::ResolveVariable, *}, ipc::StateEvent, - listener::{ServerInstance, SessionData, SessionManager, SessionStream}, + listener::{SessionData, SessionManager, SessionStream}, manager::webadmin::Resource, }; use directory::Permission; -use http_body_util::{BodyExt, Full}; +use http_proto::{ + DownloadResponse, HttpContext, HttpRequest, HttpResponse, HttpResponseBody, HttpSessionData, + JsonProblemResponse, ToHttpResponse, form_urlencoded, request::fetch_body, +}; use hyper::{ - Method, StatusCode, - body::{self, Bytes}, + Method, StatusCode, body, header::{self, CONTENT_TYPE}, server::conn::http1, service::service_fn, }; use hyper_util::rt::TokioIo; +use jmap::{ + api::{ + ToJmapHttpResponse, event_source::EventSourceHandler, request::RequestHandler, + session::SessionHandler, + }, + blob::{download::BlobDownload, upload::BlobUpload}, + websocket::upgrade::WebSocketUpgrade, +}; use jmap_proto::{ - error::request::{RequestError, RequestLimitError}, - request::{Request, capability::Session}, - response::Response, + request::Request, types::{blob::BlobId, id::Id}, }; -use std::future::Future; use store::dispatch::lookup::KeyValue; use trc::SecurityEvent; use utils::url_params::UrlParams; -#[cfg(feature = "enterprise")] -use crate::api::management::enterprise::telemetry::TelemetryApi; - use crate::{ + HttpSessionManager, auth::{ authenticate::{Authenticator, HttpHeaders}, oauth::{ FormData, auth::OAuthApiHandler, openid::OpenIdHandler, registration::ClientRegistrationHandler, token::TokenHandler, }, - rate_limit::RateLimiter, }, - blob::{DownloadResponse, UploadResponse, download::BlobDownload, upload::BlobUpload}, - websocket::upgrade::WebSocketUpgrade, -}; - -use super::{ - HtmlResponse, HttpRequest, HttpResponse, HttpResponseBody, JmapSessionManager, JsonResponse, autoconfig::Autoconfig, - event_source::EventSourceHandler, form::FormHandler, - management::{ManagementApi, ManagementApiError, troubleshoot::TroubleshootApi}, - request::RequestHandler, - session::SessionHandler, + management::{ManagementApi, ToManageHttpResponse, troubleshoot::TroubleshootApi}, }; -pub struct HttpSessionData { - pub instance: Arc, - pub local_ip: IpAddr, - pub local_port: u16, - pub remote_ip: IpAddr, - pub remote_port: u16, - pub is_tls: bool, - pub session_id: u64, -} - pub trait ParseHttp: Sync + Send { fn parse_http_request( &self, @@ -96,7 +74,7 @@ impl ParseHttp for Server { status => { // Allow loopback address to avoid lockouts if !session.remote_ip.is_loopback() { - return Ok(status.into_http_response()); + return Ok(JsonProblemResponse(status).into_http_response()); } } } @@ -214,7 +192,7 @@ impl ParseHttp for Server { .await; } (_, &Method::OPTIONS) => { - return Ok(StatusCode::NO_CONTENT.into_http_response()); + return Ok(JsonProblemResponse(StatusCode::NO_CONTENT).into_http_response()); } _ => (), } @@ -290,7 +268,7 @@ impl ParseHttp for Server { } } (_, &Method::OPTIONS) => { - return Ok(StatusCode::NO_CONTENT.into_http_response()); + return Ok(JsonProblemResponse(StatusCode::NO_CONTENT).into_http_response()); } _ => (), }, @@ -336,14 +314,14 @@ impl ParseHttp for Server { return Ok(self.core.oauth.oidc_jwks.clone().into_http_response()); } (_, &Method::OPTIONS) => { - return Ok(StatusCode::NO_CONTENT.into_http_response()); + return Ok(JsonProblemResponse(StatusCode::NO_CONTENT).into_http_response()); } _ => (), }, "api" => { // Allow CORS preflight requests if req.method() == Method::OPTIONS { - return Ok(StatusCode::NO_CONTENT.into_http_response()); + return Ok(JsonProblemResponse(StatusCode::NO_CONTENT).into_http_response()); } // Authenticate user @@ -393,6 +371,7 @@ impl ParseHttp for Server { // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] GrantType::LiveTracing | GrantType::LiveMetrics => { + use crate::management::enterprise::telemetry::TelemetryApi; self.handle_telemetry_api_request( &req, path, @@ -464,16 +443,16 @@ impl ParseHttp for Server { match path.next().unwrap_or_default() { "live" => { - return Ok(StatusCode::OK.into_http_response()); + return Ok(JsonProblemResponse(StatusCode::OK).into_http_response()); } "ready" => { - return Ok({ + return Ok(JsonProblemResponse({ if !self.core.storage.data.is_none() { StatusCode::OK } else { StatusCode::SERVICE_UNAVAILABLE } - } + }) .into_http_response()); } _ => (), @@ -553,7 +532,9 @@ impl ParseHttp for Server { return self.handle_contact_form(&session, form, form_data).await; } Method::OPTIONS => { - return Ok(StatusCode::NO_CONTENT.into_http_response()); + return Ok( + JsonProblemResponse(StatusCode::NO_CONTENT).into_http_response() + ); } _ => {} } @@ -660,7 +641,9 @@ async fn handle_session(inner: Arc, session: SessionDat ); return Ok::<_, hyper::Error>( - StatusCode::FORBIDDEN.into_http_response().build(), + JsonProblemResponse(StatusCode::FORBIDDEN) + .into_http_response() + .build(), ); } @@ -766,7 +749,7 @@ async fn handle_session(inner: Arc, session: SessionDat } } -impl SessionManager for JmapSessionManager { +impl SessionManager for HttpSessionManager { fn handle(self, session: SessionData) -> impl Future + Send { handle_session(self.inner, session) } @@ -778,464 +761,3 @@ impl SessionManager for JmapSessionManager { } } } - -pub struct HttpContext<'x> { - pub session: &'x HttpSessionData, - pub req: &'x HttpRequest, -} - -impl<'x> HttpContext<'x> { - pub fn new(session: &'x HttpSessionData, req: &'x HttpRequest) -> Self { - Self { session, req } - } - - pub async fn resolve_response_url(&self, server: &Server) -> String { - server - .eval_if( - &server.core.network.http_response_url, - self, - self.session.session_id, - ) - .await - .unwrap_or_else(|| { - format!( - "http{}://{}:{}", - if self.session.is_tls { "s" } else { "" }, - self.session.local_ip, - self.session.local_port - ) - }) - } - - pub async fn has_endpoint_access(&self, server: &Server) -> StatusCode { - server - .eval_if( - &server.core.network.http_allowed_endpoint, - self, - self.session.session_id, - ) - .await - .unwrap_or(StatusCode::OK) - } -} - -impl ResolveVariable for HttpContext<'_> { - fn resolve_variable(&self, variable: u32) -> Variable<'_> { - match variable { - V_REMOTE_IP => self.session.remote_ip.to_string().into(), - V_REMOTE_PORT => self.session.remote_port.into(), - V_LOCAL_IP => self.session.local_ip.to_string().into(), - V_LOCAL_PORT => self.session.local_port.into(), - V_TLS => self.session.is_tls.into(), - V_PROTOCOL => if self.session.is_tls { "https" } else { "http" }.into(), - V_LISTENER => self.session.instance.id.as_str().into(), - V_URL => self.req.uri().to_string().into(), - V_URL_PATH => self.req.uri().path().into(), - V_METHOD => self.req.method().as_str().into(), - V_HEADERS => self - .req - .headers() - .iter() - .map(|(h, v)| { - Variable::String( - format!("{}: {}", h.as_str(), v.to_str().unwrap_or_default()).into(), - ) - }) - .collect::>() - .into(), - _ => Variable::default(), - } - } - - fn resolve_global(&self, _: &str) -> Variable<'_> { - Variable::Integer(0) - } -} - -pub async fn fetch_body( - req: &mut HttpRequest, - max_size: usize, - session_id: u64, -) -> Option> { - let mut bytes = Vec::with_capacity(1024); - while let Some(Ok(frame)) = req.frame().await { - if let Some(data) = frame.data_ref() { - if bytes.len() + data.len() <= max_size || max_size == 0 { - bytes.extend_from_slice(data); - } else { - trc::event!( - Http(trc::HttpEvent::RequestBody), - SpanId = session_id, - Contents = std::str::from_utf8(&bytes) - .unwrap_or("[binary data]") - .to_string(), - Size = bytes.len(), - Limit = max_size, - ); - - return None; - } - } - } - - trc::event!( - Http(trc::HttpEvent::RequestBody), - SpanId = session_id, - Contents = std::str::from_utf8(&bytes) - .unwrap_or("[binary data]") - .to_string(), - Size = bytes.len(), - ); - - bytes.into() -} - -pub trait ToHttpResponse { - fn into_http_response(self) -> HttpResponse; -} - -impl HttpResponse { - pub fn new_empty(status: StatusCode) -> Self { - HttpResponse { - status, - content_type: "".into(), - content_disposition: "".into(), - cache_control: "".into(), - body: HttpResponseBody::Empty, - } - } - - pub fn new_text( - status: StatusCode, - content_type: impl Into>, - body: impl Into, - ) -> Self { - HttpResponse { - status, - content_type: content_type.into(), - content_disposition: "".into(), - cache_control: "".into(), - body: HttpResponseBody::Text(body.into()), - } - } - - pub fn new_binary( - status: StatusCode, - content_type: impl Into>, - body: impl Into>, - ) -> Self { - HttpResponse { - status, - content_type: content_type.into(), - content_disposition: "".into(), - cache_control: "".into(), - body: HttpResponseBody::Binary(body.into()), - } - } - - pub fn size(&self) -> usize { - match &self.body { - HttpResponseBody::Text(value) => value.len(), - HttpResponseBody::Binary(value) => value.len(), - _ => 0, - } - } - - pub fn build( - self, - ) -> hyper::Response> - { - let builder = hyper::Response::builder().status(self.status); - - match self.body { - HttpResponseBody::Text(body) => builder - .header(header::CONTENT_TYPE, self.content_type.as_ref()) - .body( - Full::new(Bytes::from(body)) - .map_err(|never| match never {}) - .boxed(), - ), - HttpResponseBody::Binary(body) => { - let mut builder = builder.header(header::CONTENT_TYPE, self.content_type.as_ref()); - - if !self.content_disposition.is_empty() { - builder = builder.header( - header::CONTENT_DISPOSITION, - self.content_disposition.as_ref(), - ); - } - - if !self.cache_control.is_empty() { - builder = builder.header(header::CACHE_CONTROL, self.cache_control.as_ref()); - } - - builder.body( - Full::new(Bytes::from(body)) - .map_err(|never| match never {}) - .boxed(), - ) - } - HttpResponseBody::Empty => builder.body( - Full::new(Bytes::new()) - .map_err(|never| match never {}) - .boxed(), - ), - HttpResponseBody::Stream(stream) => builder - .header(header::CONTENT_TYPE, self.content_type.as_ref()) - .header(header::CACHE_CONTROL, self.cache_control.as_ref()) - .body(stream), - HttpResponseBody::WebsocketUpgrade(derived_key) => builder - .header(header::CONNECTION, "upgrade") - .header(header::UPGRADE, "websocket") - .header("Sec-WebSocket-Accept", &derived_key) - .header("Sec-WebSocket-Protocol", "jmap") - .body( - Full::new(Bytes::from("Switching to WebSocket protocol")) - .map_err(|never| match never {}) - .boxed(), - ), - } - .unwrap() - } -} - -impl ToHttpResponse for JsonResponse { - fn into_http_response(self) -> HttpResponse { - HttpResponse { - status: self.status, - content_type: "application/json; charset=utf-8".into(), - content_disposition: "".into(), - cache_control: if !self.no_cache { - "" - } else { - "no-store, no-cache, must-revalidate" - } - .into(), - body: HttpResponseBody::Text(serde_json::to_string(&self.inner).unwrap_or_default()), - } - } -} - -impl ToHttpResponse for &trc::Error { - fn into_http_response(self) -> HttpResponse { - match self.as_ref() { - trc::EventType::Manage(cause) => { - match cause { - trc::ManageEvent::MissingParameter => ManagementApiError::FieldMissing { - field: self.value_as_str(trc::Key::Key).unwrap_or_default(), - }, - trc::ManageEvent::AlreadyExists => ManagementApiError::FieldAlreadyExists { - field: self.value_as_str(trc::Key::Key).unwrap_or_default(), - value: self.value_as_str(trc::Key::Value).unwrap_or_default(), - }, - trc::ManageEvent::NotFound => ManagementApiError::NotFound { - item: self.value_as_str(trc::Key::Key).unwrap_or_default(), - }, - trc::ManageEvent::NotSupported => ManagementApiError::Unsupported { - details: self - .value(trc::Key::Details) - .or_else(|| self.value(trc::Key::Reason)) - .and_then(|v| v.as_str()) - .unwrap_or("Requested action is unsupported"), - }, - trc::ManageEvent::AssertFailed => ManagementApiError::AssertFailed, - trc::ManageEvent::Error => ManagementApiError::Other { - reason: self.value_as_str(trc::Key::Reason), - details: self - .value_as_str(trc::Key::Details) - .unwrap_or("Unknown error"), - }, - } - } - .into_http_response(), - - _ => self.to_request_error().into_http_response(), - } - } -} - -pub trait ToRequestError { - fn to_request_error(&self) -> RequestError<'_>; -} - -impl ToRequestError for trc::Error { - fn to_request_error(&self) -> RequestError<'_> { - let details_or_reason = self - .value(trc::Key::Details) - .or_else(|| self.value(trc::Key::Reason)) - .and_then(|v| v.as_str()); - let details = details_or_reason.unwrap_or_else(|| self.as_ref().message()); - - match self.as_ref() { - trc::EventType::Jmap(cause) => match cause { - trc::JmapEvent::UnknownCapability => RequestError::unknown_capability(details), - trc::JmapEvent::NotJson => RequestError::not_json(details), - trc::JmapEvent::NotRequest => RequestError::not_request(details), - _ => RequestError::invalid_parameters(), - }, - trc::EventType::Limit(cause) => match cause { - trc::LimitEvent::SizeRequest => RequestError::limit(RequestLimitError::SizeRequest), - trc::LimitEvent::SizeUpload => RequestError::limit(RequestLimitError::SizeUpload), - trc::LimitEvent::CallsIn => RequestError::limit(RequestLimitError::CallsIn), - trc::LimitEvent::ConcurrentRequest | trc::LimitEvent::ConcurrentConnection => { - RequestError::limit(RequestLimitError::ConcurrentRequest) - } - trc::LimitEvent::ConcurrentUpload => { - RequestError::limit(RequestLimitError::ConcurrentUpload) - } - trc::LimitEvent::Quota => RequestError::over_quota(), - trc::LimitEvent::TenantQuota => RequestError::tenant_over_quota(), - trc::LimitEvent::BlobQuota => RequestError::over_blob_quota( - self.value(trc::Key::Total) - .and_then(|v| v.to_uint()) - .unwrap_or_default() as usize, - self.value(trc::Key::Size) - .and_then(|v| v.to_uint()) - .unwrap_or_default() as usize, - ), - trc::LimitEvent::TooManyRequests => RequestError::too_many_requests(), - }, - trc::EventType::Auth(cause) => match cause { - trc::AuthEvent::MissingTotp => { - RequestError::blank(402, "TOTP code required", cause.message()) - } - trc::AuthEvent::TooManyAttempts => RequestError::too_many_auth_attempts(), - _ => RequestError::unauthorized(), - }, - trc::EventType::Security(cause) => match cause { - trc::SecurityEvent::AuthenticationBan - | trc::SecurityEvent::ScanBan - | trc::SecurityEvent::AbuseBan - | trc::SecurityEvent::LoiterBan - | trc::SecurityEvent::IpBlocked => RequestError::too_many_auth_attempts(), - trc::SecurityEvent::Unauthorized => RequestError::forbidden(), - }, - trc::EventType::Resource(cause) => match cause { - trc::ResourceEvent::NotFound => RequestError::not_found(), - trc::ResourceEvent::BadParameters => RequestError::blank( - StatusCode::BAD_REQUEST.as_u16(), - "Invalid parameters", - details_or_reason.unwrap_or("One or multiple parameters could not be parsed."), - ), - trc::ResourceEvent::Error => RequestError::internal_server_error(), - _ => RequestError::internal_server_error(), - }, - _ => RequestError::internal_server_error(), - } - } -} - -impl JsonResponse { - pub fn new(inner: T) -> Self { - JsonResponse { - inner, - status: StatusCode::OK, - no_cache: false, - } - } - - pub fn with_status(status: StatusCode, inner: T) -> Self { - JsonResponse { - inner, - status, - no_cache: false, - } - } - - pub fn no_cache(mut self) -> Self { - self.no_cache = true; - self - } -} - -impl HtmlResponse { - pub fn new(body: String) -> Self { - HtmlResponse { - body, - status: StatusCode::OK, - } - } - - pub fn with_status(status: StatusCode, body: String) -> Self { - HtmlResponse { body, status } - } -} - -impl ToHttpResponse for Response { - fn into_http_response(self) -> HttpResponse { - JsonResponse::new(self).into_http_response() - } -} - -impl ToHttpResponse for Session { - fn into_http_response(self) -> HttpResponse { - JsonResponse::new(self).into_http_response() - } -} - -impl ToHttpResponse for ManagementApiError<'_> { - fn into_http_response(self) -> super::HttpResponse { - JsonResponse::new(self).into_http_response() - } -} - -impl ToHttpResponse for DownloadResponse { - fn into_http_response(self) -> HttpResponse { - HttpResponse { - status: StatusCode::OK, - content_type: self.content_type.into(), - content_disposition: format!( - "attachment; filename=\"{}\"", - self.filename.replace('\"', "\\\"") - ) - .into(), - cache_control: "private, immutable, max-age=31536000".into(), - body: HttpResponseBody::Binary(self.blob), - } - } -} - -impl ToHttpResponse for Resource> { - fn into_http_response(self) -> HttpResponse { - HttpResponse::new_binary(StatusCode::OK, self.content_type, self.contents) - } -} - -impl ToHttpResponse for UploadResponse { - fn into_http_response(self) -> HttpResponse { - JsonResponse::new(self).into_http_response() - } -} - -impl ToHttpResponse for RequestError<'_> { - fn into_http_response(self) -> HttpResponse { - HttpResponse::new_text( - StatusCode::from_u16(self.status).unwrap_or(StatusCode::BAD_REQUEST), - "application/problem+json", - serde_json::to_string(&self).unwrap_or_default(), - ) - } -} - -impl ToHttpResponse for HtmlResponse { - fn into_http_response(self) -> HttpResponse { - HttpResponse::new_text(self.status, "text/html; charset=utf-8", self.body) - } -} - -impl ToHttpResponse for StatusCode { - fn into_http_response(self) -> HttpResponse { - HttpResponse::new_text( - self, - "application/problem+json", - serde_json::to_string(&RequestError { - p_type: jmap_proto::error::request::RequestErrorType::Other, - status: self.as_u16(), - title: None, - detail: self.canonical_reason().unwrap_or_default().into(), - limit: None, - }) - .unwrap_or_default(), - ) - } -} diff --git a/crates/imap/src/core/mailbox.rs b/crates/imap/src/core/mailbox.rs index d88fde57..9bcaf094 100644 --- a/crates/imap/src/core/mailbox.rs +++ b/crates/imap/src/core/mailbox.rs @@ -1,4 +1,10 @@ -use std::{ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + + use std::{ collections::BTreeMap, sync::{Arc, atomic::Ordering}, }; diff --git a/crates/jmap-proto/src/response/mod.rs b/crates/jmap-proto/src/response/mod.rs index e6df3b30..2b4e12e6 100644 --- a/crates/jmap-proto/src/response/mod.rs +++ b/crates/jmap-proto/src/response/mod.rs @@ -6,6 +6,7 @@ pub mod references; pub mod serialize; +pub mod status; use std::collections::HashMap; @@ -25,7 +26,7 @@ use crate::{ upload::BlobUploadResponse, validate::ValidateSieveScriptResponse, }, - request::{echo::Echo, method::MethodName, Call}, + request::{Call, echo::Echo, method::MethodName}, types::any_id::AnyId, }; diff --git a/crates/jmap-proto/src/response/status.rs b/crates/jmap-proto/src/response/status.rs new file mode 100644 index 00000000..830f0aae --- /dev/null +++ b/crates/jmap-proto/src/response/status.rs @@ -0,0 +1,36 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use utils::map::vec_map::VecMap; + +use crate::types::{id::Id, state::State, type_state::DataType}; + +#[derive(serde::Serialize, serde::Deserialize, Debug)] +pub enum StateChangeType { + StateChange, +} + +#[derive(serde::Serialize, serde::Deserialize, Debug)] +pub struct StateChangeResponse { + #[serde(rename = "@type")] + pub type_: StateChangeType, + pub changed: VecMap>, +} + +impl StateChangeResponse { + pub fn new() -> Self { + Self { + type_: StateChangeType::StateChange, + changed: VecMap::new(), + } + } +} + +impl Default for StateChangeResponse { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/jmap/Cargo.toml b/crates/jmap/Cargo.toml index 5ed721b4..ba0df56e 100644 --- a/crates/jmap/Cargo.toml +++ b/crates/jmap/Cargo.toml @@ -7,10 +7,12 @@ resolver = "2" [dependencies] store = { path = "../store" } nlp = { path = "../nlp" } +http_proto = { path = "../http-proto" } jmap_proto = { path = "../jmap-proto" } smtp = { path = "../smtp" } utils = { path = "../utils" } common = { path = "../common" } +services = { path = "../services" } directory = { path = "../directory" } trc = { path = "../trc" } spam-filter = { path = "../spam-filter" } @@ -26,11 +28,8 @@ serde_json = "1.0" hyper = { version = "1.0.1", features = ["server", "http1", "http2"] } hyper-util = { version = "0.1.1", features = ["tokio"] } http-body-util = "0.1.0" -form_urlencoded = "1.1.0" tokio = { version = "1.23", features = ["rt"] } bincode = "1.3.3" -form-data = { version = "0.6.0", features = ["sync"], default-features = false } -mime = "0.3.17" futures-util = "0.3.28" async-stream = "0.3.5" base64 = "0.22" @@ -45,10 +44,6 @@ chrono = "0.4" rand = "0.9.0" pkcs8 = { version = "0.10.2", features = ["alloc", "std"] } lz4_flex = { version = "0.11", default-features = false } -rev_lines = "0.3.0" -x509-parser = "0.16.0" -quick-xml = "0.37" -memory-stats = "1.2.0" aes-gcm = "0.10.1" aes-gcm-siv = "0.11.1" rsa = "0.9.2" diff --git a/crates/jmap/src/api/event_source.rs b/crates/jmap/src/api/event_source.rs index 0762876b..8cc0aa70 100644 --- a/crates/jmap/src/api/event_source.rs +++ b/crates/jmap/src/api/event_source.rs @@ -9,18 +9,16 @@ use std::{ time::{Duration, Instant}, }; -use common::{Server, auth::AccessToken}; +use common::{LONG_1D_SLUMBER, Server, auth::AccessToken}; use http_body_util::{StreamBody, combinators::BoxBody}; use hyper::{ StatusCode, body::{Bytes, Frame}, }; -use jmap_proto::types::type_state::DataType; +use jmap_proto::{response::status::StateChangeResponse, types::type_state::DataType}; use utils::map::bitmap::Bitmap; -use crate::LONG_SLUMBER; - -use super::{HttpRequest, HttpResponse, HttpResponseBody, StateChangeResponse}; +use http_proto::*; use std::future::Future; struct Ping { @@ -48,7 +46,8 @@ impl EventSourceHandler for Server { let mut types = Bitmap::default(); let mut close_after_state = false; - for (key, value) in form_urlencoded::parse(req.uri().query().unwrap_or_default().as_bytes()) + for (key, value) in + http_proto::form_urlencoded::parse(req.uri().query().unwrap_or_default().as_bytes()) { match key.as_ref() { "types" => { @@ -114,7 +113,7 @@ impl EventSourceHandler for Server { body: HttpResponseBody::Stream(BoxBody::new(StreamBody::new(async_stream::stream! { let mut last_message = Instant::now() - throttle; let mut timeout = - ping.as_ref().map(|p| p.interval).unwrap_or(LONG_SLUMBER); + ping.as_ref().map(|p| p.interval).unwrap_or(LONG_1D_SLUMBER); loop { match tokio::time::timeout(timeout, change_rx.recv()).await { @@ -146,7 +145,7 @@ impl EventSourceHandler for Server { } response.changed.clear(); - ping.as_ref().map(|p| p.interval).unwrap_or(LONG_SLUMBER) + ping.as_ref().map(|p| p.interval).unwrap_or(LONG_1D_SLUMBER) } else { throttle - elapsed } @@ -160,7 +159,7 @@ impl EventSourceHandler for Server { ping.interval - elapsed } } else { - LONG_SLUMBER + LONG_1D_SLUMBER }; } }))), diff --git a/crates/jmap/src/api/mod.rs b/crates/jmap/src/api/mod.rs index 041a451f..9f4e9df6 100644 --- a/crates/jmap/src/api/mod.rs +++ b/crates/jmap/src/api/mod.rs @@ -4,85 +4,119 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{borrow::Cow, sync::Arc}; - -use common::Inner; +use http_proto::{HttpResponse, JsonResponse, ToHttpResponse}; use hyper::StatusCode; -use jmap_proto::types::{id::Id, state::State, type_state::DataType}; -use serde::Serialize; -use utils::map::vec_map::VecMap; +use jmap_proto::{ + error::request::{RequestError, RequestLimitError}, + request::capability::Session, + response::Response, +}; + +use crate::blob::UploadResponse; -pub mod autoconfig; pub mod event_source; -pub mod form; -pub mod http; -pub mod management; pub mod request; pub mod session; -#[derive(Clone)] -pub struct JmapSessionManager { - pub inner: Arc, -} - -impl JmapSessionManager { - pub fn new(inner: Arc) -> Self { - Self { inner } +impl ToHttpResponse for UploadResponse { + fn into_http_response(self) -> HttpResponse { + JsonResponse::new(self).into_http_response() } } -pub struct JsonResponse { - status: StatusCode, - inner: T, - no_cache: bool, +pub trait ToJmapHttpResponse { + fn into_http_response(self) -> HttpResponse; } -pub struct HtmlResponse { - status: StatusCode, - body: String, +impl ToJmapHttpResponse for Response { + fn into_http_response(self) -> HttpResponse { + JsonResponse::new(self).into_http_response() + } } -pub enum HttpResponseBody { - Text(String), - Binary(Vec), - Stream(http_body_util::combinators::BoxBody), - WebsocketUpgrade(String), - Empty, +impl ToJmapHttpResponse for Session { + fn into_http_response(self) -> HttpResponse { + JsonResponse::new(self).into_http_response() + } } -pub struct HttpResponse { - pub status: StatusCode, - pub content_type: Cow<'static, str>, - pub content_disposition: Cow<'static, str>, - pub cache_control: Cow<'static, str>, - pub body: HttpResponseBody, +impl ToJmapHttpResponse for RequestError<'_> { + fn into_http_response(self) -> HttpResponse { + HttpResponse::new_text( + StatusCode::from_u16(self.status).unwrap_or(StatusCode::BAD_REQUEST), + "application/problem+json", + serde_json::to_string(&self).unwrap_or_default(), + ) + } } -pub type HttpRequest = hyper::Request; - -#[derive(serde::Serialize, serde::Deserialize, Debug)] -pub enum StateChangeType { - StateChange, +pub trait ToRequestError { + fn to_request_error(&self) -> RequestError<'_>; } -#[derive(serde::Serialize, serde::Deserialize, Debug)] -pub struct StateChangeResponse { - #[serde(rename = "@type")] - pub type_: StateChangeType, - pub changed: VecMap>, -} +impl ToRequestError for trc::Error { + fn to_request_error(&self) -> RequestError<'_> { + let details_or_reason = self + .value(trc::Key::Details) + .or_else(|| self.value(trc::Key::Reason)) + .and_then(|v| v.as_str()); + let details = details_or_reason.unwrap_or_else(|| self.as_ref().message()); -impl StateChangeResponse { - pub fn new() -> Self { - Self { - type_: StateChangeType::StateChange, - changed: VecMap::new(), + match self.as_ref() { + trc::EventType::Jmap(cause) => match cause { + trc::JmapEvent::UnknownCapability => RequestError::unknown_capability(details), + trc::JmapEvent::NotJson => RequestError::not_json(details), + trc::JmapEvent::NotRequest => RequestError::not_request(details), + _ => RequestError::invalid_parameters(), + }, + trc::EventType::Limit(cause) => match cause { + trc::LimitEvent::SizeRequest => RequestError::limit(RequestLimitError::SizeRequest), + trc::LimitEvent::SizeUpload => RequestError::limit(RequestLimitError::SizeUpload), + trc::LimitEvent::CallsIn => RequestError::limit(RequestLimitError::CallsIn), + trc::LimitEvent::ConcurrentRequest | trc::LimitEvent::ConcurrentConnection => { + RequestError::limit(RequestLimitError::ConcurrentRequest) + } + trc::LimitEvent::ConcurrentUpload => { + RequestError::limit(RequestLimitError::ConcurrentUpload) + } + trc::LimitEvent::Quota => RequestError::over_quota(), + trc::LimitEvent::TenantQuota => RequestError::tenant_over_quota(), + trc::LimitEvent::BlobQuota => RequestError::over_blob_quota( + self.value(trc::Key::Total) + .and_then(|v| v.to_uint()) + .unwrap_or_default() as usize, + self.value(trc::Key::Size) + .and_then(|v| v.to_uint()) + .unwrap_or_default() as usize, + ), + trc::LimitEvent::TooManyRequests => RequestError::too_many_requests(), + }, + trc::EventType::Auth(cause) => match cause { + trc::AuthEvent::MissingTotp => { + RequestError::blank(402, "TOTP code required", cause.message()) + } + trc::AuthEvent::TooManyAttempts => RequestError::too_many_auth_attempts(), + _ => RequestError::unauthorized(), + }, + trc::EventType::Security(cause) => match cause { + trc::SecurityEvent::AuthenticationBan + | trc::SecurityEvent::ScanBan + | trc::SecurityEvent::AbuseBan + | trc::SecurityEvent::LoiterBan + | trc::SecurityEvent::IpBlocked => RequestError::too_many_auth_attempts(), + trc::SecurityEvent::Unauthorized => RequestError::forbidden(), + }, + trc::EventType::Resource(cause) => match cause { + trc::ResourceEvent::NotFound => RequestError::not_found(), + trc::ResourceEvent::BadParameters => RequestError::blank( + StatusCode::BAD_REQUEST.as_u16(), + "Invalid parameters", + details_or_reason.unwrap_or("One or multiple parameters could not be parsed."), + ), + trc::ResourceEvent::Error => RequestError::internal_server_error(), + _ => RequestError::internal_server_error(), + }, + _ => RequestError::internal_server_error(), } } } - -impl Default for StateChangeResponse { - fn default() -> Self { - Self::new() - } -} diff --git a/crates/jmap/src/api/request.rs b/crates/jmap/src/api/request.rs index 6f364976..015a60d2 100644 --- a/crates/jmap/src/api/request.rs +++ b/crates/jmap/src/api/request.rs @@ -7,6 +7,7 @@ use std::{sync::Arc, time::Instant}; use common::{Server, auth::AccessToken}; +use http_proto::HttpSessionData; use jmap_proto::{ method::{ get, query, @@ -39,7 +40,6 @@ use crate::{ vacation::{get::VacationResponseGet, set::VacationResponseSet}, }; -use super::http::HttpSessionData; use std::future::Future; pub trait RequestHandler: Sync + Send { diff --git a/crates/jmap/src/blob/mod.rs b/crates/jmap/src/blob/mod.rs index da70fef3..cdb2a940 100644 --- a/crates/jmap/src/blob/mod.rs +++ b/crates/jmap/src/blob/mod.rs @@ -21,9 +21,3 @@ pub struct UploadResponse { c_type: String, size: usize, } - -pub struct DownloadResponse { - pub filename: String, - pub content_type: String, - pub blob: Vec, -} diff --git a/crates/jmap/src/blob/upload.rs b/crates/jmap/src/blob/upload.rs index 196dbfea..a8671c3b 100644 --- a/crates/jmap/src/blob/upload.rs +++ b/crates/jmap/src/blob/upload.rs @@ -6,7 +6,7 @@ use std::sync::Arc; -use common::{auth::AccessToken, Server}; +use common::{Server, auth::AccessToken}; use directory::Permission; use jmap_proto::{ error::set::SetError, @@ -19,9 +19,7 @@ use jmap_proto::{ use trc::AddContext; -use crate::auth::rate_limit::RateLimiter; - -use super::{download::BlobDownload, UploadResponse}; +use super::{UploadResponse, download::BlobDownload}; use std::future::Future; #[cfg(feature = "test_mode")] diff --git a/crates/jmap/src/email/copy.rs b/crates/jmap/src/email/copy.rs index a055bb3f..f7733530 100644 --- a/crates/jmap/src/email/copy.rs +++ b/crates/jmap/src/email/copy.rs @@ -7,6 +7,7 @@ use common::{Server, auth::AccessToken}; use email::{mailbox::manage::MailboxFnc, message::copy::EmailCopy}; +use http_proto::HttpSessionData; use jmap_proto::{ error::set::SetError, method::{ @@ -29,7 +30,7 @@ use jmap_proto::{ }, }; -use crate::{api::http::HttpSessionData, changes::state::StateManager}; +use crate::changes::state::StateManager; use std::future::Future; use utils::map::vec_map::VecMap; diff --git a/crates/jmap/src/email/import.rs b/crates/jmap/src/email/import.rs index e54bfb4b..bc567ead 100644 --- a/crates/jmap/src/email/import.rs +++ b/crates/jmap/src/email/import.rs @@ -9,6 +9,7 @@ use email::{ mailbox::manage::MailboxFnc, message::ingest::{EmailIngest, IngestEmail, IngestSource}, }; +use http_proto::HttpSessionData; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::import::{ImportEmailRequest, ImportEmailResponse}, @@ -24,9 +25,7 @@ use jmap_proto::{ use mail_parser::MessageParser; use utils::map::vec_map::VecMap; -use crate::{ - api::http::HttpSessionData, blob::download::BlobDownload, changes::state::StateManager, -}; +use crate::{blob::download::BlobDownload, changes::state::StateManager}; use std::future::Future; diff --git a/crates/jmap/src/email/mod.rs b/crates/jmap/src/email/mod.rs index cb46e16f..911c47b8 100644 --- a/crates/jmap/src/email/mod.rs +++ b/crates/jmap/src/email/mod.rs @@ -6,7 +6,6 @@ pub mod body; pub mod copy; -pub mod crypto; pub mod get; pub mod headers; pub mod import; diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index f5fbaf8c..de5740c4 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -14,6 +14,7 @@ use email::{ ingest::{EmailIngest, IngestEmail, IngestSource}, }, }; +use http_proto::HttpSessionData; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{RequestArguments, SetRequest, SetResponse}, @@ -45,10 +46,7 @@ use store::{ }; use trc::AddContext; -use crate::{ - JmapMethods, api::http::HttpSessionData, blob::download::BlobDownload, - changes::state::StateManager, -}; +use crate::{JmapMethods, blob::download::BlobDownload, changes::state::StateManager}; use std::future::Future; use super::headers::{BuildHeader, ValueToHeader}; diff --git a/crates/jmap/src/lib.rs b/crates/jmap/src/lib.rs index 62a9b484..76e0e121 100644 --- a/crates/jmap/src/lib.rs +++ b/crates/jmap/src/lib.rs @@ -4,13 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{fmt::Display, future::Future, sync::Arc, time::Duration}; +use std::{fmt::Display, future::Future}; use changes::state::StateManager; -use common::{ - manager::boot::{BootManager, IpcReceivers}, - Inner, Server, -}; +use common::Server; use jmap_proto::{ method::{ query::{QueryRequest, QueryResponse}, @@ -18,19 +15,15 @@ use jmap_proto::{ }, types::collection::Collection, }; -use services::{ - housekeeper::spawn_housekeeper, index::spawn_email_queue_task, state::spawn_state_manager, -}; use store::{ fts::FtsFilter, - query::{sort::Pagination, Comparator, Filter, ResultSet, SortedResultSet}, + query::{Comparator, Filter, ResultSet, SortedResultSet, sort::Pagination}, roaring::RoaringBitmap, }; use trc::AddContext; pub mod api; -pub mod auth; pub mod blob; pub mod changes; pub mod email; @@ -39,57 +32,12 @@ pub mod mailbox; pub mod principal; pub mod push; pub mod quota; -pub mod services; pub mod sieve; pub mod submission; pub mod thread; pub mod vacation; pub mod websocket; -pub const LONG_SLUMBER: Duration = Duration::from_secs(60 * 60 * 24); - -pub trait StartServices: Sync + Send { - fn start_services(&mut self) -> impl Future + Send; -} - -pub trait SpawnServices { - fn spawn_services(&mut self, inner: Arc); -} - -impl StartServices for BootManager { - async fn start_services(&mut self) { - // Unpack webadmin - if let Err(err) = self - .inner - .data - .webadmin - .unpack(&self.inner.shared_core.load().storage.blob) - .await - { - trc::event!( - Resource(trc::ResourceEvent::Error), - Reason = err, - Details = "Failed to unpack webadmin bundle" - ); - } - - self.ipc_rxs.spawn_services(self.inner.clone()); - } -} - -impl SpawnServices for IpcReceivers { - fn spawn_services(&mut self, inner: Arc) { - // Spawn state manager - spawn_state_manager(inner.clone(), self.state_rx.take().unwrap()); - - // Spawn housekeeper - spawn_housekeeper(inner.clone(), self.housekeeper_rx.take().unwrap()); - - // Spawn index task - spawn_email_queue_task(inner); - } -} - impl JmapMethods for Server { async fn prepare_set_response( &self, @@ -218,13 +166,6 @@ impl JmapMethods for Server { Ok(response) } - - fn increment_config_version(&self) { - self.inner - .data - .config_version - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - } } pub trait JmapMethods: Sync + Send { @@ -261,8 +202,6 @@ pub trait JmapMethods: Sync + Send { paginate: Pagination, response: QueryResponse, ) -> impl Future> + Send; - - fn increment_config_version(&self); } trait UpdateResults: Sized { diff --git a/crates/jmap/src/principal/query.rs b/crates/jmap/src/principal/query.rs index c826929c..66238862 100644 --- a/crates/jmap/src/principal/query.rs +++ b/crates/jmap/src/principal/query.rs @@ -6,13 +6,14 @@ use common::Server; use directory::QueryBy; +use http_proto::HttpSessionData; use jmap_proto::{ method::query::{Filter, QueryRequest, QueryResponse, RequestArguments}, types::collection::Collection, }; use store::{query::ResultSet, roaring::RoaringBitmap}; -use crate::{api::http::HttpSessionData, JmapMethods}; +use crate::JmapMethods; use std::future::Future; pub trait PrincipalQuery: Sync + Send { @@ -77,7 +78,7 @@ impl PrincipalQuery for Server { other => { return Err(trc::JmapEvent::UnsupportedFilter .into_err() - .details(other.to_string())) + .details(other.to_string())); } } } diff --git a/crates/jmap/src/push/get.rs b/crates/jmap/src/push/get.rs index 5b4aeac9..3661f4ab 100644 --- a/crates/jmap/src/push/get.rs +++ b/crates/jmap/src/push/get.rs @@ -7,7 +7,7 @@ use common::{ Server, auth::AccessToken, - ipc::{StateEvent, UpdateSubscription}, + ipc::{EncryptionKeys, PushSubscription, StateEvent, UpdateSubscription}, }; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, @@ -22,10 +22,9 @@ use store::{ BitmapKey, ValueKey, write::{Archive, ValueClass, now}, }; -use trc::AddContext; +use trc::{AddContext, ServerEvent}; use utils::map::bitmap::Bitmap; -use super::{EncryptionKeys, PushSubscription}; use std::future::Future; pub trait PushSubscriptionFetch: Sync + Send { @@ -39,6 +38,8 @@ pub trait PushSubscriptionFetch: Sync + Send { &self, account_id: u32, ) -> impl Future> + Send; + + fn update_push_subscriptions(&self, account_id: u32) -> impl Future + Send; } impl PushSubscriptionFetch for Server { @@ -216,4 +217,32 @@ impl PushSubscriptionFetch for Server { subscriptions, }) } + + async fn update_push_subscriptions(&self, account_id: u32) -> bool { + let push_subs = match self.fetch_push_subscriptions(account_id).await { + Ok(push_subs) => push_subs, + Err(err) => { + trc::error!( + err.account_id(account_id) + .details("Failed to fetch push subscriptions") + ); + return false; + } + }; + + let state_tx = self.inner.ipc.state_tx.clone(); + for event in [StateEvent::UpdateSharedAccounts { account_id }, push_subs] { + if state_tx.send(event).await.is_err() { + trc::event!( + Server(ServerEvent::ThreadError), + Details = "Error sending state change.", + CausedBy = trc::location!() + ); + + return false; + } + } + + true + } } diff --git a/crates/jmap/src/push/mod.rs b/crates/jmap/src/push/mod.rs index 65669632..2caab5e7 100644 --- a/crates/jmap/src/push/mod.rs +++ b/crates/jmap/src/push/mod.rs @@ -4,60 +4,5 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -pub mod ece; pub mod get; -pub mod manager; pub mod set; - -use std::time::Instant; - -use common::ipc::{EncryptionKeys, PushSubscription}; -use jmap_proto::types::{id::Id, state::StateChange}; - -#[derive(Debug)] -pub enum Event { - Update { - updates: Vec, - }, - Push { - ids: Vec, - state_change: StateChange, - }, - DeliverySuccess { - id: Id, - }, - DeliveryFailure { - id: Id, - state_changes: Vec, - }, - Reset, -} - -#[derive(Debug)] -pub enum PushUpdate { - Verify { - id: u32, - account_id: u32, - url: String, - code: String, - keys: Option, - }, - Register { - id: Id, - url: String, - keys: Option, - }, - Unregister { - id: Id, - }, -} - -#[derive(Debug)] -pub struct PushServer { - url: String, - keys: Option, - num_attempts: u32, - last_request: Instant, - state_changes: Vec, - in_flight: bool, -} diff --git a/crates/jmap/src/push/set.rs b/crates/jmap/src/push/set.rs index a176284d..b7a1def4 100644 --- a/crates/jmap/src/push/set.rs +++ b/crates/jmap/src/push/set.rs @@ -29,7 +29,7 @@ use store::{ use trc::AddContext; use utils::map::bitmap::Bitmap; -use crate::services::state::StateManager; +use super::get::PushSubscriptionFetch; const EXPIRES_MAX: i64 = 7 * 24 * 3600; // 7 days const VERIFICATION_CODE_LEN: usize = 32; diff --git a/crates/jmap/src/sieve/set.rs b/crates/jmap/src/sieve/set.rs index 289c1851..e46bafae 100644 --- a/crates/jmap/src/sieve/set.rs +++ b/crates/jmap/src/sieve/set.rs @@ -9,9 +9,8 @@ use common::{ auth::{AccessToken, ResourceToken}, storage::index::ObjectIndexBuilder, }; -use email::sieve::{ - SieveScript, activate::SieveScriptActivate, delete::SieveScriptDelete, -}; +use email::sieve::{SieveScript, activate::SieveScriptActivate, delete::SieveScriptDelete}; +use http_proto::HttpSessionData; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{SetRequest, SetResponse}, @@ -36,7 +35,7 @@ use store::{ }; use trc::AddContext; -use crate::{JmapMethods, api::http::HttpSessionData, blob::download::BlobDownload}; +use crate::{JmapMethods, blob::download::BlobDownload}; use std::future::Future; pub struct SetContext<'x> { diff --git a/crates/jmap/src/websocket/stream.rs b/crates/jmap/src/websocket/stream.rs index c820586f..96c871ad 100644 --- a/crates/jmap/src/websocket/stream.rs +++ b/crates/jmap/src/websocket/stream.rs @@ -8,6 +8,7 @@ use std::{sync::Arc, time::Instant}; use common::{Server, auth::AccessToken}; use futures_util::{SinkExt, StreamExt}; +use http_proto::HttpSessionData; use hyper::upgrade::Upgraded; use hyper_util::rt::TokioIo; use jmap_proto::{ @@ -22,10 +23,7 @@ use trc::JmapEvent; use tungstenite::Message; use utils::map::bitmap::Bitmap; -use crate::api::{ - http::{HttpSessionData, ToRequestError}, - request::RequestHandler, -}; +use crate::api::{ToRequestError, request::RequestHandler}; use std::future::Future; pub trait WebSocketHandler: Sync + Send { diff --git a/crates/jmap/src/websocket/upgrade.rs b/crates/jmap/src/websocket/upgrade.rs index cdafc5ad..9fdfa5cb 100644 --- a/crates/jmap/src/websocket/upgrade.rs +++ b/crates/jmap/src/websocket/upgrade.rs @@ -6,14 +6,14 @@ use std::sync::Arc; -use common::{auth::AccessToken, Server}; +use common::{Server, auth::AccessToken}; use hyper::StatusCode; use hyper_util::rt::TokioIo; use tokio_tungstenite::WebSocketStream; use trc::JmapEvent; use tungstenite::{handshake::derive_accept_key, protocol::Role}; -use crate::api::{http::HttpSessionData, HttpRequest, HttpResponse, HttpResponseBody}; +use http_proto::*; use std::future::Future; use super::stream::WebSocketHandler; diff --git a/crates/main/Cargo.toml b/crates/main/Cargo.toml index f53973c4..c30b9b18 100644 --- a/crates/main/Cargo.toml +++ b/crates/main/Cargo.toml @@ -27,6 +27,10 @@ managesieve = { path = "../managesieve" } common = { path = "../common" } email = { path = "../email" } directory = { path = "../directory" } +http = { path = "../http" } +dav = { path = "../dav" } +groupware = { path = "../groupware" } +services = { path = "../services" } trc = { path = "../trc" } utils = { path = "../utils" } tokio = { version = "1.23", features = ["full"] } @@ -53,4 +57,8 @@ enterprise = [ "jmap/enterprise", "managesieve/enterprise", "directory/enterprise", "email/enterprise", - "spam-filter/enterprise" ] + "spam-filter/enterprise", + "http/enterprise", + "dav/enterprise", + "groupware/enterprise", + "services/enterprise" ] diff --git a/crates/main/src/main.rs b/crates/main/src/main.rs index c35c24d1..601fdab6 100644 --- a/crates/main/src/main.rs +++ b/crates/main/src/main.rs @@ -8,11 +8,12 @@ use std::time::Duration; use common::{config::server::ServerProtocol, core::BuildServer, manager::boot::BootManager}; use directory::backend::internal::MigrateDirectory; +use http::HttpSessionManager; use imap::core::ImapSessionManager; -use jmap::{api::JmapSessionManager, services::gossip::spawn::GossiperBuilder, StartServices}; use managesieve::core::ManageSieveSessionManager; use pop3::Pop3SessionManager; -use smtp::{core::SmtpSessionManager, StartQueueManager}; +use services::{StartServices, gossip::spawn::GossiperBuilder}; +use smtp::{StartQueueManager, core::SmtpSessionManager}; use trc::Collector; use utils::wait_for_shutdown; @@ -61,7 +62,7 @@ async fn main() -> std::io::Result<()> { shutdown_rx, ), ServerProtocol::Http => server.spawn( - JmapSessionManager::new(init.inner.clone()), + HttpSessionManager::new(init.inner.clone()), init.inner.clone(), acceptor, shutdown_rx, diff --git a/crates/nlp/src/lib.rs b/crates/nlp/src/lib.rs index 557f9c5a..659721b0 100644 --- a/crates/nlp/src/lib.rs +++ b/crates/nlp/src/lib.rs @@ -1,4 +1,10 @@ -pub mod bayes; +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + + pub mod bayes; pub mod language; pub mod tokenizers; diff --git a/crates/services/Cargo.toml b/crates/services/Cargo.toml new file mode 100644 index 00000000..f0b87d5d --- /dev/null +++ b/crates/services/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "services" +version = "0.11.5" +edition = "2024" +resolver = "2" + +[dependencies] +store = { path = "../store" } +common = { path = "../common" } +utils = { path = "../utils" } +trc = { path = "../trc" } +email = { path = "../email" } +smtp = { path = "../smtp" } +jmap_proto = { path = "../jmap-proto" } +directory = { path = "../directory" } +tokio = { version = "1.23", features = ["rt"] } +mail-parser = { version = "0.10", features = ["full_encoding", "serde_support"] } +serde = { version = "1.0", features = ["derive"]} +serde_json = "1.0" +memory-stats = "1.2.0" +aes-gcm = "0.10.1" +aes-gcm-siv = "0.11.1" +rsa = "0.9.2" +p256 = { version = "0.13", features = ["ecdh"] } +hkdf = "0.12.3" +sha2 = "0.10" +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-webpki-roots", "http2"]} +base64 = "0.22" + +[dev-dependencies] + +[features] +test_mode = [] +enterprise = [] diff --git a/crates/jmap/src/services/gossip/heartbeat.rs b/crates/services/src/gossip/heartbeat.rs similarity index 100% rename from crates/jmap/src/services/gossip/heartbeat.rs rename to crates/services/src/gossip/heartbeat.rs diff --git a/crates/jmap/src/services/gossip/leave.rs b/crates/services/src/gossip/leave.rs similarity index 97% rename from crates/jmap/src/services/gossip/leave.rs rename to crates/services/src/gossip/leave.rs index 3c729ecf..fa124330 100644 --- a/crates/jmap/src/services/gossip/leave.rs +++ b/crates/services/src/gossip/leave.rs @@ -6,7 +6,7 @@ use trc::ClusterEvent; -use crate::services::gossip::State; +use crate::gossip::State; use super::request::Request; use super::{Gossiper, PeerStatus}; diff --git a/crates/jmap/src/services/gossip/mod.rs b/crates/services/src/gossip/mod.rs similarity index 95% rename from crates/jmap/src/services/gossip/mod.rs rename to crates/services/src/gossip/mod.rs index cbfe9804..10ff5d8e 100644 --- a/crates/jmap/src/services/gossip/mod.rs +++ b/crates/services/src/gossip/mod.rs @@ -12,10 +12,9 @@ pub mod request; pub mod spawn; use common::Inner; -use serde::{Deserialize, Serialize}; use std::{ net::{IpAddr, SocketAddr}, - sync::{atomic::Ordering, Arc}, + sync::{Arc, atomic::Ordering}, time::Instant, }; use tokio::sync::mpsc; @@ -76,7 +75,7 @@ pub struct Peer { pub hb_is_full: bool, } -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Clone)] pub struct PeerStatus { pub addr: IpAddr, pub epoch: EpochId, diff --git a/crates/jmap/src/services/gossip/peer.rs b/crates/services/src/gossip/peer.rs similarity index 100% rename from crates/jmap/src/services/gossip/peer.rs rename to crates/services/src/gossip/peer.rs diff --git a/crates/jmap/src/services/gossip/ping.rs b/crates/services/src/gossip/ping.rs similarity index 100% rename from crates/jmap/src/services/gossip/ping.rs rename to crates/services/src/gossip/ping.rs diff --git a/crates/jmap/src/services/gossip/request.rs b/crates/services/src/gossip/request.rs similarity index 100% rename from crates/jmap/src/services/gossip/request.rs rename to crates/services/src/gossip/request.rs diff --git a/crates/jmap/src/services/gossip/spawn.rs b/crates/services/src/gossip/spawn.rs similarity index 100% rename from crates/jmap/src/services/gossip/spawn.rs rename to crates/services/src/gossip/spawn.rs diff --git a/crates/jmap/src/services/housekeeper.rs b/crates/services/src/housekeeper/mod.rs similarity index 99% rename from crates/jmap/src/services/housekeeper.rs rename to crates/services/src/housekeeper/mod.rs index 4d92faad..d15f9645 100644 --- a/crates/jmap/src/services/housekeeper.rs +++ b/crates/services/src/housekeeper/mod.rs @@ -12,7 +12,7 @@ use std::{ }; use common::{ - Inner, KV_LOCK_HOUSEKEEPER, Server, + Inner, KV_LOCK_HOUSEKEEPER, LONG_1D_SLUMBER, Server, config::telemetry::OtelMetrics, core::BuildServer, ipc::{HousekeeperEvent, PurgeType}, @@ -30,8 +30,6 @@ use store::{PurgeStore, write::now}; use tokio::sync::mpsc; use trc::{Collector, MetricType, PurgeEvent}; -use crate::{JmapMethods, LONG_SLUMBER}; - #[derive(PartialEq, Eq)] struct Action { due: Instant, @@ -769,7 +767,7 @@ impl Queue { self.heap .peek() .map(|e| e.due.saturating_duration_since(Instant::now())) - .unwrap_or(LONG_SLUMBER) + .unwrap_or(LONG_1D_SLUMBER) } pub fn pop(&mut self) -> Option { diff --git a/crates/jmap/src/services/index.rs b/crates/services/src/index/mod.rs similarity index 96% rename from crates/jmap/src/services/index.rs rename to crates/services/src/index/mod.rs index dfd5a315..0b4a3c7b 100644 --- a/crates/jmap/src/services/index.rs +++ b/crates/services/src/index/mod.rs @@ -30,8 +30,6 @@ use std::future::Future; use trc::{AddContext, TaskQueueEvent}; use utils::{BLOB_HASH_LEN, BlobHash}; -use crate::blob::download::BlobDownload; - #[derive(Debug, Clone)] pub struct EmailTask { account_id: u32, @@ -149,18 +147,21 @@ impl Indexer for Server { } // Obtain raw message - let raw_message = - if let Ok(Some(raw_message)) = self.get_blob(&event.hash, 0..usize::MAX).await { - raw_message - } else { - trc::event!( - TaskQueue(TaskQueueEvent::BlobNotFound), - AccountId = event.account_id, - DocumentId = event.document_id, - BlobId = event.hash.as_slice(), - ); - continue; - }; + let raw_message = if let Ok(Some(raw_message)) = self + .blob_store() + .get_blob(event.hash.as_slice(), 0..usize::MAX) + .await + { + raw_message + } else { + trc::event!( + TaskQueue(TaskQueueEvent::BlobNotFound), + AccountId = event.account_id, + DocumentId = event.document_id, + BlobId = event.hash.as_slice(), + ); + continue; + }; match event.action { EmailTaskAction::Index => { diff --git a/crates/services/src/lib.rs b/crates/services/src/lib.rs new file mode 100644 index 00000000..5769b1f9 --- /dev/null +++ b/crates/services/src/lib.rs @@ -0,0 +1,62 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::sync::Arc; + +use common::{ + Inner, + manager::boot::{BootManager, IpcReceivers}, +}; +use housekeeper::spawn_housekeeper; +use index::spawn_email_queue_task; +use state_manager::manager::spawn_state_manager; + +pub mod gossip; +pub mod housekeeper; +pub mod index; +pub mod state_manager; + +pub trait StartServices: Sync + Send { + fn start_services(&mut self) -> impl Future + Send; +} + +pub trait SpawnServices { + fn spawn_services(&mut self, inner: Arc); +} + +impl StartServices for BootManager { + async fn start_services(&mut self) { + // Unpack webadmin + if let Err(err) = self + .inner + .data + .webadmin + .unpack(&self.inner.shared_core.load().storage.blob) + .await + { + trc::event!( + Resource(trc::ResourceEvent::Error), + Reason = err, + Details = "Failed to unpack webadmin bundle" + ); + } + + self.ipc_rxs.spawn_services(self.inner.clone()); + } +} + +impl SpawnServices for IpcReceivers { + fn spawn_services(&mut self, inner: Arc) { + // Spawn state manager + spawn_state_manager(inner.clone(), self.state_rx.take().unwrap()); + + // Spawn housekeeper + spawn_housekeeper(inner.clone(), self.housekeeper_rx.take().unwrap()); + + // Spawn index task + spawn_email_queue_task(inner); + } +} diff --git a/crates/jmap/src/push/ece.rs b/crates/services/src/state_manager/ece.rs similarity index 100% rename from crates/jmap/src/push/ece.rs rename to crates/services/src/state_manager/ece.rs diff --git a/crates/services/src/state_manager/http.rs b/crates/services/src/state_manager/http.rs new file mode 100644 index 00000000..e65edb89 --- /dev/null +++ b/crates/services/src/state_manager/http.rs @@ -0,0 +1,127 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::time::{Duration, Instant}; + +use base64::Engine; +use common::ipc::EncryptionKeys; +use jmap_proto::{response::status::StateChangeResponse, types::id::Id}; +use reqwest::header::{CONTENT_ENCODING, CONTENT_TYPE}; +use tokio::sync::mpsc; +use trc::PushSubscriptionEvent; + +use super::{Event, PushServer, ece::ece_encrypt}; + +impl PushServer { + pub fn send(&mut self, id: Id, push_tx: mpsc::Sender, push_timeout: Duration) { + let url = self.url.clone(); + let keys = self.keys.clone(); + let state_changes = std::mem::take(&mut self.state_changes); + + self.in_flight = true; + self.last_request = Instant::now(); + + tokio::spawn(async move { + let mut response = StateChangeResponse::new(); + for state_change in &state_changes { + for (type_state, change_id) in &state_change.types { + response + .changed + .get_mut_or_insert(state_change.account_id.into()) + .set(*type_state, (*change_id).into()); + } + } + + push_tx + .send( + if http_request( + url, + serde_json::to_string(&response).unwrap(), + keys, + push_timeout, + ) + .await + { + Event::DeliverySuccess { id } + } else { + Event::DeliveryFailure { id, state_changes } + }, + ) + .await + .ok(); + }); + } +} + +pub(crate) async fn http_request( + url: String, + mut body: String, + keys: Option, + push_timeout: Duration, +) -> bool { + let client_builder = reqwest::Client::builder().timeout(push_timeout); + + #[cfg(feature = "test_mode")] + let client_builder = client_builder.danger_accept_invalid_certs(true); + + let mut client = client_builder + .build() + .unwrap_or_default() + .post(&url) + .header(CONTENT_TYPE, "application/json") + .header("TTL", "86400"); + + if let Some(keys) = keys { + match ece_encrypt(&keys.p256dh, &keys.auth, body.as_bytes()) + .map(|b| base64::engine::general_purpose::URL_SAFE.encode(b)) + { + Ok(body_) => { + body = body_; + client = client.header(CONTENT_ENCODING, "aes128gcm"); + } + Err(err) => { + // Do not reattempt if encryption fails. + + trc::event!( + PushSubscription(PushSubscriptionEvent::Error), + Details = "Failed to encrypt push subscription", + Url = url, + Reason = err + ); + return true; + } + } + } + + match client.body(body).send().await { + Ok(response) => { + if response.status().is_success() { + trc::event!(PushSubscription(PushSubscriptionEvent::Success), Url = url,); + + true + } else { + trc::event!( + PushSubscription(PushSubscriptionEvent::Error), + Details = "HTTP POST failed", + Url = url, + Code = response.status().as_u16(), + ); + + false + } + } + Err(err) => { + trc::event!( + PushSubscription(PushSubscriptionEvent::Error), + Details = "HTTP POST failed", + Url = url, + Reason = err.to_string() + ); + + false + } + } +} diff --git a/crates/jmap/src/services/state.rs b/crates/services/src/state_manager/manager.rs similarity index 85% rename from crates/jmap/src/services/state.rs rename to crates/services/src/state_manager/manager.rs index a25b28db..86ebe04e 100644 --- a/crates/jmap/src/services/state.rs +++ b/crates/services/src/state_manager/manager.rs @@ -6,52 +6,24 @@ use std::{ sync::Arc, - time::{Duration, Instant, SystemTime}, + time::{Instant, SystemTime}, }; use common::{ - Inner, Server, + Inner, core::BuildServer, ipc::{PushSubscription, StateEvent, UpdateSubscription}, }; use jmap_proto::types::{id::Id, state::StateChange, type_state::DataType}; -use std::future::Future; -use store::ahash::AHashMap; +use store::{ahash::AHashMap, rand}; use tokio::sync::mpsc; use trc::ServerEvent; use utils::map::bitmap::Bitmap; -use crate::push::{get::PushSubscriptionFetch, manager::spawn_push_manager}; - -#[derive(Debug)] -struct Subscriber { - types: Bitmap, - subscription: SubscriberType, -} - -#[derive(Debug)] -pub enum SubscriberType { - Ipc { tx: mpsc::Sender }, - Push { expires: u64 }, -} - -impl Subscriber { - fn is_valid(&self, current_time: u64) -> bool { - match &self.subscription { - SubscriberType::Ipc { tx } => !tx.is_closed(), - SubscriberType::Push { expires } => expires > ¤t_time, - } - } -} - -const PURGE_EVERY: Duration = Duration::from_secs(3600); -const SEND_TIMEOUT: Duration = Duration::from_millis(500); - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -enum SubscriberId { - Ipc(u32), - Push(u32), -} +use super::{ + Event, PURGE_EVERY, PushUpdate, SEND_TIMEOUT, Subscriber, SubscriberId, SubscriberType, + push::spawn_push_manager, +}; #[allow(clippy::unwrap_or_default)] pub fn spawn_state_manager(inner: Arc, mut change_rx: mpsc::Receiver) { @@ -71,7 +43,7 @@ pub fn spawn_state_manager(inner: Arc, mut change_rx: mpsc::Receiver { - if push_tx.send(crate::push::Event::Reset).await.is_err() { + if push_tx.send(Event::Reset).await.is_err() { trc::event!( Server(ServerEvent::ThreadError), Details = "Error sending push reset.", @@ -229,7 +201,7 @@ pub fn spawn_state_manager(inner: Arc, mut change_rx: mpsc::Receiver, mut change_rx: mpsc::Receiver, mut change_rx: mpsc::Receiver { - push_updates.push(crate::push::PushUpdate::Verify { + push_updates.push(PushUpdate::Verify { id, account_id, url, @@ -305,7 +277,7 @@ pub fn spawn_state_manager(inner: Arc, mut change_rx: mpsc::Receiver, mut change_rx: mpsc::Receiver, mut change_rx: mpsc::Receiver impl Future + Send; -} - -impl StateManager for Server { - async fn update_push_subscriptions(&self, account_id: u32) -> bool { - let push_subs = match self.fetch_push_subscriptions(account_id).await { - Ok(push_subs) => push_subs, - Err(err) => { - trc::error!( - err.account_id(account_id) - .details("Failed to fetch push subscriptions") - ); - return false; - } - }; - - let state_tx = self.inner.ipc.state_tx.clone(); - for event in [StateEvent::UpdateSharedAccounts { account_id }, push_subs] { - if state_tx.send(event).await.is_err() { - trc::event!( - Server(ServerEvent::ThreadError), - Details = "Error sending state change.", - CausedBy = trc::location!() - ); - - return false; - } - } - - true - } -} - -impl From for u32 { - fn from(subscriber_id: SubscriberId) -> u32 { - match subscriber_id { - SubscriberId::Ipc(id) => id, - SubscriberId::Push(id) => id, - } - } -} diff --git a/crates/services/src/state_manager/mod.rs b/crates/services/src/state_manager/mod.rs new file mode 100644 index 00000000..6c4fad61 --- /dev/null +++ b/crates/services/src/state_manager/mod.rs @@ -0,0 +1,104 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod ece; +pub mod http; +pub mod manager; +pub mod push; + +use std::time::{Duration, Instant}; + +use common::ipc::EncryptionKeys; +use jmap_proto::types::{id::Id, state::StateChange, type_state::DataType}; +use tokio::sync::mpsc; +use utils::map::bitmap::Bitmap; + +#[derive(Debug)] +struct Subscriber { + types: Bitmap, + subscription: SubscriberType, +} + +#[derive(Debug)] +pub enum SubscriberType { + Ipc { tx: mpsc::Sender }, + Push { expires: u64 }, +} + +#[derive(Debug)] +pub struct PushServer { + url: String, + keys: Option, + num_attempts: u32, + last_request: Instant, + state_changes: Vec, + in_flight: bool, +} + +#[derive(Debug)] +pub enum Event { + Update { + updates: Vec, + }, + Push { + ids: Vec, + state_change: StateChange, + }, + DeliverySuccess { + id: Id, + }, + DeliveryFailure { + id: Id, + state_changes: Vec, + }, + Reset, +} + +#[derive(Debug)] +pub enum PushUpdate { + Verify { + id: u32, + account_id: u32, + url: String, + code: String, + keys: Option, + }, + Register { + id: Id, + url: String, + keys: Option, + }, + Unregister { + id: Id, + }, +} + +impl Subscriber { + fn is_valid(&self, current_time: u64) -> bool { + match &self.subscription { + SubscriberType::Ipc { tx } => !tx.is_closed(), + SubscriberType::Push { expires } => expires > ¤t_time, + } + } +} + +const PURGE_EVERY: Duration = Duration::from_secs(3600); +const SEND_TIMEOUT: Duration = Duration::from_millis(500); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum SubscriberId { + Ipc(u32), + Push(u32), +} + +impl From for u32 { + fn from(subscriber_id: SubscriberId) -> u32 { + match subscriber_id { + SubscriberId::Ipc(id) => id, + SubscriberId::Push(id) => id, + } + } +} diff --git a/crates/jmap/src/push/manager.rs b/crates/services/src/state_manager/push.rs similarity index 74% rename from crates/jmap/src/push/manager.rs rename to crates/services/src/state_manager/push.rs index c543f400..59faac53 100644 --- a/crates/jmap/src/push/manager.rs +++ b/crates/services/src/state_manager/push.rs @@ -4,24 +4,20 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use base64::{engine::general_purpose, Engine}; -use common::{core::BuildServer, Inner, IPC_CHANNEL_BUFFER}; -use jmap_proto::types::id::Id; -use store::ahash::{AHashMap, AHashSet}; -use tokio::sync::mpsc; -use trc::PushSubscriptionEvent; - -use crate::{api::StateChangeResponse, LONG_SLUMBER}; - -use super::{ece::ece_encrypt, EncryptionKeys, Event, PushServer, PushUpdate}; - -use reqwest::header::{CONTENT_ENCODING, CONTENT_TYPE}; use std::{ collections::hash_map::Entry, sync::Arc, time::{Duration, Instant}, }; +use common::{IPC_CHANNEL_BUFFER, Inner, LONG_1Y_SLUMBER, core::BuildServer}; +use jmap_proto::types::id::Id; +use store::ahash::{AHashMap, AHashSet}; +use tokio::sync::mpsc; +use trc::PushSubscriptionEvent; + +use super::{Event, PushServer, PushUpdate, http::http_request}; + pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { let (push_tx_, mut push_rx) = mpsc::channel::(IPC_CHANNEL_BUFFER); let push_tx = push_tx_.clone(); @@ -30,7 +26,7 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { let mut subscriptions = AHashMap::default(); let mut last_verify: AHashMap = AHashMap::default(); let mut last_retry = Instant::now(); - let mut retry_timeout = LONG_SLUMBER; + let mut retry_timeout = LONG_1Y_SLUMBER; let mut retry_ids = AHashSet::default(); loop { @@ -222,127 +218,16 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { push_retry_interval } else { retry_ids.clear(); - LONG_SLUMBER + LONG_1Y_SLUMBER } } else { push_retry_interval - last_retry_elapsed } } else { - LONG_SLUMBER + LONG_1Y_SLUMBER }; } }); push_tx_ } - -impl PushServer { - fn send(&mut self, id: Id, push_tx: mpsc::Sender, push_timeout: Duration) { - let url = self.url.clone(); - let keys = self.keys.clone(); - let state_changes = std::mem::take(&mut self.state_changes); - - self.in_flight = true; - self.last_request = Instant::now(); - - tokio::spawn(async move { - let mut response = StateChangeResponse::new(); - for state_change in &state_changes { - for (type_state, change_id) in &state_change.types { - response - .changed - .get_mut_or_insert(state_change.account_id.into()) - .set(*type_state, (*change_id).into()); - } - } - - push_tx - .send( - if http_request( - url, - serde_json::to_string(&response).unwrap(), - keys, - push_timeout, - ) - .await - { - Event::DeliverySuccess { id } - } else { - Event::DeliveryFailure { id, state_changes } - }, - ) - .await - .ok(); - }); - } -} - -async fn http_request( - url: String, - mut body: String, - keys: Option, - push_timeout: Duration, -) -> bool { - let client_builder = reqwest::Client::builder().timeout(push_timeout); - - #[cfg(feature = "test_mode")] - let client_builder = client_builder.danger_accept_invalid_certs(true); - - let mut client = client_builder - .build() - .unwrap_or_default() - .post(&url) - .header(CONTENT_TYPE, "application/json") - .header("TTL", "86400"); - - if let Some(keys) = keys { - match ece_encrypt(&keys.p256dh, &keys.auth, body.as_bytes()) - .map(|b| general_purpose::URL_SAFE.encode(b)) - { - Ok(body_) => { - body = body_; - client = client.header(CONTENT_ENCODING, "aes128gcm"); - } - Err(err) => { - // Do not reattempt if encryption fails. - - trc::event!( - PushSubscription(PushSubscriptionEvent::Error), - Details = "Failed to encrypt push subscription", - Url = url, - Reason = err - ); - return true; - } - } - } - - match client.body(body).send().await { - Ok(response) => { - if response.status().is_success() { - trc::event!(PushSubscription(PushSubscriptionEvent::Success), Url = url,); - - true - } else { - trc::event!( - PushSubscription(PushSubscriptionEvent::Error), - Details = "HTTP POST failed", - Url = url, - Code = response.status().as_u16(), - ); - - false - } - } - Err(err) => { - trc::event!( - PushSubscription(PushSubscriptionEvent::Error), - Details = "HTTP POST failed", - Url = url, - Reason = err.to_string() - ); - - false - } - } -} diff --git a/crates/smtp/src/inbound/milter/protocol.rs b/crates/smtp/src/inbound/milter/protocol.rs index f8bcba93..902d0b5d 100644 --- a/crates/smtp/src/inbound/milter/protocol.rs +++ b/crates/smtp/src/inbound/milter/protocol.rs @@ -47,7 +47,7 @@ pub const SMFIC_RCPT: u8 = b'R'; /* RCPT to */ pub const SMFIC_DATA: u8 = b'T'; /* DATA */ pub const SMFIC_UNKNOWN: u8 = b'U'; /* Any unknown command */ -impl<'x> Command<'x> { +impl Command<'_> { fn build(command: u8, len: u32) -> Vec { let mut buf = Vec::with_capacity(len as usize + 1 + std::mem::size_of::()); buf.extend_from_slice((len + 1).to_be_bytes().as_ref()); @@ -182,7 +182,7 @@ impl<'x> Command<'x> { } #[cfg(feature = "test_mode")] - pub fn deserialize(bytes: &'x [u8]) -> Command<'x> { + pub fn deserialize(bytes: &[u8]) -> Command<'_> { let mut reader = PacketReader::new(bytes); match reader.byte() { SMFIC_ABORT => Command::Abort, diff --git a/crates/utils/src/json/mod.rs b/crates/utils/src/json/mod.rs index 574fe6d1..9824cb37 100644 --- a/crates/utils/src/json/mod.rs +++ b/crates/utils/src/json/mod.rs @@ -1,4 +1,10 @@ -pub mod parser; +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + + pub mod parser; pub mod pointer; use downcast_rs::{Downcast, impl_downcast}; diff --git a/crates/utils/src/json/pointer.rs b/crates/utils/src/json/pointer.rs index a23ade97..befe58e2 100644 --- a/crates/utils/src/json/pointer.rs +++ b/crates/utils/src/json/pointer.rs @@ -1,4 +1,10 @@ -use super::{JsonPointerItem, JsonQueryable}; +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + + use super::{JsonPointerItem, JsonQueryable}; use std::hash::BuildHasher; use std::{collections::HashMap, slice::Iter}; diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 2e310906..31739f5f 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -26,6 +26,9 @@ jmap = { path = "../crates/jmap", features = ["test_mode", "enterprise"] } jmap_proto = { path = "../crates/jmap-proto" } imap = { path = "../crates/imap", features = ["test_mode"] } imap_proto = { path = "../crates/imap-proto" } +http = { path = "../crates/http", features = ["test_mode", "enterprise"] } +http_proto = { path = "../crates/http-proto" } +services = { path = "../crates/services", features = ["test_mode", "enterprise"] } pop3 = { path = "../crates/pop3", features = ["test_mode"] } smtp = { path = "../crates/smtp", features = ["test_mode", "enterprise"] } common = { path = "../crates/common", features = ["test_mode", "enterprise"] } diff --git a/tests/src/directory/oidc.rs b/tests/src/directory/oidc.rs index 876b4884..77d651e2 100644 --- a/tests/src/directory/oidc.rs +++ b/tests/src/directory/oidc.rs @@ -10,17 +10,17 @@ use std::sync::Arc; -use base64::{engine::general_purpose, Engine}; -use directory::{backend::internal::PrincipalField, QueryBy}; +use base64::{Engine, engine::general_purpose}; +use directory::{QueryBy, backend::internal::PrincipalField}; +use http_proto::{JsonProblemResponse, JsonResponse, ToHttpResponse}; use hyper::{Method, StatusCode}; -use jmap::api::{http::ToHttpResponse, JsonResponse}; use mail_send::Credentials; use serde_json::json; use trc::{AuthEvent, EventType}; use crate::{ directory::DirectoryTest, - http_server::{spawn_mock_http_server, HttpMessage}, + http_server::{HttpMessage, spawn_mock_http_server}, }; static TEST_TOKEN: &str = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ"; @@ -42,7 +42,7 @@ async fn oidc_directory() { match (req.method.clone(), req.uri.path().split('/').nth(1)) { (Method::GET, Some("userinfo")) => match req.headers.get("authorization") { Some(auth) if auth == &format!("Bearer {TEST_TOKEN}") => success_response, - Some(_) => StatusCode::UNAUTHORIZED.into_http_response(), + Some(_) => JsonProblemResponse(StatusCode::UNAUTHORIZED).into_http_response(), None => panic!("Missing Authorization header: {req:#?}"), }, (Method::POST, Some("introspect-none")) => { @@ -50,7 +50,7 @@ async fn oidc_directory() { if req.get_url_encoded("token").as_deref() == Some(TEST_TOKEN) { success_response } else { - StatusCode::UNAUTHORIZED.into_http_response() + JsonProblemResponse(StatusCode::UNAUTHORIZED).into_http_response() } } (Method::POST, Some("introspect-user-token")) => match req.headers.get("authorization") @@ -61,7 +61,7 @@ async fn oidc_directory() { { success_response } - Some(_) => StatusCode::UNAUTHORIZED.into_http_response(), + Some(_) => JsonProblemResponse(StatusCode::UNAUTHORIZED).into_http_response(), None => panic!("Missing Authorization header: {req:#?}"), }, (Method::POST, Some("introspect-token")) => match req.headers.get("authorization") { @@ -71,7 +71,7 @@ async fn oidc_directory() { { success_response } - Some(_) => StatusCode::UNAUTHORIZED.into_http_response(), + Some(_) => JsonProblemResponse(StatusCode::UNAUTHORIZED).into_http_response(), None => panic!("Missing Authorization header: {req:#?}"), }, (Method::POST, Some("introspect-basic")) => match req.headers.get("authorization") { @@ -85,7 +85,7 @@ async fn oidc_directory() { { success_response } - Some(_) => StatusCode::UNAUTHORIZED.into_http_response(), + Some(_) => JsonProblemResponse(StatusCode::UNAUTHORIZED).into_http_response(), None => panic!("Missing Authorization header: {req:#?}"), }, _ => panic!("Unexpected request: {:?}", req), diff --git a/tests/src/http_server.rs b/tests/src/http_server.rs index 6b0abecc..50990067 100644 --- a/tests/src/http_server.rs +++ b/tests/src/http_server.rs @@ -7,14 +7,14 @@ use std::sync::Arc; use ahash::AHashMap; -use common::{config::server::Listeners, listener::SessionData, Caches, Core, Data, Inner}; -use hyper::{body, server::conn::http1, service::service_fn, Method, Uri}; +use common::{Caches, Core, Data, Inner, config::server::Listeners, listener::SessionData}; +use http_proto::{HttpResponse, request::fetch_body}; +use hyper::{Method, Uri, body, server::conn::http1, service::service_fn}; use hyper_util::rt::TokioIo; -use jmap::api::{http::fetch_body, HttpResponse}; use tokio::sync::watch; use utils::config::Config; -use crate::{add_test_certs, AssertConfig}; +use crate::{AssertConfig, add_test_certs}; const MOCK_HTTP_SERVER: &str = r#" [server] diff --git a/tests/src/imap/mod.rs b/tests/src/imap/mod.rs index 8dc0236a..ccc418a5 100644 --- a/tests/src/imap/mod.rs +++ b/tests/src/imap/mod.rs @@ -36,13 +36,14 @@ use common::{ core::BuildServer, manager::boot::build_ipc, }; +use http::HttpSessionManager; use ::store::Stores; use ahash::AHashSet; use imap::core::ImapSessionManager; use imap_proto::ResponseType; -use jmap::{SpawnServices, api::JmapSessionManager}; use pop3::Pop3SessionManager; +use services::SpawnServices; use smtp::{SpawnQueueManager, core::SmtpSessionManager}; use tokio::{ io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines, ReadHalf, WriteHalf}, @@ -363,7 +364,7 @@ async fn init_imap_tests(store_id: &str, delete_if_exists: bool) -> IMAPTest { shutdown_rx, ), ServerProtocol::Http => server.spawn( - JmapSessionManager::new(inner.clone()), + HttpSessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, diff --git a/tests/src/jmap/auth_oauth.rs b/tests/src/jmap/auth_oauth.rs index fb6f5fe8..c15b5ce0 100644 --- a/tests/src/jmap/auth_oauth.rs +++ b/tests/src/jmap/auth_oauth.rs @@ -6,35 +6,35 @@ use std::time::{Duration, Instant}; -use base64::{engine::general_purpose, Engine}; -use biscuit::{jwk::JWKSet, SingleOrMultiple, JWT}; +use base64::{Engine, engine::general_purpose}; +use biscuit::{JWT, SingleOrMultiple, jwk::JWKSet}; use bytes::Bytes; use common::auth::oauth::{ introspect::OAuthIntrospect, oidc::StandardClaims, registration::{ClientRegistrationRequest, ClientRegistrationResponse}, }; -use imap_proto::ResponseType; -use jmap::auth::oauth::{ - auth::OAuthMetadata, openid::OpenIdMetadata, DeviceAuthResponse, ErrorType, OAuthCodeRequest, - TokenResponse, +use http::auth::oauth::{ + DeviceAuthResponse, ErrorType, OAuthCodeRequest, TokenResponse, auth::OAuthMetadata, + openid::OpenIdMetadata, }; +use imap_proto::ResponseType; use jmap_client::{ client::{Client, Credentials}, mailbox::query::Filter, }; use jmap_proto::types::id::Id; -use serde::{de::DeserializeOwned, Serialize}; +use serde::{Serialize, de::DeserializeOwned}; use store::ahash::AHashMap; use crate::{ directory::internal::TestInternalDirectory, imap::{ - pop::{self, Pop3Connection}, ImapConnection, Type, + pop::{self, Pop3Connection}, }, jmap::{ - assert_is_empty, delivery::SmtpConnection, mailbox::destroy_all_mailboxes, ManagementApi, + ManagementApi, assert_is_empty, delivery::SmtpConnection, mailbox::destroy_all_mailboxes, }, }; @@ -148,12 +148,14 @@ pub async fn test(params: &mut JMAPTest) { .await .unwrap(); assert_eq!(john_client.default_account_id(), john_id); - assert!(!john_client - .mailbox_query(None::, None::>) - .await - .unwrap() - .ids() - .is_empty()); + assert!( + !john_client + .mailbox_query(None::, None::>) + .await + .unwrap() + .ids() + .is_empty() + ); // Verify ID token using the JWK set let id_token = JWT::::new_encoded(&id_token) @@ -324,12 +326,14 @@ pub async fn test(params: &mut JMAPTest) { .await .unwrap(); assert_eq!(john_client.default_account_id(), john_id); - assert!(!john_client - .mailbox_query(None::, None::>) - .await - .unwrap() - .ids() - .is_empty()); + assert!( + !john_client + .mailbox_query(None::, None::>) + .await + .unwrap() + .ids() + .is_empty() + ); // Connecting using the refresh token should not work assert_unauthorized("https://127.0.0.1:8899", &refresh_token).await; diff --git a/tests/src/jmap/enterprise.rs b/tests/src/jmap/enterprise.rs index d9ff6fb7..b1d5e118 100644 --- a/tests/src/jmap/enterprise.rs +++ b/tests/src/jmap/enterprise.rs @@ -11,20 +11,20 @@ use std::{sync::Arc, time::Duration}; use common::{ + Core, Server, config::telemetry::{StoreTracer, TelemetrySubscriberType}, core::BuildServer, enterprise::{ - config::parse_metric_alerts, license::LicenseKey, undelete::DeletedBlob, Enterprise, - MetricStore, TraceStore, Undelete, + Enterprise, MetricStore, TraceStore, Undelete, config::parse_metric_alerts, + license::LicenseKey, undelete::DeletedBlob, }, telemetry::{ metrics::store::{Metric, MetricsStore, SharedMetricHistory}, tracers::store::{TracingQuery, TracingStore}, }, - Core, Server, }; +use http::management::enterprise::undelete::{UndeleteRequest, UndeleteResponse}; use imap_proto::ResponseType; -use jmap::api::management::enterprise::undelete::{UndeleteRequest, UndeleteResponse}; use store::{ rand::{self, Rng}, write::now, @@ -33,16 +33,16 @@ use trc::{ ipc::{bitset::Bitset, subscriber::SubscriberBuilder}, *, }; -use utils::config::{cron::SimpleCron, Config}; +use utils::config::{Config, cron::SimpleCron}; use crate::{ + AssertConfig, directory::internal::TestInternalDirectory, imap::{ImapConnection, Type}, jmap::delivery::SmtpConnection, - AssertConfig, }; -use super::{delivery::AssertResult, JMAPTest, ManagementApi}; +use super::{JMAPTest, ManagementApi, delivery::AssertResult}; const METRICS_CONFIG: &str = r#" [metrics.alerts.expected] @@ -114,12 +114,14 @@ pub async fn test(params: &mut JMAPTest) { config.assert_no_errors(); assert_ne!(core.enterprise.as_ref().unwrap().metrics_alerts.len(), 0); params.server.inner.shared_core.store(core.into()); - assert!(params - .server - .inner - .shared_core - .load() - .is_enterprise_edition()); + assert!( + params + .server + .inner + .shared_core + .load() + .is_enterprise_edition() + ); // Create test account params @@ -475,10 +477,7 @@ pub async fn insert_test_metrics(core: Arc) { Collector::update_event_counter(event_type, rand::rng().random_range(0..=100)) } - Collector::update_gauge( - MetricType::QueueCount, - rand::rng().random_range(0..=1000), - ); + Collector::update_gauge(MetricType::QueueCount, rand::rng().random_range(0..=1000)); Collector::update_gauge( MetricType::ServerMemory, rand::rng().random_range(100 * 1024 * 1024..=300 * 1024 * 1024), diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 2983ddfd..f113d9b2 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -25,15 +25,16 @@ use common::{ }; use email::message::delete::EmailDeletion; use enterprise::{EnterpriseCore, insert_test_metrics}; +use http::HttpSessionManager; use hyper::{Method, header::AUTHORIZATION}; use imap::core::ImapSessionManager; -use jmap::{SpawnServices, api::JmapSessionManager}; use jmap_client::client::{Client, Credentials}; use jmap_proto::{error::request::RequestError, types::id::Id}; use managesieve::core::ManageSieveSessionManager; use pop3::Pop3SessionManager; use reqwest::header; use serde::{Deserialize, Serialize, de::DeserializeOwned}; +use services::SpawnServices; use smtp::{SpawnQueueManager, core::SmtpSessionManager}; use store::{ @@ -625,7 +626,7 @@ async fn init_jmap_tests(store_id: &str, delete_if_exists: bool) -> JMAPTest { shutdown_rx, ), ServerProtocol::Http => server.spawn( - JmapSessionManager::new(inner.clone()), + HttpSessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, diff --git a/tests/src/jmap/push_subscription.rs b/tests/src/jmap/push_subscription.rs index fbcd0d0e..a224f433 100644 --- a/tests/src/jmap/push_subscription.rs +++ b/tests/src/jmap/push_subscription.rs @@ -6,36 +6,33 @@ use std::{ sync::{ - atomic::{AtomicBool, Ordering}, Arc, + atomic::{AtomicBool, Ordering}, }, time::Duration, }; -use base64::{engine::general_purpose, Engine}; -use common::{config::server::Listeners, listener::SessionData, Caches, Core, Data, Inner}; +use base64::{Engine, engine::general_purpose}; +use common::{Caches, Core, Data, Inner, config::server::Listeners, listener::SessionData}; use ece::EcKeyComponents; -use hyper::{body, header::CONTENT_ENCODING, server::conn::http1, service::service_fn, StatusCode}; +use http_proto::{HtmlResponse, ToHttpResponse, request::fetch_body}; +use hyper::{StatusCode, body, header::CONTENT_ENCODING, server::conn::http1, service::service_fn}; use hyper_util::rt::TokioIo; -use jmap::{ - api::{ - http::{fetch_body, ToHttpResponse}, - HtmlResponse, StateChangeResponse, - }, - push::ece::ece_encrypt, -}; use jmap_client::{mailbox::Role, push_subscription::Keys}; -use jmap_proto::types::{id::Id, type_state::DataType}; +use jmap_proto::{ + response::status::StateChangeResponse, + types::{id::Id, type_state::DataType}, +}; +use services::state_manager::ece::ece_encrypt; use store::ahash::AHashSet; use tokio::sync::mpsc; use utils::config::Config; use crate::{ - add_test_certs, + AssertConfig, add_test_certs, directory::internal::TestInternalDirectory, jmap::{assert_is_empty, mailbox::destroy_all_mailboxes, test_account_login}, - AssertConfig, }; use super::JMAPTest; @@ -313,9 +310,7 @@ impl common::listener::SessionManager for SessionManager { let is_encrypted = req .headers() .get(CONTENT_ENCODING) - .is_some_and(|encoding| { - encoding.to_str().unwrap() == "aes128gcm" - }); + .is_some_and(|encoding| encoding.to_str().unwrap() == "aes128gcm"); let body = fetch_body(&mut req, 1024 * 1024, 0).await.unwrap(); let message = serde_json::from_slice::(&if is_encrypted { ece::decrypt( diff --git a/tests/src/jmap/webhooks.rs b/tests/src/jmap/webhooks.rs index 3988cdfe..1e0dad34 100644 --- a/tests/src/jmap/webhooks.rs +++ b/tests/src/jmap/webhooks.rs @@ -6,17 +6,18 @@ use std::{ sync::{ - atomic::{AtomicBool, Ordering}, Arc, + atomic::{AtomicBool, Ordering}, }, time::Duration, }; -use base64::{engine::general_purpose::STANDARD, Engine}; +use base64::{Engine, engine::general_purpose::STANDARD}; use common::manager::webadmin::Resource; +use http_proto::{ToHttpResponse, request::fetch_body}; use hyper::{body, server::conn::http1, service::service_fn}; use hyper_util::rt::TokioIo; -use jmap::api::http::{fetch_body, ToHttpResponse}; +use jmap::api::ToJmapHttpResponse; use jmap_proto::error::request::RequestError; use ring::hmac; use store::parking_lot::Mutex; diff --git a/tests/src/smtp/inbound/antispam.rs b/tests/src/smtp/inbound/antispam.rs index 07de86ae..d9773e7c 100644 --- a/tests/src/smtp/inbound/antispam.rs +++ b/tests/src/smtp/inbound/antispam.rs @@ -7,22 +7,22 @@ use std::{ use ahash::{AHashMap, AHashSet}; use common::{ + Core, auth::AccessToken, config::spamfilter::SpamFilterAction, enterprise::{ + SpamFilterLlmConfig, llm::{ AiApiConfig, ChatCompletionChoice, ChatCompletionRequest, ChatCompletionResponse, Message, }, - SpamFilterLlmConfig, }, - Core, }; +use http_proto::{JsonResponse, ToHttpResponse}; use hyper::Method; -use jmap::api::{http::ToHttpResponse, JsonResponse}; use mail_auth::{ - dkim::Signature, dmarc::Policy, ArcOutput, DkimOutput, DkimResult, DmarcResult, IprevOutput, - IprevResult, SpfOutput, SpfResult, MX, + ArcOutput, DkimOutput, DkimResult, DmarcResult, IprevOutput, IprevResult, MX, SpfOutput, + SpfResult, dkim::Signature, dmarc::Policy, }; use mail_parser::MessageParser; use smtp::core::{Session, SessionAddress}; @@ -40,15 +40,15 @@ use spam_filter::{ subject::SpamFilterAnalyzeSubject, trusted_reply::SpamFilterAnalyzeTrustedReply, url::SpamFilterAnalyzeUrl, }, - modules::html::{html_to_tokens, HtmlToken}, + modules::html::{HtmlToken, html_to_tokens}, }; use store::Stores; use utils::config::Config; use crate::{ - http_server::{spawn_mock_http_server, HttpMessage}, + http_server::{HttpMessage, spawn_mock_http_server}, jmap::enterprise::EnterpriseCore, - smtp::{session::TestSession, DnsCache, TempDir, TestSMTP}, + smtp::{DnsCache, TempDir, TestSMTP, session::TestSession}, }; const CONFIG: &str = r#" diff --git a/tests/src/smtp/inbound/milter.rs b/tests/src/smtp/inbound/milter.rs index c708a2c0..55c8bf59 100644 --- a/tests/src/smtp/inbound/milter.rs +++ b/tests/src/smtp/inbound/milter.rs @@ -8,14 +8,14 @@ use std::{fs, net::SocketAddr, path::PathBuf, sync::Arc, time::Duration}; use ahash::AHashSet; use common::{ + Core, config::smtp::session::{Milter, MilterVersion, Stage}, expr::if_block::IfBlock, manager::webadmin::Resource, - Core, }; +use http_proto::{ToHttpResponse, request::fetch_body}; use hyper::{body, server::conn::http1, service::service_fn}; use hyper_util::rt::TokioIo; -use jmap::api::http::{fetch_body, ToHttpResponse}; use mail_auth::AuthenticatedMessage; use mail_parser::MessageParser; use serde::Deserialize; @@ -24,8 +24,8 @@ use smtp::{ inbound::{ hooks::{self, Request, SmtpResponse}, milter::{ - receiver::{FrameResult, Receiver}, Action, Command, Macros, MilterClient, Modification, Options, Response, + receiver::{FrameResult, Receiver}, }, }, }; @@ -38,9 +38,9 @@ use tokio::{ use utils::config::Config; use crate::smtp::{ - inbound::TestMessage, - session::{load_test_message, TestSession, VerifyResponse}, TempDir, TestSMTP, + inbound::TestMessage, + session::{TestSession, VerifyResponse, load_test_message}, }; #[derive(Debug, Deserialize)] @@ -377,38 +377,40 @@ fn milter_address_modifications() { ); // ChangeFrom - assert!(data - .apply_milter_modifications( + assert!( + data.apply_milter_modifications( vec![Modification::ChangeFrom { sender: "<>".to_string(), args: String::new() }], &parsed_test_message ) - .is_none()); + .is_none() + ); let addr = data.mail_from.as_ref().unwrap(); assert_eq!(addr.address_lcase, ""); assert_eq!(addr.dsn_info, None); assert_eq!(addr.flags, 0); // ChangeFrom with parameters - assert!(data - .apply_milter_modifications( + assert!( + data.apply_milter_modifications( vec![Modification::ChangeFrom { sender: "john@example.org".to_string(), args: "REQUIRETLS ENVID=abc123".to_string(), //"NOTIFY=SUCCESS,FAILURE ENVID=abc123\n".to_string() }], &parsed_test_message ) - .is_none()); + .is_none() + ); let addr = data.mail_from.as_ref().unwrap(); assert_eq!(addr.address_lcase, "john@example.org"); assert_ne!(addr.flags, 0); assert_eq!(addr.dsn_info, Some("abc123".to_string())); // Add recipients - assert!(data - .apply_milter_modifications( + assert!( + data.apply_milter_modifications( vec![ Modification::AddRcpt { recipient: "bill@example.org".to_string(), @@ -429,7 +431,8 @@ fn milter_address_modifications() { ], &parsed_test_message ) - .is_none()); + .is_none() + ); assert_eq!(data.rcpt_to.len(), 2); let addr = data.rcpt_to.first().unwrap(); assert_eq!(addr.address_lcase, "bill@example.org"); @@ -441,8 +444,8 @@ fn milter_address_modifications() { assert_eq!(addr.dsn_info, Some("Jane.Doe@Foobar.org".to_string())); // Remove recipients - assert!(data - .apply_milter_modifications( + assert!( + data.apply_milter_modifications( vec![ Modification::DeleteRcpt { recipient: "bill@example.org".to_string(), @@ -453,7 +456,8 @@ fn milter_address_modifications() { ], &parsed_test_message ) - .is_none()); + .is_none() + ); assert_eq!(data.rcpt_to.len(), 1); let addr = data.rcpt_to.last().unwrap(); assert_eq!(addr.address_lcase, "jane@foobar.org"); diff --git a/tests/src/smtp/management/queue.rs b/tests/src/smtp/management/queue.rs index 0a553b74..e8dd5e6e 100644 --- a/tests/src/smtp/management/queue.rs +++ b/tests/src/smtp/management/queue.rs @@ -9,16 +9,16 @@ use std::time::{Duration, Instant}; use ahash::{AHashMap, HashMap, HashSet}; use common::config::server::ServerProtocol; -use jmap::api::management::queue::Message; +use http::management::queue::Message; use mail_auth::MX; use mail_parser::DateTime; -use reqwest::{header::AUTHORIZATION, Method, StatusCode}; +use reqwest::{Method, StatusCode, header::AUTHORIZATION}; use crate::{ jmap::ManagementApi, - smtp::{session::TestSession, DnsCache, TestSMTP}, + smtp::{DnsCache, TestSMTP, session::TestSession}, }; -use smtp::queue::{manager::SpawnQueue, QueueId, Status}; +use smtp::queue::{QueueId, Status, manager::SpawnQueue}; const LOCAL: &str = r#" [storage] @@ -296,14 +296,15 @@ async fn manage_queue() { // Retry delivery for id in [id_map.get("e").unwrap(), id_map.get("f").unwrap()] { - assert!(api - .request::(Method::PATCH, &format!("/api/queue/messages/{id}",)) - .await - .unwrap() - .unwrap_data(),); + assert!( + api.request::(Method::PATCH, &format!("/api/queue/messages/{id}",)) + .await + .unwrap() + .unwrap_data(), + ); } - assert!(api - .request::( + assert!( + api.request::( Method::PATCH, &format!( "/api/queue/messages/{}?filter=example1.org&at=2200-01-01T00:00:00Z", @@ -312,7 +313,8 @@ async fn manage_queue() { ) .await .unwrap() - .unwrap_data()); + .unwrap_data() + ); // Expect delivery to john@foobar.org tokio::time::sleep(Duration::from_millis(100)).await; @@ -456,11 +458,12 @@ async fn manage_queue() { .len(), 3 ); - assert!(api - .request::(Method::DELETE, "/api/queue/messages?text=example2.com") - .await - .unwrap() - .unwrap_data()); + assert!( + api.request::(Method::DELETE, "/api/queue/messages?text=example2.com") + .await + .unwrap() + .unwrap_data() + ); tokio::time::sleep(Duration::from_millis(100)).await; assert_eq!( api.request::>(Method::GET, "/api/queue/messages") @@ -471,11 +474,12 @@ async fn manage_queue() { .len(), 2 ); - assert!(api - .request::(Method::DELETE, "/api/queue/messages") - .await - .unwrap() - .unwrap_data()); + assert!( + api.request::(Method::DELETE, "/api/queue/messages") + .await + .unwrap() + .unwrap_data() + ); tokio::time::sleep(Duration::from_millis(100)).await; assert_eq!( api.request::>(Method::GET, "/api/queue/messages") @@ -508,7 +512,9 @@ fn assert_timestamp(timestamp: &DateTime, expected: i64, ctx: &str, message: &Me let timestamp = timestamp.to_timestamp(); let diff = timestamp - expected; if ![-2, -1, 0, 1, 2].contains(&diff) { - panic!("Got timestamp {timestamp}, expected {expected} (diff {diff} for {ctx}) for {message:?}"); + panic!( + "Got timestamp {timestamp}, expected {expected} (diff {diff} for {ctx}) for {message:?}" + ); } } diff --git a/tests/src/smtp/management/report.rs b/tests/src/smtp/management/report.rs index 6937e2f0..f14718b3 100644 --- a/tests/src/smtp/management/report.rs +++ b/tests/src/smtp/management/report.rs @@ -12,23 +12,23 @@ use common::{ ipc::{DmarcEvent, PolicyType, TlsEvent}, }; -use jmap::api::management::queue::Report; +use http::management::queue::Report; use mail_auth::{ common::parse::TxtRecordParser, dmarc::Dmarc, mta_sts::TlsRpt, report::{ - tlsrpt::{FailureDetails, ResultType}, ActionDisposition, DmarcResult, Record, + tlsrpt::{FailureDetails, ResultType}, }, }; use reqwest::Method; use crate::{ jmap::ManagementApi, - smtp::{management::queue::List, TestSMTP}, + smtp::{TestSMTP, management::queue::List}, }; -use smtp::reporting::{scheduler::SpawnReport, SmtpReporting}; +use smtp::reporting::{SmtpReporting, scheduler::SpawnReport}; const CONFIG: &str = r#" [storage] @@ -225,11 +225,12 @@ async fn manage_reports() { assert!(ids.next().unwrap().is_some()); // Cancel all reports - assert!(api - .request::(Method::DELETE, "/api/queue/reports") - .await - .unwrap() - .unwrap_data()); + assert!( + api.request::(Method::DELETE, "/api/queue/reports") + .await + .unwrap() + .unwrap_data() + ); assert_eq!( api.request::>(Method::GET, "/api/queue/reports") .await diff --git a/tests/src/smtp/mod.rs b/tests/src/smtp/mod.rs index f3015eb1..10028256 100644 --- a/tests/src/smtp/mod.rs +++ b/tests/src/smtp/mod.rs @@ -11,18 +11,18 @@ use std::{ }; use common::{ + Core, Data, Inner, Server, config::{ server::{Listeners, ServerProtocol}, smtp::resolver::Tlsa, spamfilter::IpResolver, }, ipc::{QueueEvent, ReportingEvent}, - manager::boot::{build_ipc, IpcReceivers}, - Core, Data, Inner, Server, + manager::boot::{IpcReceivers, build_ipc}, }; -use jmap::api::JmapSessionManager; -use mail_auth::{common::resolver::IntoFqdn, Txt, MX}; +use http::HttpSessionManager; +use mail_auth::{MX, Txt, common::resolver::IntoFqdn}; use session::{DummyIo, TestSession}; use smtp::core::{Session, SmtpSessionManager}; use store::{BlobStore, Store, Stores}; @@ -265,7 +265,7 @@ impl TestSMTP { shutdown_rx, ), ServerProtocol::Http => server.spawn( - JmapSessionManager::new(self.server.inner.clone()), + HttpSessionManager::new(self.server.inner.clone()), self.server.inner.clone(), acceptor, shutdown_rx,