/* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ AssertConfig, smtp::session::{DummyIo, TestSession}, utils::{ account::Account, cleanup::{search_store_destroy, store_blob_expire_all, store_destroy}, registry::UnwrapRegistryId, storage::{RegistryEnvStores, assert_is_empty, build_data_store, wait_for_tasks}, temp_dir::TempDir, }, }; use ahash::AHashMap; use common::{ BuildServer, Caches, Core, Data, DavResources, Inner, Server, auth::RECOVERY_ADMIN_ID, config::{ server::{Listeners, ServerProtocol}, storage::Storage, telemetry::Telemetry, }, ipc::{QueueEvent, ReportingEvent}, manager::{ boot::{IpcReceivers, build_ipc}, defaults::BootstrapDefaults, }, psl, }; use email::message::metadata::MessageMetadata; use groupware::cache::GroupwareCache; use http::HttpSessionManager; use imap::core::ImapSessionManager; use jmap_client::client::Client; use managesieve::core::ManageSieveSessionManager; use pop3::Pop3SessionManager; use registry::{ schema::{ enums::{DataStoreType, EventPolicy, NetworkListenerProtocol, TracingLevel}, prelude::{Object, ObjectType, SocketAddr}, structs::{ Authentication, Certificate, Domain, NetworkListener, PublicText, SecretKeyFile, SecretText, SystemSettings, Tracer, TracerStdout, }, }, types::{EnumImpl, datetime::UTCDateTime, map::Map}, }; use services::{SpawnServices, broadcast::subscriber::spawn_broadcast_subscriber}; use smtp::{ SpawnQueueManager, core::{Session, SmtpSessionManager}, queue::{ manager::{Queue, SpawnQueue}, spool::{QueuedMessages, SmtpSpool}, }, reporting::scheduler::SpawnReport, }; use std::{path::PathBuf, str::FromStr, sync::Arc}; use store::{ RegistryStore, Store, ValueKey, registry::{RegistryQuery, bootstrap::Bootstrap, write::RegistryWrite}, write::{AlignedBytes, Archive, now}, }; use tokio::sync::{mpsc, watch}; use trc::EventType; use types::{collection::Collection, field::EmailField, id::Id}; pub struct TestServer { pub server: Server, pub accounts: AHashMap<&'static str, Account>, pub temp_dir: TempDir, pub queue_rx: mpsc::Receiver, pub report_rx: mpsc::Receiver, shutdown_tx: watch::Sender, reset: bool, } pub struct TestServerBuilder { bootstrap: Bootstrap, temp_dir: TempDir, http_listener_port: u16, reset: bool, logging_enabled: bool, capture_queue: bool, capture_reporting: bool, disable_services: bool, } impl TestServerBuilder { pub async fn new(test_name: &str) -> Self { let reset = std::env::var("NO_INSERT").is_err(); Self::new_with_role(test_name, "mail.example.org".to_string(), None, reset).await } pub async fn new_with_role( test_name: &str, hostname: String, node_role: Option, reset: bool, ) -> Self { let temp_dir = TempDir::new(test_name, reset); let path = temp_dir.path.to_string_lossy().to_string(); let data_store = build_data_store( std::env::var("STORE") .map(|store| DataStoreType::parse(&store).expect("Invalid store type")) .expect(concat!( "Missing or invalid store type. Try ", "running `STORE= cargo test`" )), &path, ); let store = Store::build(data_store).await.unwrap(); store.create_tables().await.unwrap(); // Delete old store if requested if reset { store_destroy(&store).await; } Self { bootstrap: Bootstrap::new( RegistryStore::new(&path, store, hostname, 1, node_role).await, ) .await, http_listener_port: 8899, temp_dir, reset, logging_enabled: false, capture_queue: false, capture_reporting: false, disable_services: false, } } pub async fn with_default_listeners(self) -> Self { let mut this = self; for (protocol, name, port, use_tls) in [ (NetworkListenerProtocol::Http, "jmap", 8899, true), (NetworkListenerProtocol::Imap, "imap", 9991, false), (NetworkListenerProtocol::Imap, "imaptls", 9992, true), (NetworkListenerProtocol::ManageSieve, "sieve", 4190, true), (NetworkListenerProtocol::Pop3, "pop3", 4110, true), (NetworkListenerProtocol::Lmtp, "lmtp-debug", 11200, false), ] { this = this.with_listener(protocol, name, port, use_tls).await; } this } pub async fn with_http_listener(self, port: u16) -> Self { self.with_listener(NetworkListenerProtocol::Http, "jmap", port, true) .await } pub async fn with_smtp_listener(self, port: u16) -> Self { self.with_listener(NetworkListenerProtocol::Smtp, "smtp", port, false) .await } pub async fn with_imap_listener(self, port: u16) -> Self { self.with_listener(NetworkListenerProtocol::Imap, "imap", port, false) .await } pub async fn with_dummy_tls_cert(self, sans: impl IntoIterator) -> Self { let mut cert_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); cert_path.push("resources"); let mut cert = cert_path.clone(); cert.push("tls_cert.pem"); let mut pk = cert_path.clone(); pk.push("tls_privatekey.pem"); self.with_object(Certificate { private_key: SecretText::File(SecretKeyFile { file_path: pk.to_string_lossy().to_string(), }), certificate: PublicText::File(SecretKeyFile { file_path: cert.to_string_lossy().to_string(), }), issuer: "Stalwart Test CA".to_string(), not_valid_after: UTCDateTime::from_timestamp((now() + 86400) as i64), subject_alternative_names: Map::new( sans.into_iter().map(|san| san.to_string()).collect(), ), ..Default::default() }) .await } pub async fn with_listener( mut self, protocol: NetworkListenerProtocol, name: &str, port: u16, tls_implicit: bool, ) -> Self { if protocol == NetworkListenerProtocol::Http { self.http_listener_port = port; } self.insert_object(NetworkListener { bind: Map::new(vec![ SocketAddr::from_str(&format!("0.0.0.0:{port}")).unwrap(), ]), name: name.to_string(), protocol, use_tls: true, tls_implicit, ..Default::default() }) .await; self } pub async fn with_object(self, object: impl Into) -> Self { self.insert_object(object).await; self } pub fn with_logging(mut self) -> Self { self.logging_enabled = true; self } pub fn capture_queue(mut self) -> Self { self.capture_queue = true; self } pub fn capture_reporting(mut self) -> Self { self.capture_reporting = true; self } pub fn disable_services(mut self) -> Self { self.disable_services = true; self } pub async fn insert_object(&self, object: impl Into) -> Id { self.bootstrap .registry .write(RegistryWrite::insert(&object.into())) .await .unwrap() .unwrap_id(trc::location!()) } pub async fn build(self) -> TestServer { self.build_with_opts(true).await } pub async fn build_with_opts(mut self, init_store: bool) -> TestServer { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); if init_store { // Register stores from environment self.bootstrap.registry.insert_stores_from_env().await; // Enable logging if requested let level = std::env::var("LOG") .map(|log| TracingLevel::parse(&log).expect("Invalid log level")) .ok(); // Add default domain let default_domain = psl::domain_str(self.bootstrap.registry.local_hostname()).unwrap(); let default_domain_id = self .insert_object(Domain { name: default_domain.to_string(), ..Default::default() }) .await; self.insert_object(SystemSettings { default_hostname: self.bootstrap.registry.local_hostname().to_string(), default_domain_id, ..Default::default() }) .await; self.insert_object(Tracer::Stdout(TracerStdout { enable: level.is_some() || self.logging_enabled, level: level.unwrap_or(TracingLevel::Info), ansi: true, multiline: false, events: Map::new( EventType::variants() .iter() .filter(|ev| { let ev = ev.as_str(); ev.starts_with("network.") || ev.starts_with("http.connection-") || ev == "telemetry.webhook-error" || ev == "http.request-body" || ev == "http.request-url" || ev == "tls.no-certificates-available" || ev == "store.cache-hit" }) .copied() .collect(), ), events_policy: EventPolicy::Exclude, ..Default::default() })) .await; } // Start listeners let mut servers = Listeners::parse(&mut self.bootstrap).await; servers.bind_and_drop_priv(&mut self.bootstrap); // Set HTTP port self.bootstrap.registry = self .bootstrap .registry .clone_with_port(self.http_listener_port); if init_store { // Add safe defaults if missing self.bootstrap.insert_safe_defaults().await; // Add directory if let Some(directory_id) = self .bootstrap .registry .query::>(RegistryQuery::new(ObjectType::Directory)) .await .unwrap() .first() { let mut auth = self .bootstrap .registry .object::(Id::singleton()) .await .unwrap() .unwrap(); auth.directory_id = Some(*directory_id); self.bootstrap .registry .write(RegistryWrite::insert(&auth.into())) .await .unwrap(); } } // Parse storage let storage = Storage::parse(&mut self.bootstrap).await; // Reset search store if init_store && self.reset { search_store_destroy(&storage.search).await; } // Parse telemetry let telemetry = Telemetry::parse(&mut self.bootstrap, &storage).await; // Parse components let core = Box::pin(Core::parse(&mut self.bootstrap, storage)).await; let data = Data::parse(&mut self.bootstrap).await; let cache = Caches::parse(&mut self.bootstrap).await; // Enable telemetry telemetry.enable(true); // Build inner let (ipc, mut ipc_rxs) = build_ipc(!core.storage.coordinator.is_none()); let inner = Arc::new(Inner { shared_core: core.into_shared(), data, ipc, cache, }); // Parse TCP acceptors servers .parse_tcp_acceptors(&mut self.bootstrap, inner.clone()) .await; // Start services self.bootstrap.assert_no_errors(); if !self.disable_services { ipc_rxs.spawn_services(inner.clone()); } // Spawn queue manager if not capturing let (_, mut queue_rx) = mpsc::channel(100); let (_, mut report_rx) = mpsc::channel(100); if !self.capture_queue && !self.capture_reporting { ipc_rxs.spawn_queue_manager(inner.clone()); } else { let queue_rx_ = ipc_rxs.queue_rx.take().unwrap(); let report_rx_ = ipc_rxs.report_rx.take().unwrap(); if !self.capture_queue { queue_rx_.spawn(inner.clone()); } else { queue_rx = queue_rx_; } if !self.capture_reporting { report_rx_.spawn(inner.clone()); } else { report_rx = report_rx_; } } // Spawn servers let (shutdown_tx, shutdown_rx) = servers.spawn(|server, acceptor, shutdown_rx| { match &server.protocol { ServerProtocol::Smtp | ServerProtocol::Lmtp => server.spawn( SmtpSessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Http => server.spawn( HttpSessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Imap => server.spawn( ImapSessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Pop3 => server.spawn( Pop3SessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::ManageSieve => server.spawn( ManageSieveSessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), }; }); // Start broadcast subscriber if !self.disable_services { spawn_broadcast_subscriber(inner.clone(), shutdown_rx); } tokio::time::sleep(std::time::Duration::from_millis(200)).await; let mut admin = Account::new( "admin", "popolna_zapora", &[], "Recovery Admin", Id::from(RECOVERY_ADMIN_ID), ); admin.http_listener_port = self.http_listener_port; TestServer { server: inner.build_server(), temp_dir: self.temp_dir, accounts: AHashMap::from_iter([("admin", admin)]), queue_rx, report_rx, shutdown_tx, reset: self.reset, } } } impl TestServer { pub fn reload_core(&mut self) { self.server = self.server.inner.build_server(); } pub fn account(&self, name: &str) -> &Account { self.accounts.get(name).unwrap() } pub async fn wait_for_tasks(&self) { wait_for_tasks(&self.server, false, false).await; } pub async fn wait_for_tasks_skip_failures(&self) { wait_for_tasks(&self.server, false, true).await; } pub async fn wait_for_tasks_skip_not_due(&self) { wait_for_tasks(&self.server, true, false).await; } pub async fn blob_expire_all(&self) { store_blob_expire_all(&self.server.core.storage.data).await; } pub async fn assert_is_empty(&self) { assert_is_empty(&self.server, true).await; } pub async fn cleanup(&self) { self.assert_is_empty().await; self.server.invalidate_all_local_caches(); } pub async fn destroy_store(&self) { store_destroy(self.server.store()).await; } pub fn is_reset(&self) -> bool { self.reset } pub fn tmp_dir(&self) -> &str { self.temp_dir.path.as_os_str().to_str().unwrap() } pub fn shutdown(&self) { let _ = self.shutdown_tx.send(true); } pub fn new_mta_session(&self) -> Session { Session::test(self.server.clone()) } pub fn new_mta_session_with_shutdown(&self) -> (Session, watch::Sender) { let (tx, rx) = watch::channel(true); (Session::test_with_shutdown(self.server.clone(), rx), tx) } pub async fn resources(&self, name: &'static str, collection: Collection) -> Arc { let account_id = self.account(name).id().document_id(); self.server .fetch_dav_resources(account_id, account_id, collection.into()) .await .unwrap() } pub async fn fetch_email(&self, account_id: u32, document_id: u32) -> Vec { let metadata_ = self .server .store() .get_value::>(ValueKey::property( account_id, Collection::Email, document_id, EmailField::Metadata, )) .await .unwrap() .unwrap(); self.server .blob_store() .get_blob( metadata_ .unarchive::() .unwrap() .blob_hash .0 .as_slice(), 0..usize::MAX, ) .await .unwrap() .unwrap() } pub async fn all_queued_messages(&self) -> QueuedMessages { self.server .next_event(&mut Queue::new( self.server.inner.clone(), mpsc::channel(100).1, )) .await } pub async fn destroy_all_mailboxes(&self, account: &Account) { self.wait_for_tasks().await; account.jmap_client().await.destroy_all_mailboxes().await; } pub async fn inner_with_rxs(&self) -> (Arc, IpcReceivers) { let (ipc, ipc_rxs) = build_ipc(false); let mut bp = Bootstrap::new_uninitialized(self.server.registry().clone()); ( Inner { shared_core: self.server.core.as_ref().clone().into_shared(), data: Default::default(), ipc, cache: Caches::parse(&mut bp).await, } .into(), ipc_rxs, ) } } impl Account { pub async fn destroy_all_mailboxes_for_account(&self, account_id: u32) { let mut client = self.jmap_client().await; client.set_default_account_id(Id::from(account_id)); client.destroy_all_mailboxes().await; } } pub trait DestroyAllMailboxes { fn destroy_all_mailboxes(&self) -> impl Future; } impl DestroyAllMailboxes for Client { async fn destroy_all_mailboxes(&self) { let mut request = self.build(); request.query_mailbox().arguments().sort_as_tree(true); let mut ids = request.send_query_mailbox().await.unwrap().take_ids(); ids.reverse(); for id in ids { self.mailbox_destroy(&id, true).await.unwrap(); } } }