diff --git a/crates/common/src/auth/permissions.rs b/crates/common/src/auth/permissions.rs index 5f200e80..525f5198 100644 --- a/crates/common/src/auth/permissions.rs +++ b/crates/common/src/auth/permissions.rs @@ -270,7 +270,7 @@ impl Default for DefaultPermissions { || name.starts_with("sysArchivedItem") || name.starts_with("sysAccountSettings") || name.starts_with("sysPublicKey") - || name.starts_with("sysSpamTrainingSample") + || (name.starts_with("sysSpamTrainingSample") && !name.contains("Create")) { default.user.push(permission); default.group.push(permission); diff --git a/crates/common/src/cache/principals.rs b/crates/common/src/cache/principals.rs index f4b96429..eba7abb6 100644 --- a/crates/common/src/cache/principals.rs +++ b/crates/common/src/cache/principals.rs @@ -16,7 +16,7 @@ use crate::{ }, config::smtp::auth::DkimSigner, expr::if_block::BootstrapExprExt, - network::{masked::MaskedAddress, mta::AddressResolver}, + network::mta::AddressResolver, storage::{ ObjectQuota, TenantQuota, encryption::{EncryptionMethod, parse_public_key}, @@ -544,8 +544,15 @@ impl Server { local_part = Cow::Borrowed(new_local_part); } } - if let Cow::Borrowed(addr) = &local_part - && let Some(masked_id) = MaskedAddress::parse(addr) + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + + #[cfg(feature = "enterprise")] + if self.is_enterprise_edition() + && let Cow::Borrowed(addr) = &local_part + && let Some(masked_id) = + crate::enterprise::masked::MaskedAddress::parse(addr) && let Some(masked_entry) = self .registry() .object::(Id::new(masked_id)) @@ -558,6 +565,7 @@ impl Server { { return Ok(Some(masked_entry.account_id.document_id())); } + // SPDX-SnippetEnd } let mut result = self diff --git a/crates/common/src/network/masked.rs b/crates/common/src/enterprise/masked.rs similarity index 92% rename from crates/common/src/network/masked.rs rename to crates/common/src/enterprise/masked.rs index bdad8a8d..2601d07a 100644 --- a/crates/common/src/network/masked.rs +++ b/crates/common/src/enterprise/masked.rs @@ -1,7 +1,11 @@ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * SPDX-License-Identifier: LicenseRef-SEL + * + * This file is subject to the Stalwart Enterprise License Agreement (SEL) and + * is NOT open source software. + * */ use store::write::now; diff --git a/crates/common/src/enterprise/mod.rs b/crates/common/src/enterprise/mod.rs index a5f89b94..b68cde9d 100644 --- a/crates/common/src/enterprise/mod.rs +++ b/crates/common/src/enterprise/mod.rs @@ -12,6 +12,7 @@ pub mod alerts; pub mod config; pub mod license; pub mod llm; +pub mod masked; use crate::{ Core, LogoCache, Server, config::groupware::CalendarTemplateVariable, expr::Expression, diff --git a/crates/common/src/network/mod.rs b/crates/common/src/network/mod.rs index 8295c301..67b4d082 100644 --- a/crates/common/src/network/mod.rs +++ b/crates/common/src/network/mod.rs @@ -29,7 +29,6 @@ pub mod dkim; pub mod dns; pub mod limiter; pub mod listen; -pub mod masked; pub mod mta; pub mod security; pub mod stream; diff --git a/crates/common/src/network/mta.rs b/crates/common/src/network/mta.rs index 924a54a4..5f642dd3 100644 --- a/crates/common/src/network/mta.rs +++ b/crates/common/src/network/mta.rs @@ -19,7 +19,7 @@ use crate::{ }, expr::{Variable, functions::ResolveVariable}, manager::SPAM_CLASSIFIER_KEY, - network::{RcptResolution, masked::MaskedAddress}, + network::RcptResolution, }; use directory::Recipient; use mail_auth::IpLookupStrategy; @@ -68,10 +68,16 @@ impl Server { } } - // Masked email resolution - if let Cow::Borrowed(addr) = &local_part - && let Some(masked_id) = MaskedAddress::parse(addr) + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + + #[cfg(feature = "enterprise")] + if self.is_enterprise_edition() + && let Cow::Borrowed(addr) = &local_part + && let Some(masked_id) = crate::enterprise::masked::MaskedAddress::parse(addr) { + // Masked email resolution return if let Some(masked_entry) = self .registry() .object::(Id::new(masked_id)) @@ -95,6 +101,7 @@ impl Server { Ok(RcptResolution::UnknownRecipient) }; } + // SPDX-SnippetEnd // Try resolving address from registry if let Some(address_type) = self diff --git a/crates/common/src/network/security.rs b/crates/common/src/network/security.rs index 20440d38..19589462 100644 --- a/crates/common/src/network/security.rs +++ b/crates/common/src/network/security.rs @@ -148,8 +148,14 @@ impl Security { #[cfg(not(feature = "test_mode"))] { // Add loopback addresses - allowed_ip_addresses.insert(IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)); - allowed_ip_addresses.insert(IpAddr::V6(std::net::Ipv6Addr::LOCALHOST)); + allowed_ip_addresses.insert(IpWithTtl::new( + IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), + u64::MAX, + )); + allowed_ip_addresses.insert(IpWithTtl::new( + IpAddr::V6(std::net::Ipv6Addr::LOCALHOST), + u64::MAX, + )); } let security = bp.setting_infallible::().await; diff --git a/crates/common/src/storage/encryption.rs b/crates/common/src/storage/encryption.rs index 9e40c8e2..9b4247ad 100644 --- a/crates/common/src/storage/encryption.rs +++ b/crates/common/src/storage/encryption.rs @@ -4,13 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::auth::EncryptionKeys; use mail_parser::decoders::base64::base64_decode; use registry::schema::structs::PublicKey; use sequoia_openpgp::{Cert, parse::Parse, policy::StandardPolicy, types::KeyFlags}; use std::borrow::Cow; -use crate::auth::EncryptionKeys; - const P: StandardPolicy<'static> = StandardPolicy::new(); #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/crates/email/src/message/crypto.rs b/crates/email/src/message/crypto.rs index dacc299a..87ed3527 100644 --- a/crates/email/src/message/crypto.rs +++ b/crates/email/src/message/crypto.rs @@ -4,10 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{collections::BTreeSet, io::Cursor}; - use aes::cipher::{BlockEncryptMut, KeyIvInit, block_padding::Pkcs7}; - use common::auth::{ ACCOUNT_FLAG_ENCRYPT_ALGO_AES256, ACCOUNT_FLAG_ENCRYPT_METHOD_PGP, ACCOUNT_FLAG_ENCRYPT_TRAIN_SPAM_FILTER, EncryptionKeys, @@ -30,6 +27,7 @@ use rasn_cms::{ }; use rsa::{Pkcs1v15Encrypt, RsaPublicKey, pkcs1::DecodeRsaPublicKey}; use sequoia_openpgp as openpgp; +use std::{collections::BTreeSet, io::Cursor}; #[derive(Debug)] pub enum EncryptMessageError { @@ -392,10 +390,10 @@ impl EncryptionFlags for u64 { fn encrypt(&self, key: &[u8], iv: &[u8], contents: &[u8]) -> Vec { if *self & ACCOUNT_FLAG_ENCRYPT_ALGO_AES256 != 0 { - cbc::Encryptor::::new(key.into(), iv.into()) + cbc::Encryptor::::new(key.into(), iv.into()) .encrypt_padded_vec_mut::(contents) } else { - cbc::Encryptor::::new(key.into(), iv.into()) + cbc::Encryptor::::new(key.into(), iv.into()) .encrypt_padded_vec_mut::(contents) } } diff --git a/crates/jmap/src/registry/get.rs b/crates/jmap/src/registry/get.rs index 0a64c3de..f3fc7e19 100644 --- a/crates/jmap/src/registry/get.rs +++ b/crates/jmap/src/registry/get.rs @@ -4,17 +4,13 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::registry::mapping::{ - RegistryGetResponse, - account::account_get, - archived_item::archived_item_get, - dkim::generate_dkim_public_key, - log::log_get, - queued_message::queued_message_get, - report::report_get, - spam_sample::spam_sample_get, - task::task_get, - telemetry::{metric_get, trace_get}, +use crate::registry::{ + EnterpriseRegistry, + mapping::{ + RegistryGetResponse, account::account_get, dkim::generate_dkim_public_key, log::log_get, + queued_message::queued_message_get, report::report_get, spam_sample::spam_sample_get, + task::task_get, + }, }; use common::{Server, auth::AccessToken}; use jmap_proto::{ @@ -54,6 +50,8 @@ impl RegistryGet for Server { mut request: GetRequest, access_token: &AccessToken, ) -> trc::Result> { + self.assert_enterprise_object(object_type)?; + let object_flags = object_type.flags(); let is_tenant_filtered = (object_flags & OBJ_FILTER_TENANT) != 0 && access_token.tenant_id().is_some(); @@ -289,12 +287,27 @@ impl RegistryGet for Server { | ObjectType::DmarcInternalReport | ObjectType::TlsInternalReport => report_get(get).await.map(|get| get.into_response()), - ObjectType::ArchivedItem => archived_item_get(get).await.map(|get| get.into_response()), + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + ObjectType::ArchivedItem => { + crate::registry::mapping::archived_item::archived_item_get(get) + .await + .map(|get| get.into_response()) + } + #[cfg(feature = "enterprise")] + ObjectType::Metric => crate::registry::mapping::telemetry::metric_get(get) + .await + .map(|get| get.into_response()), + #[cfg(feature = "enterprise")] + ObjectType::Trace => crate::registry::mapping::telemetry::trace_get(get) + .await + .map(|get| get.into_response()), + // SPDX-SnippetEnd ObjectType::SpamTrainingSample => { spam_sample_get(get).await.map(|get| get.into_response()) } - ObjectType::Metric => metric_get(get).await.map(|get| get.into_response()), - ObjectType::Trace => trace_get(get).await.map(|get| get.into_response()), ObjectType::Log => log_get(get).await.map(|get| get.into_response()), ObjectType::AccountSettings | ObjectType::Credential => { account_get(get).await.map(|get| get.into_response()) diff --git a/crates/jmap/src/registry/mapping/archived_item.rs b/crates/jmap/src/registry/mapping/archived_item.rs index e8aa7deb..83f0a3cc 100644 --- a/crates/jmap/src/registry/mapping/archived_item.rs +++ b/crates/jmap/src/registry/mapping/archived_item.rs @@ -1,7 +1,11 @@ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * SPDX-License-Identifier: LicenseRef-SEL + * + * This file is subject to the Stalwart Enterprise License Agreement (SEL) and + * is NOT open source software. + * */ use crate::{ @@ -223,6 +227,7 @@ pub(crate) async fn archived_item_set( .write(batch.build_all()) .await .caused_by(trc::location!())?; + set.server.notify_task_queue(); } Ok(set) diff --git a/crates/jmap/src/registry/mapping/masked_email.rs b/crates/jmap/src/registry/mapping/masked_email.rs index afa924a5..4d7fb3e3 100644 --- a/crates/jmap/src/registry/mapping/masked_email.rs +++ b/crates/jmap/src/registry/mapping/masked_email.rs @@ -1,11 +1,15 @@ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * SPDX-License-Identifier: LicenseRef-SEL + * + * This file is subject to the Stalwart Enterprise License Agreement (SEL) and + * is NOT open source software. + * */ use crate::registry::mapping::{ObjectResponse, RegistrySetResponse, ValidationResult}; -use common::network::masked::MaskedAddress; +use common::enterprise::masked::MaskedAddress; use jmap_proto::error::set::SetError; use rand::{Rng, distr::Alphanumeric}; use registry::{ diff --git a/crates/jmap/src/registry/mapping/mod.rs b/crates/jmap/src/registry/mapping/mod.rs index 38522da5..f98e85b3 100644 --- a/crates/jmap/src/registry/mapping/mod.rs +++ b/crates/jmap/src/registry/mapping/mod.rs @@ -23,17 +23,27 @@ use utils::map::vec_map::VecMap; pub mod account; pub mod action; -pub mod archived_item; pub mod dkim; pub mod log; -pub mod masked_email; pub mod principal; pub mod public_key; pub mod queued_message; pub mod report; pub mod spam_sample; pub mod task; + +// SPDX-SnippetBegin +// SPDX-FileCopyrightText: 2020 Stalwart Labs LLC +// SPDX-License-Identifier: LicenseRef-SEL +#[cfg(feature = "enterprise")] +pub mod masked_email; + +#[cfg(feature = "enterprise")] +pub mod archived_item; + +#[cfg(feature = "enterprise")] pub mod telemetry; +// SPDX-SnippetEnd pub(crate) struct RegistryGetResponse<'x> { pub server: &'x Server, diff --git a/crates/jmap/src/registry/mapping/public_key.rs b/crates/jmap/src/registry/mapping/public_key.rs index 036af889..042d0813 100644 --- a/crates/jmap/src/registry/mapping/public_key.rs +++ b/crates/jmap/src/registry/mapping/public_key.rs @@ -27,7 +27,7 @@ pub(crate) async fn validate_public_key( } } else { // Validate quotas - let num_masked = set + let num_keys = set .server .registry() .query::( @@ -36,17 +36,21 @@ pub(crate) async fn validate_public_key( .await? .0 as u32; let account = set.server.account(set.account_id).await?; - let masked_quota = set + let key_quota = set .server .object_quota(account.object_quotas(), StorageQuota::MaxPublicKeys); - if num_masked >= masked_quota { + if num_keys >= key_quota { return Ok(Err(SetError::over_quota().with_description(format!( "You have exceeded your quota of {} public keys.", - masked_quota + key_quota )))); } } + if !key.key.ends_with('\n') { + key.key.push('\n'); + } + match parse_public_key(key) { Ok(Some(_)) => Ok(Ok(response)), Ok(None) => Ok(Err(SetError::invalid_properties() diff --git a/crates/jmap/src/registry/mapping/spam_sample.rs b/crates/jmap/src/registry/mapping/spam_sample.rs index 6e4cf621..70bb2c89 100644 --- a/crates/jmap/src/registry/mapping/spam_sample.rs +++ b/crates/jmap/src/registry/mapping/spam_sample.rs @@ -63,7 +63,9 @@ pub(crate) async fn spam_sample_set( continue; }; if let Err(err) = sample.patch( - JsonPointerPatch::new(&JsonPointer::new(vec![])).with_create(true), + JsonPointerPatch::new(&JsonPointer::new(vec![])) + .with_create(true) + .with_can_set_account(!set.is_account_filtered), value, ) { set.response.not_created.append(id, err.into()); @@ -117,20 +119,20 @@ pub(crate) async fn spam_sample_set( continue; }; - let (Some(subject), Some(from)) = ( - message.subject().map(thread_name), - message - .from() - .and_then(|from| from.first().and_then(|addr| addr.address())), - ) else { + let subject = message.subject().map(thread_name).unwrap_or_default(); + let from = message + .from() + .and_then(|from| from.first().and_then(|addr| addr.address())) + .unwrap_or_default(); + if subject.is_empty() && from.is_empty() { set.response.not_created.append( id, SetError::invalid_properties() .with_property(Property::BlobId) - .with_description("Email message must have a subject and from header"), + .with_description("Email message must have a subject or a from header"), ); continue; - }; + } sample.subject = subject.to_string(); sample.from = from.to_lowercase(); diff --git a/crates/jmap/src/registry/mapping/task.rs b/crates/jmap/src/registry/mapping/task.rs index d2376a14..c02d74fc 100644 --- a/crates/jmap/src/registry/mapping/task.rs +++ b/crates/jmap/src/registry/mapping/task.rs @@ -23,7 +23,7 @@ use registry::{ pickle::Pickle, schema::{ enums::{TaskStatusType, TaskType}, - prelude::{Object, Property}, + prelude::Property, structs::Task, }, types::{ @@ -339,6 +339,8 @@ pub(crate) async fn task_set( due, })) .commit_point(); + + set.response.destroyed.push(id); } if !batch.is_empty() { @@ -370,7 +372,7 @@ pub(crate) async fn task_get( if let Some(task) = get .server .store() - .get_value::(ValueKey::from(ValueClass::TaskQueue( + .get_value::(ValueKey::from(ValueClass::TaskQueue( TaskQueueClass::Task { id: id.id() }, ))) .await? @@ -387,7 +389,7 @@ pub(crate) async fn task_get( pub(crate) async fn task_query( mut req: RegistryQueryResponse<'_>, ) -> trc::Result { - let mut due_from = 0u64; + let mut due_from = 100u64; let mut due_to = u64::MAX; req.request @@ -497,7 +499,7 @@ pub(crate) async fn task_query( async fn task_ids(server: &Server, max_results: usize) -> trc::Result> { let mut tasks = Vec::with_capacity(8); - let from_key = ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Due { id: 0, due: 0 })); + let from_key = ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Due { id: 0, due: 1 })); let to_key = ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Due { id: u64::MAX, due: u64::MAX, diff --git a/crates/jmap/src/registry/mapping/telemetry.rs b/crates/jmap/src/registry/mapping/telemetry.rs index a7e6e99b..c7bdf8c2 100644 --- a/crates/jmap/src/registry/mapping/telemetry.rs +++ b/crates/jmap/src/registry/mapping/telemetry.rs @@ -1,11 +1,13 @@ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * SPDX-License-Identifier: LicenseRef-SEL + * + * This file is subject to the Stalwart Enterprise License Agreement (SEL) and + * is NOT open source software. + * */ -use std::str::FromStr; - use crate::{ api::query::QueryResponseBuilder, registry::{ @@ -20,6 +22,7 @@ use registry::{ schema::prelude::{Object, Property}, types::datetime::UTCDateTime, }; +use std::str::FromStr; use store::{ IterateParams, ValueKey, registry::RegistryFilterOp, diff --git a/crates/jmap/src/registry/mod.rs b/crates/jmap/src/registry/mod.rs index 97130314..fea47e27 100644 --- a/crates/jmap/src/registry/mod.rs +++ b/crates/jmap/src/registry/mod.rs @@ -4,7 +4,42 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use common::Server; +use registry::schema::prelude::ObjectType; + pub mod get; pub mod mapping; pub mod query; pub mod set; + +pub trait EnterpriseRegistry { + fn assert_enterprise_object(&self, object_type: ObjectType) -> trc::Result<()>; +} + +impl EnterpriseRegistry for Server { + fn assert_enterprise_object(&self, object_type: ObjectType) -> trc::Result<()> { + if !matches!( + object_type, + ObjectType::MaskedEmail + | ObjectType::ArchivedItem + | ObjectType::Metric + | ObjectType::Trace + ) { + return Ok(()); + } + + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + if self.is_enterprise_edition() { + return Ok(()); + } + // SPDX-SnippetEnd + + Err(trc::JmapEvent::Forbidden.into_err().details(concat!( + "This feature is only available in the Enterprise edition. ", + "Obtain your trial license at https://license.stalw.art/trial." + ))) + } +} diff --git a/crates/jmap/src/registry/query.rs b/crates/jmap/src/registry/query.rs index ad70e3a6..6b6e588b 100644 --- a/crates/jmap/src/registry/query.rs +++ b/crates/jmap/src/registry/query.rs @@ -6,16 +6,13 @@ use crate::{ api::query::QueryResponseBuilder, - registry::mapping::{ - RegistryQueryResponse, - account::credential_query, - archived_item::archived_item_query, - log::log_query, - queued_message::queued_message_query, - report::report_query, - spam_sample::spam_sample_query, - task::task_query, - telemetry::{metric_query, trace_query}, + registry::{ + EnterpriseRegistry, + mapping::{ + RegistryQueryResponse, account::credential_query, log::log_query, + queued_message::queued_message_query, report::report_query, + spam_sample::spam_sample_query, task::task_query, + }, }, }; use common::{Server, auth::AccessToken}; @@ -55,6 +52,8 @@ impl RegistryQuery for Server { mut request: QueryRequest, access_token: &AccessToken, ) -> trc::Result { + self.assert_enterprise_object(object_type)?; + match object_type { ObjectType::ArfExternalReport | ObjectType::DmarcExternalReport @@ -69,7 +68,23 @@ impl RegistryQuery for Server { .await .and_then(|response| response.build()), - ObjectType::ArchivedItem => archived_item_query(RegistryQueryResponse { + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + ObjectType::ArchivedItem => { + super::mapping::archived_item::archived_item_query(RegistryQueryResponse { + server: self, + access_token, + object_type, + request, + }) + .await + .and_then(|response| response.build()) + } + + #[cfg(feature = "enterprise")] + ObjectType::Metric => super::mapping::telemetry::metric_query(RegistryQueryResponse { server: self, access_token, object_type, @@ -78,6 +93,16 @@ impl RegistryQuery for Server { .await .and_then(|response| response.build()), + #[cfg(feature = "enterprise")] + ObjectType::Trace => super::mapping::telemetry::trace_query(RegistryQueryResponse { + server: self, + access_token, + object_type, + request, + }) + .await + .and_then(|response| response.build()), + // SPDX-SnippetEnd ObjectType::SpamTrainingSample => spam_sample_query(RegistryQueryResponse { server: self, access_token, @@ -123,24 +148,6 @@ impl RegistryQuery for Server { .await .and_then(|response| response.build()), - ObjectType::Metric => metric_query(RegistryQueryResponse { - server: self, - access_token, - object_type, - request, - }) - .await - .and_then(|response| response.build()), - - ObjectType::Trace => trace_query(RegistryQueryResponse { - server: self, - access_token, - object_type, - request, - }) - .await - .and_then(|response| response.build()), - ObjectType::Action => Err(trc::JmapEvent::InvalidArguments .into_err() .details("Actions cannot be queried")), diff --git a/crates/jmap/src/registry/set.rs b/crates/jmap/src/registry/set.rs index 197f2a5c..27b1ea4e 100644 --- a/crates/jmap/src/registry/set.rs +++ b/crates/jmap/src/registry/set.rs @@ -4,22 +4,23 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::registry::mapping::{ - ObjectResponse, RegistrySetResponse, - account::account_set, - action::action_set, - archived_item::archived_item_set, - dkim::validate_dkim_signature, - map_bootstrap_error, - masked_email::validate_masked_email, - principal::{ - schedule_account_destruction, validate_account, validate_role, validate_tenant_quota, +use crate::registry::{ + EnterpriseRegistry, + mapping::{ + ObjectResponse, RegistrySetResponse, + account::account_set, + action::action_set, + dkim::validate_dkim_signature, + map_bootstrap_error, + principal::{ + schedule_account_destruction, validate_account, validate_role, validate_tenant_quota, + }, + public_key::validate_public_key, + queued_message::queued_message_set, + report::report_set, + spam_sample::spam_sample_set, + task::task_set, }, - public_key::validate_public_key, - queued_message::queued_message_set, - report::report_set, - spam_sample::spam_sample_set, - task::task_set, }; use common::{ Server, auth::AccessToken, cache::invalidate::CacheInvalidationBuilder, @@ -84,6 +85,8 @@ impl RegistrySet for Server { access_token: &AccessToken, session: &HttpSessionData, ) -> trc::Result> { + self.assert_enterprise_object(object_type)?; + let object_flags = object_type.flags(); let is_singleton = (object_flags & OBJ_SINGLETON) != 0; let has_account_id = (object_flags & OBJ_FILTER_ACCOUNT) != 0; @@ -395,8 +398,12 @@ impl RegistrySet for Server { ObjectInner::Role(role) => { validate_role(&set, role, modification.as_role()).await? } + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] ObjectInner::MaskedEmail(masked_email) => { - validate_masked_email( + crate::registry::mapping::masked_email::validate_masked_email( &set, masked_email, is_create, @@ -404,6 +411,7 @@ impl RegistrySet for Server { ) .await? } + // SPDX-SnippetEnd ObjectInner::PublicKey(key) => { validate_public_key(&set, key, modification.as_public_key()).await? } @@ -536,11 +544,11 @@ impl RegistrySet for Server { .await .caused_by(trc::location!())? .filter(|object| { - !(is_tenant_filtered + !((is_tenant_filtered && access_token.tenant_id().map(Id::from) != object.inner.member_tenant_id()) || (is_account_filtered - && object.inner.account_id() != Some(Id::from(set.account_id))) + && object.inner.account_id() != Some(Id::from(set.account_id)))) }) { match self @@ -585,8 +593,14 @@ impl RegistrySet for Server { | ObjectType::DmarcInternalReport | ObjectType::TlsInternalReport => report_set(set).await.map(|set| set.into_response()), - ObjectType::ArchivedItem => archived_item_set(set).await.map(|set| set.into_response()), - + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + ObjectType::ArchivedItem => super::mapping::archived_item::archived_item_set(set) + .await + .map(|set| set.into_response()), + // SPDX-SnippetEnd ObjectType::SpamTrainingSample => { spam_sample_set(set).await.map(|set| set.into_response()) } diff --git a/crates/services/src/task_manager/destroy_account.rs b/crates/services/src/task_manager/destroy_account.rs index a0e1088b..2113dfed 100644 --- a/crates/services/src/task_manager/destroy_account.rs +++ b/crates/services/src/task_manager/destroy_account.rs @@ -140,10 +140,6 @@ async fn destroy_account(server: &Server, task: &TaskDestroyAccount) -> trc::Res SearchIndex::Contacts, SearchIndex::Calendar, ] { - let c = println!( - "Unindexing search index {:?} for account {}", - index, account_id - ); server .search_store() .unindex(SearchQuery::new(index).with_account_id(account_id)) diff --git a/crates/services/src/task_manager/maintenance.rs b/crates/services/src/task_manager/maintenance.rs index 9f8120b9..9ec7a8cf 100644 --- a/crates/services/src/task_manager/maintenance.rs +++ b/crates/services/src/task_manager/maintenance.rs @@ -37,7 +37,7 @@ use registry::{ use smtp::reporting::index::ExternalReportIndex; use store::{ Serialize, ValueKey, - rand::{self, Rng}, + rand::{self}, registry::{RegistryFilter, RegistryQuery}, roaring::RoaringBitmap, write::{AlignedBytes, Archive, Archiver, BatchBuilder, RegistryClass, ValueClass, now}, @@ -106,10 +106,17 @@ async fn store_maintenance( .query::(RegistryQuery::new(ObjectType::Account)) .await? { + #[cfg(feature = "test_mode")] + let status = TaskStatus::at(now); + + #[cfg(not(feature = "test_mode"))] + let status = + TaskStatus::at(now + rand::Rng::random_range(&mut rand::rng(), 0..=300)); + batch.schedule_task(Task::AccountMaintenance(TaskAccountMaintenance { account_id: account_id.into(), maintenance_type, - status: TaskStatus::at(now + rand::rng().random_range(0..=300)), + status, })); if batch.is_large_batch() { diff --git a/crates/smtp/src/inbound/vrfy.rs b/crates/smtp/src/inbound/vrfy.rs index e6442500..d3ce4c9e 100644 --- a/crates/smtp/src/inbound/vrfy.rs +++ b/crates/smtp/src/inbound/vrfy.rs @@ -17,9 +17,7 @@ impl Session { .rcpt_resolve(&address.to_lowercase(), self.data.session_id) .await { - Ok( - RcptResolution::Accept | RcptResolution::Rewrite(_) | RcptResolution::Expand(_), - ) => { + Ok(RcptResolution::Accept | RcptResolution::Rewrite(_)) => { trc::event!( Smtp(SmtpEvent::Vrfy), SpanId = self.data.session_id, @@ -29,7 +27,11 @@ impl Session { self.write(format!("250 {}\r\n", address.as_ref()).as_bytes()) .await } - Ok(RcptResolution::UnknownRecipient) | Ok(RcptResolution::UnknownDomain) => { + Ok( + RcptResolution::UnknownRecipient + | RcptResolution::UnknownDomain + | RcptResolution::Expand(_), + ) => { trc::event!( Smtp(SmtpEvent::VrfyNotFound), SpanId = self.data.session_id, diff --git a/crates/spam-filter/src/modules/classifier.rs b/crates/spam-filter/src/modules/classifier.rs index 1088f274..d54e5955 100644 --- a/crates/spam-filter/src/modules/classifier.rs +++ b/crates/spam-filter/src/modules/classifier.rs @@ -84,6 +84,7 @@ pub struct TrainingSample { account_id: u32, } +#[derive(Debug)] struct TrainingTask { id: u64, sample: TrainingSample, @@ -273,7 +274,11 @@ impl SpamClassifier for Server { Elapsed = started.elapsed() ); - return Ok(()); + return if duplicate_samples.is_empty() { + Ok(()) + } else { + delete_samples(self, samples, duplicate_samples).await + }; } else if (trainer.reservoir.ham.total_seen < config.min_ham_samples) || (trainer.reservoir.spam.total_seen < config.min_spam_samples) { @@ -291,7 +296,11 @@ impl SpamClassifier for Server { Elapsed = started.elapsed() ); - return Ok(()); + return if duplicate_samples.is_empty() { + Ok(()) + } else { + delete_samples(self, samples, duplicate_samples).await + }; } // Balance classes if needed @@ -555,44 +564,10 @@ impl SpamClassifier for Server { // Remove samples marked for deletion if remove_entries { - let mut batch = BatchBuilder::new(); - for sample in samples.into_iter().chain(duplicate_samples.into_iter()) { - if let Some(until) = sample.remove { - batch - .with_account_id(sample.sample.account_id) - .clear(BlobOp::Link { - hash: sample.sample.hash, - to: BlobLink::Temporary { until }, - }) - .clear(ValueClass::Registry(RegistryClass::Item { - object_id, - item_id: sample.id, - })) - .clear(ValueClass::Registry(RegistryClass::Index { - index_id: Property::AccountId.to_id(), - object_id, - item_id: sample.id, - key: (sample.sample.account_id as u64).serialize(), - })); - - if batch.is_large_batch() { - self.store() - .write(batch.build_all()) - .await - .caused_by(trc::location!())?; - batch = BatchBuilder::new(); - } - } - } - if !batch.is_empty() { - self.store() - .write(batch.build_all()) - .await - .caused_by(trc::location!())?; - } + delete_samples(self, samples, duplicate_samples).await + } else { + Ok(()) } - - Ok(()) } async fn spam_classify(&self, ctx: &mut SpamFilterContext<'_>) -> trc::Result<()> { @@ -899,6 +874,53 @@ impl SpamClassifier for Server { } } +async fn delete_samples( + server: &Server, + samples: Vec, + duplicate_samples: Vec, +) -> trc::Result<()> { + let object_id = ObjectType::SpamTrainingSample.to_id(); + let mut batch = BatchBuilder::new(); + for sample in samples.into_iter().chain(duplicate_samples.into_iter()) { + if let Some(until) = sample.remove { + batch + .with_account_id(sample.sample.account_id) + .clear(BlobOp::Link { + hash: sample.sample.hash, + to: BlobLink::Temporary { until }, + }) + .clear(ValueClass::Registry(RegistryClass::Item { + object_id, + item_id: sample.id, + })) + .clear(ValueClass::Registry(RegistryClass::Index { + index_id: Property::AccountId.to_id(), + object_id, + item_id: sample.id, + key: (sample.sample.account_id as u64).serialize(), + })); + + if batch.is_large_batch() { + server + .store() + .write(batch.build_all()) + .await + .caused_by(trc::location!())?; + batch = BatchBuilder::new(); + batch.with_account_id(sample.sample.account_id); + } + } + } + if !batch.is_empty() { + server + .store() + .write(batch.build_all()) + .await + .caused_by(trc::location!())?; + } + Ok(()) +} + struct FhTrainJob { samples: Vec>, done: oneshot::Sender<()>, diff --git a/crates/store/src/dispatch/blob.rs b/crates/store/src/dispatch/blob.rs index a38b169e..8a2141f1 100644 --- a/crates/store/src/dispatch/blob.rs +++ b/crates/store/src/dispatch/blob.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{BlobStore, CompressionAlgo, Store}; +use crate::{BlobStore, CompressionAlgo, Store, U32_LEN}; use std::{ops::Range, time::Instant}; use trc::{AddContext, StoreEvent}; @@ -57,11 +57,11 @@ impl BlobStore { Size = result.as_ref().map_or(0, |data| data.len()), ); - let Some(data) = result else { + let Some(mut data) = result else { return Ok(None); }; - let data = match data.last().copied() { + let mut data = match data.last().copied() { Some(LZ4_MARKER) => { lz4_flex::decompress_size_prepended(data.get(..data.len() - 1).unwrap_or_default()) .map_err(|err| { @@ -72,17 +72,28 @@ impl BlobStore { })? } Some(NONE_MARKER) => { - trc::event!(Store(StoreEvent::BlobMissingMarker), Key = key); + if !data.is_empty() { + data.truncate(data.len() - 1); + } + data + } + Some(_) => { + trc::event!(Store(StoreEvent::BlobMissingMarker), Key = key); + data } - Some(_) => data, None => { return Ok(Some(data)); } }; - if range.end > data.len() { - Ok(Some(data)) + if range.start == 0 { + if range.end > data.len() { + Ok(Some(data)) + } else { + data.truncate(range.end); + Ok(Some(data)) + } } else { Ok(Some( data.get(range.start..range.end) @@ -106,8 +117,21 @@ impl BlobStore { uncompressed } CompressionAlgo::Lz4 => { - let mut compressed = lz4_flex::compress_prepend_size(data); - compressed.push(LZ4_MARKER); + let mut compressed = + vec![ + LZ4_MARKER; + lz4_flex::block::get_maximum_output_size(data.len()) + U32_LEN + 1 + ]; + + // Compress the data + let compressed_len = + lz4_flex::compress_into(data, &mut compressed[U32_LEN..]).unwrap(); + + // Prepend the length of the uncompressed data + compressed[..U32_LEN].copy_from_slice(&(data.len() as u32).to_le_bytes()); + + // Truncate to the actual size + compressed.truncate(compressed_len + U32_LEN + 1); compressed } }; diff --git a/crates/store/src/dispatch/store.rs b/crates/store/src/dispatch/store.rs index 924e2e66..4e598555 100644 --- a/crates/store/src/dispatch/store.rs +++ b/crates/store/src/dispatch/store.rs @@ -14,7 +14,7 @@ use crate::{ }, }; use compact_str::ToCompactString; -use std::{ops::Range, time::Instant}; +use std::time::Instant; use trc::{AddContext, StoreEvent}; use types::collection::Collection; @@ -375,75 +375,6 @@ impl Store { Ok(()) } - pub async fn get_blob(&self, key: &[u8], range: Range) -> trc::Result>> { - match self { - #[cfg(feature = "sqlite")] - Self::SQLite(store) => store.get_blob(key, range).await, - #[cfg(feature = "foundation")] - Self::FoundationDb(store) => store.get_blob(key, range).await, - #[cfg(feature = "postgres")] - Self::PostgreSQL(store) => store.get_blob(key, range).await, - #[cfg(feature = "mysql")] - Self::MySQL(store) => store.get_blob(key, range).await, - #[cfg(feature = "rocks")] - Self::RocksDb(store) => store.get_blob(key, range).await, - // SPDX-SnippetBegin - // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - // SPDX-License-Identifier: LicenseRef-SEL - #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] - Self::SQLReadReplica(store) => store.get_blob(key, range).await, - // SPDX-SnippetEnd - Self::None => Err(trc::StoreEvent::NotConfigured.into()), - } - .caused_by(trc::location!()) - } - - pub async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> { - match self { - #[cfg(feature = "sqlite")] - Self::SQLite(store) => store.put_blob(key, data).await, - #[cfg(feature = "foundation")] - Self::FoundationDb(store) => store.put_blob(key, data).await, - #[cfg(feature = "postgres")] - Self::PostgreSQL(store) => store.put_blob(key, data).await, - #[cfg(feature = "mysql")] - Self::MySQL(store) => store.put_blob(key, data).await, - #[cfg(feature = "rocks")] - Self::RocksDb(store) => store.put_blob(key, data).await, - // SPDX-SnippetBegin - // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - // SPDX-License-Identifier: LicenseRef-SEL - #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] - Self::SQLReadReplica(store) => store.put_blob(key, data).await, - // SPDX-SnippetEnd - Self::None => Err(trc::StoreEvent::NotConfigured.into()), - } - .caused_by(trc::location!()) - } - - pub async fn delete_blob(&self, key: &[u8]) -> trc::Result { - match self { - #[cfg(feature = "sqlite")] - Self::SQLite(store) => store.delete_blob(key).await, - #[cfg(feature = "foundation")] - Self::FoundationDb(store) => store.delete_blob(key).await, - #[cfg(feature = "postgres")] - Self::PostgreSQL(store) => store.delete_blob(key).await, - #[cfg(feature = "mysql")] - Self::MySQL(store) => store.delete_blob(key).await, - #[cfg(feature = "rocks")] - Self::RocksDb(store) => store.delete_blob(key).await, - // SPDX-SnippetBegin - // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - // SPDX-License-Identifier: LicenseRef-SEL - #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] - Self::SQLReadReplica(store) => store.delete_blob(key).await, - // SPDX-SnippetEnd - Self::None => Err(trc::StoreEvent::NotConfigured.into()), - } - .caused_by(trc::location!()) - } - pub async fn create_tables(&self) -> trc::Result<()> { match self { #[cfg(feature = "sqlite")] diff --git a/crates/store/src/registry/get.rs b/crates/store/src/registry/get.rs index 23eda7d4..fe3807e5 100644 --- a/crates/store/src/registry/get.rs +++ b/crates/store/src/registry/get.rs @@ -7,7 +7,10 @@ use crate::{ IterateParams, RegistryStore, SUBSPACE_REGISTRY, U16_LEN, U64_LEN, ValueKey, registry::RegistryObject, - write::{AnyClass, RegistryClass, ValueClass, key::KeySerializer}, + write::{ + AnyClass, RegistryClass, ValueClass, + key::{DeserializeBigEndian, KeySerializer}, + }, }; use registry::{ pickle::PickledStream, @@ -16,7 +19,6 @@ use registry::{ }; use trc::AddContext; use types::id::Id; -use utils::codec::leb128::Leb128Reader; impl RegistryStore { pub async fn get(&self, object_id: ObjectId) -> trc::Result> { @@ -76,17 +78,7 @@ impl RegistryStore { })), ), |key, value| { - let id = key - .get(U16_LEN..) - .and_then(|key| key.read_leb128::()) - .map(|r| r.0) - .ok_or_else(|| { - trc::EventType::Registry(trc::RegistryEvent::DeserializationError) - .into_err() - .caused_by(trc::location!()) - .details(object_type.as_str()) - .ctx(trc::Key::Key, key) - })?; + let id = key.deserialize_be_u64(U16_LEN)?; let mut stream = PickledStream::new(value); let object = T::unpickle(&mut stream).ok_or_else(|| { trc::EventType::Registry(trc::RegistryEvent::DeserializationError) diff --git a/crates/store/src/write/key.rs b/crates/store/src/write/key.rs index fb9a4ccf..bd96a4ae 100644 --- a/crates/store/src/write/key.rs +++ b/crates/store/src/write/key.rs @@ -302,7 +302,7 @@ impl ValueClass { }, ValueClass::Registry(registry) => match registry { RegistryClass::Item { object_id, item_id } => { - serializer.write(*object_id).write_leb128(*item_id) + serializer.write(*object_id).write(*item_id) } RegistryClass::IndexId { object_id, item_id } => { serializer.write(u16::MAX).write(*object_id).write(*item_id) diff --git a/tests/docker/INSTRUCTIONS.md b/tests/docker/INSTRUCTIONS.md new file mode 100644 index 00000000..6f51a454 --- /dev/null +++ b/tests/docker/INSTRUCTIONS.md @@ -0,0 +1,167 @@ +# Stalwart – Test Infrastructure + +Ephemeral Docker Compose stack for testing Stalwart against external services. +All data is lost on `docker compose down` – every restart is a clean slate. + +## Quick Start + +```bash +cd stalwart-test +docker compose up -d +``` + +Wait ~30 seconds for all services to initialize (Keycloak takes the longest). + +## Connection Reference + +| Service | Host | Port(s) | Credentials / Notes | +|----------------|-------------------|-----------------|--------------------------------------------| +| PostgreSQL | localhost | 5432 | `stalwart` / `stalwart`, db: `stalwart` | +| MySQL | localhost | 3306 | `stalwart` / `stalwart`, db: `stalwart` | +| FoundationDB | localhost | 4500 | Cluster file from container | +| Redis | localhost | 6379 | No auth | +| OpenSearch | localhost | 9200 | No auth, security plugin disabled | +| Meilisearch | localhost | 7700 | Master key: `stalwart-master-key` | +| MinIO (S3) | localhost | 9000 / 9001 | `minioadmin` / `minioadmin`, bucket: `stalwart` | +| Keycloak (OIDC)| localhost | 9080 | Admin: `admin` / `admin` | +| OpenLDAP | localhost | 389 / 636 (TLS) | Admin DN: `cn=admin,dc=stalwart,dc=test`, pw: `admin` | +| Pebble (ACME) | localhost | 14000 / 15000 | Self-signed TLS, auto-valid challenges | +| PowerDNS | localhost | 5300 / 8081 | API key: `stalwart-api-key` | +| NATS | localhost | 4222 / 8222 | No auth | + +## OIDC (Keycloak) Details + +- **OIDC Discovery**: `http://localhost:9080/realms/stalwart/.well-known/openid-configuration` +- **Token Endpoint**: `http://localhost:9080/realms/stalwart/protocol/openid-connect/token` +- **Client ID**: `stalwart` +- **Client Secret**: `stalwart-secret` +- **Realm**: `stalwart` + +### Test Users + +| Username | Password | Groups | +|---------------------------|--------------------------|----------------------------------------| +| john.doe@example.org | this is an OIDC password | sales@example.org | +| jane.smith@example.org | this is an OIDC password | sales@example.org, corporate@example.org | +| bill.foobar@example.org | this is an OIDC password | corporate@example.org | + +### Example: Get a Token + +```bash +curl -X POST http://localhost:9080/realms/stalwart/protocol/openid-connect/token \ + -d "grant_type=password" \ + -d "client_id=stalwart" \ + -d "client_secret=stalwart-secret" \ + -d "username=john.doe@example.org" \ + -d "password=this is an OIDC password" +``` + +## LDAP Details + +- **Base DN**: `dc=stalwart,dc=test` +- **Admin DN**: `cn=admin,dc=stalwart,dc=test` +- **Admin Password**: `admin` +- **Read-only DN**: `cn=readonly,dc=stalwart,dc=test` +- **Read-only Password**: `readonly` +- **User DN pattern**: `uid={username},ou=users,dc=stalwart,dc=test` + +### Test Users + +| DN | Mail | Password | +|------------------------------------------------|--------------------------|--------------------------| +| uid=john.doe,ou=users,dc=stalwart,dc=test | john.doe@example.org | this is an LDAP password | +| uid=jane.smith,ou=users,dc=stalwart,dc=test | jane.smith@example.org | this is an LDAP password | +| uid=bill.foobar,ou=users,dc=stalwart,dc=test | bill.foobar@example.org | this is an LDAP password | + +### Groups + +| DN | Mail | Members | +|---------------------------------------------|-------------------------|------------------| +| cn=sales,ou=groups,dc=stalwart,dc=test | sales@example.org | john.doe, jane.smith | +| cn=corporate,ou=groups,dc=stalwart,dc=test | corporate@example.org | bill.foobar, jane.smith | + +### Example: Search by Email + +```bash +ldapsearch -x -H ldap://localhost:389 \ + -D "cn=admin,dc=stalwart,dc=test" -w admin \ + -b "dc=stalwart,dc=test" "(mail=john.doe@example.org)" +``` + +## S3 (MinIO) Details + +- **Endpoint**: `http://localhost:9000` +- **Access Key**: `minioadmin` +- **Secret Key**: `minioadmin` +- **Bucket**: `stalwart` +- **Console**: `http://localhost:9001` +- **Region**: `us-east-1` (MinIO default) + +## DNS (PowerDNS) Details + +- **DNS port**: 5300 (TCP+UDP) +- **API**: `http://localhost:8081` (API key: `stalwart-api-key`) +- **Zone**: `stalwart.test` +- **TSIG key name**: `stalwart-update-key` +- **TSIG algorithm**: `hmac-sha256` +- **TSIG secret (base64)**: `c3RhbHdhcnQtdGVzdC10c2lnLXNlY3JldC1rZXkxMjM0NTY3ODkw` + +> **Note on SIG(0):** PowerDNS does not support SIG(0) authentication for RFC2136 +> updates. Only BIND has (limited) SIG(0) support. If you need to test SIG(0), +> a separate BIND instance would be required. + +### Example: Query TLSA Record + +```bash +dig @localhost -p 5300 _25._tcp.mail.stalwart.test TLSA +``` + +### Example: RFC2136 Dynamic Update + +```bash +nsupdate -y hmac-sha256:stalwart-update-key:c3RhbHdhcnQtdGVzdC10c2lnLXNlY3JldC1rZXkxMjM0NTY3ODkw < + start-dev --import-realm + tmpfs: + - /opt/keycloak/data:uid=1000,gid=1000 + + # --------------------------------------------------------------------------- + # OpenLDAP + # --------------------------------------------------------------------------- + openldap: + image: osixia/openldap:1.5.0 + depends_on: + cert-init: + condition: service_completed_successfully + environment: + LDAP_ORGANISATION: "Stalwart Test" + LDAP_DOMAIN: "stalwart.test" + LDAP_BASE_DN: "dc=stalwart,dc=test" + LDAP_ADMIN_PASSWORD: "admin" + LDAP_READONLY_USER: "true" + LDAP_READONLY_USER_USERNAME: "readonly" + LDAP_READONLY_USER_PASSWORD: "readonly" + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "cert.pem" + LDAP_TLS_KEY_FILENAME: "key.pem" + LDAP_TLS_CA_CRT_FILENAME: "cert.pem" + LDAP_TLS_VERIFY_CLIENT: "never" + LDAP_SEED_INTERNAL_LDIF_PATH: "/seed" + ports: + - "127.0.0.1:389:389" + - "127.0.0.1:636:636" + volumes: + - ./ldap/seed.ldif:/seed/50-stalwart.ldif:ro + - certs:/certs-shared:ro + entrypoint: [ "/bin/bash", "-c", "cp /certs-shared/* /container/service/slapd/assets/certs/ 2>/dev/null; exec /container/tool/run" ] + + # --------------------------------------------------------------------------- + # Pebble (ACME server) – ports 14000 (directory) + 15000 (management) + # --------------------------------------------------------------------------- + pebble: + image: ghcr.io/letsencrypt/pebble:latest + environment: + PEBBLE_VA_NOSLEEP: "1" + PEBBLE_VA_ALWAYS_VALID: "1" + ports: + - "127.0.0.1:14000:14000" + - "127.0.0.1:15000:15000" + volumes: + - ./pebble/pebble-config.json:/test/config/pebble-config.json:ro + command: -config /test/config/pebble-config.json + + # --------------------------------------------------------------------------- + # PowerDNS (DNS with TLSA + RFC2136) – port 5300 (moved from 53) + # --------------------------------------------------------------------------- + powerdns: + image: powerdns/pdns-auth-49:latest + environment: + PDNS_AUTH_API_KEY: stalwart-api-key + ports: + - "127.0.0.1:5300:53/tcp" + - "127.0.0.1:5300:53/udp" + - "127.0.0.1:8081:8081" + volumes: + - ./powerdns/pdns.conf:/etc/powerdns/pdns.d/stalwart.conf:ro + - ./powerdns/init-zone.sh:/etc/powerdns/init-zone.sh:ro + tmpfs: + - /var/lib/powerdns + + powerdns-init: + image: powerdns/pdns-auth-49:latest + depends_on: + - powerdns + volumes: + - ./powerdns/entrypoint.sh:/init.sh:ro + - ./powerdns/init-zone.sh:/etc/powerdns/init-zone.sh:ro + entrypoint: [ "bash", "/init.sh" ] + network_mode: "service:powerdns" + + # --------------------------------------------------------------------------- + # NATS (message queue, core mode) + # --------------------------------------------------------------------------- + nats: + image: nats:latest + ports: + - "127.0.0.1:4222:4222" + - "127.0.0.1:8222:8222" + command: "--addr 0.0.0.0 --port 4222 --http_port 8222" + +# ============================================================================= +# Shared volumes (ephemeral – docker compose down removes them) +# ============================================================================= +volumes: + certs: + driver: local + fdb-config: + driver: local diff --git a/tests/docker/keycloak/stalwart-realm.json b/tests/docker/keycloak/stalwart-realm.json new file mode 100644 index 00000000..e387a0cc --- /dev/null +++ b/tests/docker/keycloak/stalwart-realm.json @@ -0,0 +1,113 @@ +{ + "realm": "stalwart", + "enabled": true, + "registrationAllowed": false, + "loginWithEmailAllowed": true, + "duplicateEmailsAllowed": false, + "sslRequired": "none", + "clients": [ + { + "clientId": "stalwart", + "enabled": true, + "clientAuthenticatorType": "client-secret", + "secret": "stalwart-secret", + "redirectUris": ["*"], + "webOrigins": ["*"], + "publicClient": false, + "protocol": "openid-connect", + "directAccessGrantsEnabled": true, + "standardFlowEnabled": true, + "serviceAccountsEnabled": true, + "defaultClientScopes": ["openid", "email", "profile", "roles"], + "protocolMappers": [ + { + "name": "groups", + "protocol": "openid-connect", + "protocolMapper": "oidc-group-membership-mapper", + "consentRequired": false, + "config": { + "full.path": "false", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "groups", + "userinfo.token.claim": "true" + } + }, + { + "name": "email-claim", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "user.attribute": "email", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email", + "userinfo.token.claim": "true", + "jsonType.label": "String" + } + } + ] + } + ], + "users": [ + { + "username": "john.doe@example.org", + "enabled": true, + "email": "john.doe@example.org", + "emailVerified": true, + "firstName": "John", + "lastName": "Doe", + "credentials": [ + { + "type": "password", + "value": "this is an OIDC password", + "temporary": false + } + ], + "groups": ["/sales@example.org"] + }, + { + "username": "jane.smith@example.org", + "enabled": true, + "email": "jane.smith@example.org", + "emailVerified": true, + "firstName": "Jane", + "lastName": "Smith", + "credentials": [ + { + "type": "password", + "value": "this is an OIDC password", + "temporary": false + } + ], + "groups": ["/sales@example.org", "/corporate@example.org"] + }, + { + "username": "bill.foobar@example.org", + "enabled": true, + "email": "bill.foobar@example.org", + "emailVerified": true, + "firstName": "Bill", + "lastName": "Foobar", + "credentials": [ + { + "type": "password", + "value": "this is an OIDC password", + "temporary": false + } + ], + "groups": ["/corporate@example.org"] + } + ], + "groups": [ + { + "name": "sales@example.org", + "path": "/sales@example.org" + }, + { + "name": "corporate@example.org", + "path": "/corporate@example.org" + } + ] +} diff --git a/tests/docker/ldap/seed.ldif b/tests/docker/ldap/seed.ldif new file mode 100644 index 00000000..b65aa9a7 --- /dev/null +++ b/tests/docker/ldap/seed.ldif @@ -0,0 +1,69 @@ +# Organizational Units +dn: ou=users,dc=stalwart,dc=test +objectClass: organizationalUnit +ou: users + +dn: ou=groups,dc=stalwart,dc=test +objectClass: organizationalUnit +ou: groups + +# Users +dn: uid=john.doe,ou=users,dc=stalwart,dc=test +objectClass: inetOrgPerson +objectClass: posixAccount +objectClass: shadowAccount +uid: john.doe +cn: John Doe +sn: Doe +givenName: John +mail: john.doe@example.org +userPassword: this is an LDAP password +uidNumber: 10001 +gidNumber: 10001 +homeDirectory: /home/john.doe +loginShell: /bin/bash + +dn: uid=jane.smith,ou=users,dc=stalwart,dc=test +objectClass: inetOrgPerson +objectClass: posixAccount +objectClass: shadowAccount +uid: jane.smith +cn: Jane Smith +sn: Smith +givenName: Jane +mail: jane.smith@example.org +userPassword: this is an LDAP password +uidNumber: 10002 +gidNumber: 10002 +homeDirectory: /home/jane.smith +loginShell: /bin/bash + +dn: uid=bill.foobar,ou=users,dc=stalwart,dc=test +objectClass: inetOrgPerson +objectClass: posixAccount +objectClass: shadowAccount +uid: bill.foobar +cn: Bill Foobar +sn: Foobar +givenName: Bill +mail: bill.foobar@example.org +userPassword: this is an LDAP password +uidNumber: 10003 +gidNumber: 10003 +homeDirectory: /home/bill.foobar +loginShell: /bin/bash + +# Groups with email addresses +dn: cn=sales,ou=groups,dc=stalwart,dc=test +objectClass: groupOfNames +cn: sales +mail: sales@example.org +member: uid=john.doe,ou=users,dc=stalwart,dc=test +member: uid=jane.smith,ou=users,dc=stalwart,dc=test + +dn: cn=corporate,ou=groups,dc=stalwart,dc=test +objectClass: groupOfNames +cn: corporate +mail: corporate@example.org +member: uid=bill.foobar,ou=users,dc=stalwart,dc=test +member: uid=jane.smith,ou=users,dc=stalwart,dc=test diff --git a/tests/docker/pebble/pebble-config.json b/tests/docker/pebble/pebble-config.json new file mode 100644 index 00000000..8de8b18d --- /dev/null +++ b/tests/docker/pebble/pebble-config.json @@ -0,0 +1,17 @@ +{ + "pebble": { + "listenAddress": "0.0.0.0:14000", + "managementListenAddress": "0.0.0.0:15000", + "certificate": "/test/certs/localhost/cert.pem", + "privateKey": "/test/certs/localhost/key.pem", + "httpPort": 5002, + "tlsPort": 5001, + "ocspResponderURL": "", + "externalAccountBindingRequired": false, + "domainBlocklist": [], + "retryAfter": { + "authz": 3, + "order": 5 + } + } +} \ No newline at end of file diff --git a/tests/docker/powerdns/entrypoint.sh b/tests/docker/powerdns/entrypoint.sh new file mode 100755 index 00000000..30344aef --- /dev/null +++ b/tests/docker/powerdns/entrypoint.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +# Wait for PowerDNS to be ready (started by default entrypoint) +echo "Waiting for PowerDNS to start..." +for i in $(seq 1 30); do + if pdnsutil list-all-zones 2>/dev/null; then + break + fi + sleep 1 +done + +# Run zone initialization +bash /etc/powerdns/init-zone.sh diff --git a/tests/docker/powerdns/init-zone.sh b/tests/docker/powerdns/init-zone.sh new file mode 100755 index 00000000..890258a2 --- /dev/null +++ b/tests/docker/powerdns/init-zone.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -e + +# Wait for the SQLite database to be ready +sleep 2 + +# Create the zone +pdnsutil create-zone stalwart.test ns1.stalwart.test +pdnsutil set-kind stalwart.test native + +# Add basic records +pdnsutil add-record stalwart.test '' SOA 'ns1.stalwart.test. admin.stalwart.test. 2024010101 3600 900 604800 86400' +pdnsutil add-record stalwart.test '' NS 'ns1.stalwart.test.' +pdnsutil add-record stalwart.test 'ns1' A '127.0.0.1' +pdnsutil add-record stalwart.test '' A '127.0.0.1' +pdnsutil add-record stalwart.test '' MX '10 mail.stalwart.test.' +pdnsutil add-record stalwart.test 'mail' A '127.0.0.1' + +# Add a sample TLSA record +# Usage=3 (DANE-EE), Selector=1 (SubjectPublicKeyInfo), Matching=1 (SHA-256) +pdnsutil add-record stalwart.test '_25._tcp.mail' TLSA '3 1 1 0000000000000000000000000000000000000000000000000000000000000000' + +# Import static TSIG key for RFC2136 dynamic updates +# Key: stalwart-update-key / HMAC-SHA256 +# Base64 secret: c3RhbHdhcnQtdGVzdC10c2lnLXNlY3JldC1rZXkxMjM0NTY3ODkw +pdnsutil import-tsig-key stalwart-update-key hmac-sha256 'c3RhbHdhcnQtdGVzdC10c2lnLXNlY3JldC1rZXkxMjM0NTY3ODkw' +pdnsutil activate-tsig-key stalwart.test stalwart-update-key master +pdnsutil set-meta stalwart.test TSIG-ALLOW-DNSUPDATE stalwart-update-key +pdnsutil set-meta stalwart.test ALLOW-DNSUPDATE-FROM '0.0.0.0/0' + +echo "PowerDNS zone setup complete." +echo "TSIG key name: stalwart-update-key" +echo "TSIG algorithm: hmac-sha256" +echo "TSIG secret (b64): c3RhbHdhcnQtdGVzdC10c2lnLXNlY3JldC1rZXkxMjM0NTY3ODkw" diff --git a/tests/docker/powerdns/pdns.conf b/tests/docker/powerdns/pdns.conf new file mode 100644 index 00000000..fd12db81 --- /dev/null +++ b/tests/docker/powerdns/pdns.conf @@ -0,0 +1,2 @@ +dnsupdate=yes +allow-dnsupdate-from=0.0.0.0/0 diff --git a/tests/docker/scripts/gen-certs.sh b/tests/docker/scripts/gen-certs.sh new file mode 100755 index 00000000..1e9fdc3d --- /dev/null +++ b/tests/docker/scripts/gen-certs.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -e + +CERT_DIR=/certs + +if [ ! -f "$CERT_DIR/cert.pem" ]; then + echo "Generating self-signed certificate..." + openssl req -x509 -newkey rsa:2048 -nodes \ + -keyout "$CERT_DIR/key.pem" \ + -out "$CERT_DIR/cert.pem" \ + -days 365 \ + -subj "/CN=localhost/O=Stalwart Test/C=US" \ + -addext "subjectAltName=DNS:localhost,DNS:keycloak,DNS:openldap,DNS:pebble,DNS:*.stalwart.test,IP:127.0.0.1" + + # Create combined PEM for services that need it + cat "$CERT_DIR/cert.pem" "$CERT_DIR/key.pem" > "$CERT_DIR/combined.pem" + + # Create PKCS12 for Keycloak + openssl pkcs12 -export -in "$CERT_DIR/cert.pem" -inkey "$CERT_DIR/key.pem" \ + -out "$CERT_DIR/keystore.p12" -name localhost -password pass:changeit + + chmod 644 "$CERT_DIR"/* + echo "Certificates generated." +else + echo "Certificates already exist." +fi diff --git a/tests/docker/scripts/init-fdb.sh b/tests/docker/scripts/init-fdb.sh new file mode 100755 index 00000000..90ffd369 --- /dev/null +++ b/tests/docker/scripts/init-fdb.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -e + +fdbcli --exec "configure new single memory" +echo "FoundationDB configured." +exit 0 + diff --git a/tests/docker/scripts/init-minio.sh b/tests/docker/scripts/init-minio.sh new file mode 100755 index 00000000..69f8f4cd --- /dev/null +++ b/tests/docker/scripts/init-minio.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e + +# Wait for MinIO to be ready +echo "Waiting for MinIO..." +until mc alias set local http://minio:9000 minioadmin minioadmin 2>/dev/null; do + sleep 1 +done + +# Create the stalwart bucket +mc mb local/stalwart --ignore-existing +echo "MinIO bucket 'stalwart' created." diff --git a/tests/src/imap/antispam.rs b/tests/src/imap/antispam.rs index 27163392..889ddee9 100644 --- a/tests/src/imap/antispam.rs +++ b/tests/src/imap/antispam.rs @@ -244,549 +244,3 @@ pub async fn spam_training_samples(server: &Server) -> TrainingSamples { samples } - -pub const SPAM: [&str; 10] = [ - concat!( - "Subject: save up to = on life insurance\r\n\r\n wh", - "y spend more than you have to life quote savings e", - "nsuring your family s financial security is very i", - "mportant life quote savings makes buying life insu", - "rance simple and affordable we provide free access", - " to the very best companies and the lowest rates l", - "ife quote savings is fast easy and saves you money", - " let us help you get started with the best values ", - "in the country on new coverage you can save hundre", - "ds or even thousands of dollars by requesting a fr", - "ee quote from lifequote savings our service will t", - "ake you less than = minutes to complete shop ", - "and compare save up to = on all types of life", - " insurance hyperlink click here for your free quot", - "e protecting your family is the best investment yo", - "u ll ever make if you are in receipt of this email", - " in error and or wish to be removed from our list ", - "hyperlink please click here and type remove if you", - " reside in any state which prohibits e mail solici", - "tations for insurance please disregard this email\r\n", - " \r\n" - ), - concat!( - "Subject: a powerhouse gifting program\r\n\r\nyou don t ", - "want to miss get in with the founders the major pl", - "ayers are on this one for once be where the player", - "s are this is your private invitation experts are ", - "calling this the fastest way to huge cash flow eve", - "r conceived leverage = = into = NUM", - "BER over and over again the question here is you e", - "ither want to be wealthy or you don t which one ar", - "e you i am tossing you a financial lifeline and fo", - "r your sake i hope you grab onto it and hold on ti", - "ght for the ride of your life testimonials hear wh", - "at average people are doing their first few days w", - "e ve received = = in = day and we a", - "re doing that over and over again q s in al i m a ", - "single mother in fl and i ve received = NUMBE", - "R in the last = days d s in fl i was not sure", - " about this when i sent off my = = pledg", - "e but i got back = = the very next day l", - " l in ky i didn t have the money so i found myself", - " a partner to work this with we have received NUMB", - "ER = over the last = days i think i made", - " the right decision don t you k c in fl i pick up ", - "= = my first day and i they gave me free", - " leads and all the training you can too j w in ca ", - "announcing we will close your sales for you and he", - "lp you get a fax blast immediately upon your entry", - " you make the money free leads training don t wait", - " call now fax back to = = = = ", - "or call = = = = name__________", - "________________________phone_____________________", - "______________________ fax________________________", - "_____________email________________________________", - "____________ best time to call____________________", - "_____time zone____________________________________", - "____ this message is sent in compliance of the new", - " e mail bill per section = paragraph a =", - " c of s = further transmissions by the sender", - " of this email may be stopped at no cost to you by", - " sending a reply to this email address with the wo", - "rd remove in the subject line errors omissions and", - " exceptions excluded this is not spam i have compi", - "led this list from our replicate database relative", - " to seattle marketing group the gigt or turbo team", - " for the sole purpose of these communications your", - " continued inclusion is only by your gracious perm", - "ission if you wish to not receive this mail from m", - "e please send an email to tesrewinter with rem", - "ove in the subject and you will be deleted immedia", - "tely\r\n\r\n" - ), - concat!( - "Subject: help wanted \r\n\r\nwe are a = year old f", - "ortune = company that is growing at a tremend", - "ous rate we are looking for individuals who want t", - "o work from home this is an opportunity to make an", - " excellent income no experience is required we wil", - "l train you so if you are looking to be employed f", - "rom home with a career that has vast opportunities", - " then go we are looking for energetic and self", - " motivated people if that is you than click on the", - " link and fill out the form and one of our employe", - "ment specialist will contact you to be removed fro", - "m our link simple go to \r\n\r\n" - ), - concat!( - "Subject: tired of the bull out there\r\n\r\n want to st", - "op losing money want a real money maker receive NU", - "MBER = = = today experts are callin", - "g this the fastest way to huge cash flow ever conc", - "eived a powerhouse gifting program you don t want ", - "to miss we work as a team this is your private inv", - "itation get in with the founders this is where the", - " big boys play the major players are on this one f", - "or once be where the players are this is a system ", - "that will drive = = s to your doorstep i", - "n a short period of time leverage = = in", - "to = = over and over again the question ", - "here is you either want to be wealthy or you don t", - " which one are you i am tossing you a financial li", - "feline and for your sake i hope you grab onto it a", - "nd hold on tight for the ride of your life testimo", - "nials hear what average people are doing their fir", - "st few days we ve received = = in =", - " day and we are doing that over and over again q s", - " in al i m a single mother in fl and i ve received", - " = = in the last = days d s in fl i", - " was not sure about this when i sent off my =", - " = pledge but i got back = = the ve", - "ry next day l l in ky i didn t have the money so i", - " found myself a partner to work this with we have ", - "received = = over the last = days i", - " think i made the right decision don t you k c in ", - "fl i pick up = = my first day and i they", - " gave me free leads and all the training you can t", - "oo j w in ca this will be the most important call ", - "you make this year free leads training announcing ", - "we will close your sales for you and help you get ", - "a fax blast immediately upon your entry you make t", - "he money free leads training don t wait call now N", - "UMBER = = = print and fax to =", - " = = = or send an email requesting ", - "more information to successleads please includ", - "e your name and telephone number receive = NU", - "MBER free leads just for responding a = NUMBE", - "R value name___________________________________ ph", - "one___________________________________ fax________", - "_____________________________ email_______________", - "____________________ this message is sent in compl", - "iance of the new e mail bill per section = pa", - "ragraph a = c of s = further transmissio", - "ns by the sender of this email may be stopped at n", - "o cost to you by sending a reply to this email add", - "ress with the word remove in the subject line erro", - "rs omissions and exceptions excluded this is not s", - "pam i have compiled this list from our replicate d", - "atabase relative to seattle marketing group the gi", - "gt or turbo team for the sole purpose of these com", - "munications your continued inclusion is only by yo", - "ur gracious permission if you wish to not receive ", - "this mail from me please send an email to tesrewin", - "ter with remove in the subject and you will be", - " deleted immediately\r\n\r\n" - ), - concat!( - "Subject: cellular phone accessories \r\n\r\n all at bel", - "ow wholesale prices http = = = NUMB", - "ER = sites merchant sales hands free ear buds", - " = = phone holsters = = booste", - "r antennas only = = phone cases = N", - "UMBER car chargers = = face plates as lo", - "w as = = lithium ion batteries as low as", - " = = http = = = = NU", - "MBER sites merchant sales click below for accessor", - "ies on all nokia motorola lg nextel samsung qualco", - "mm ericsson audiovox phones at below wholesale pri", - "ces http = = = = = sites ", - "merchant sales if you need assistance please call ", - "us = = = to be removed from future ", - "mailings please send your remove request to remove", - " me now = thank you and have a super day\r\n", - " \r\n" - ), - concat!( - "Subject: conferencing made easy\r\n\r\n only = cen", - "ts per minute including long distance no setup fee", - "s no contracts or monthly fees call anytime from a", - "nywhere to anywhere connects up to = particip", - "ants simplicity in set up and administration opera", - "tor help available = = the highest quali", - "ty service for the lowest rate in the industry fil", - "l out the form below to find out how you can lower", - " your phone bill every month required input field ", - "name web address company name state business phone", - " home phone email address type of business to be r", - "emoved from our distribution lists please hyperlin", - "k click here\r\n\r\n" - ), - concat!( - "Subject: dear friend\r\n\r\n i am mrs sese seko widow o", - "f late president mobutu sese seko of zaire now kno", - "wn as democratic republic of congo drc i am moved ", - "to write you this letter this was in confidence co", - "nsidering my presentcircumstance and situation i e", - "scaped along with my husband and two of our sons g", - "eorge kongolo and basher out of democratic republi", - "c of congo drc to abidjan cote d ivoire where my f", - "amily and i settled while we later moved to settle", - "d in morroco where my husband later died of cancer", - " disease however due to this situation we decided ", - "to changed most of my husband s billions of dollar", - "s deposited in swiss bank and other countries into", - " other forms of money coded for safe purpose becau", - "se the new head of state of dr mr laurent kabila h", - "as made arrangement with the swiss government and ", - "other european countries to freeze all my late hus", - "band s treasures deposited in some european countr", - "ies hence my children and i decided laying low in ", - "africa to study the situation till when things get", - "s better like now that president kabila is dead an", - "d the son taking over joseph kabila one of my late", - " husband s chateaux in southern france was confisc", - "ated by the french government and as such i had to", - " change my identity so that my investment will not", - " be traced and confiscated i have deposited the su", - "m eighteen million united state dollars us = ", - "= = = with a security company for s", - "afekeeping the funds are security coded to prevent", - " them from knowing the content what i want you to ", - "do is to indicate your interest that you will assi", - "st us by receiving the money on our behalf acknowl", - "edge this message so that i can introduce you to m", - "y son kongolo who has the out modalities for the c", - "laim of the said funds i want you to assist in inv", - "esting this money but i will not want my identity ", - "revealed i will also want to buy properties and st", - "ock in multi national companies and to engage in o", - "ther safe and non speculative investments may i at", - " this point emphasise the high level of confidenti", - "ality which this business demands and hope you wil", - "l not betray the trust and confidence which i repo", - "se in you in conclusion if you want to assist us m", - "y son shall put you in the picture of the business", - " tell you where the funds are currently being main", - "tained and also discuss other modalities including", - " remunerationfor your services for this reason kin", - "dly furnish us your contact information that is yo", - "ur personal telephone and fax number for confident", - "ial regards mrs m sese seko\r\n\r\n" - ), - concat!( - "Subject: lowest rates available for term life insu", - "rance\r\n\r\n take a moment and fill out our online for", - "m to see the low rate you qualify for save up to N", - "UMBER from regular rates smokers accepted repr", - "esenting quality nationwide carriers act now to ea", - "sily remove your address from the list go to p", - "lease allow = = hours for removal\r\n\r\n" - ), - concat!( - "Subject: central bank of nigeria foreign remittanc", - "e \r\n\r\n dept tinubu square lagos nigeria email smith", - "_j =th of august = attn president ce", - "o strictly private business proposal i am mr johns", - "on s abu the bills and exchange director at the fo", - "reignremittance department of the central bank of ", - "nigeria i am writingyou this letter to ask for you", - "r support and cooperation to carrying thisbusiness", - " opportunity in my department we discovered abando", - "ned the sumof us = = = = thirt", - "y seven million four hundred thousand unitedstates", - " dollars in an account that belong to one of our f", - "oreign customers an american late engr john creek ", - "junior an oil merchant with the federal government", - " of nigeria who died along with his entire family ", - "of a wifeand two children in kenya airbus a= ", - "= flight kq= in november= since we ", - "heard of his death we have been expecting his next", - " of kin tocome over and put claims for his money a", - "s the heir because we cannotrelease the fund from ", - "his account unless someone applies for claims asth", - "e next of kin to the deceased as indicated in our ", - "banking guidelines unfortunately neither their fam", - "ily member nor distant relative hasappeared to cla", - "im the said fund upon this discovery i and other o", - "fficialsin my department have agreed to make busin", - "ess with you release the totalamount into your acc", - "ount as the heir of the fund since no one came for", - "it or discovered either maintained account with ou", - "r bank other wisethe fund will be returned to the ", - "bank treasury as unclaimed fund we have agreed tha", - "t our ratio of sharing will be as stated thus NUMB", - "ER for you as foreign partner and = for us th", - "e officials in my department upon the successful c", - "ompletion of this transfer my colleague and i will", - "come to your country and mind our share it is from", - " our = we intendto import computer accessorie", - "s into my country as way of recycling thefund to c", - "ommence this transaction we require you to immedia", - "tely indicateyour interest by calling me or sendin", - "g me a fax immediately on the abovetelefax and enc", - "lose your private contact telephone fax full namea", - "nd address and your designated banking co ordinate", - "s to enable us fileletter of claim to the appropri", - "ate department for necessary approvalsbefore the t", - "ransfer can be made note also this transaction mus", - "t be kept strictly confidential becauseof its natu", - "re nb please remember to give me your phone and fa", - "x no mr johnson smith abu irish linux users group ", - "ilug for un subscription information list ", - "maintainer listmaster \r\n\r\n" - ), - concat!( - "Subject: dear stuart\r\n\r\n are you tired of searching", - " for love in all the wrong places find love now at", - " browse through thousands of personals in ", - "your area join for free search e mail chat use", - " to meet cool guys and hot girls go = on ", - "= or use our private chat rooms click on the ", - "link to get started find love now you have rec", - "eived this email because you have registerd with e", - "mailrewardz or subscribed through one of our marke", - "ting partners if you have received this message in", - " error or wish to stop receiving these great offer", - "s please click the remove link above to unsubscrib", - "e from these mailings please click here \r\n\r\n" - ), -]; - -pub const HAM: [&str; 10] = [ - concat!( - "Message-ID: \r\nSubject: i have been", - " trying to research via sa mirrors and search engi", - "nes\r\n\r\nif a canned script exists giving clients acce", - "ss to their user_prefs options via a web based cgi", - " interface numerous isps provide this feature to c", - "lients but so far i can find nothing our configura", - "tion uses amavis postfix and clamav for virus filt", - "ering and procmail with spamassassin for spam filt", - "ering i would prefer not to have to write a script", - " myself but will appreciate any suggestions this U", - "RL email is sponsored by osdn tired of that same o", - "ld cell phone get a new here for free ________", - "_______________________________________ spamassass", - "in talk mailing list spamassassin talk \r\n\r\n" - ), - concat!( - "Message-ID: mid2@foobar.org\r\nSubject: hello\r\n\r\nhave y", - "ou seen and discussed this article and his approac", - "h thank you hell there are no rules here we re", - " trying to accomplish something thomas alva edison", - " this email is sponsored by osdn tired of that", - " same old cell phone get a new here for free _", - "______________________________________________ spa", - "massassin devel mailing list spamassassin devel UR", - "L \r\n\r\n" - ), - concat!( - "Message-ID: \r\nSubject: hi all apol", - "ogies for the possible silly question\r\n\r\ni don t thi", - "nk it is but but is eircom s adsl service nat ed a", - "nd what implications would that have for voip i kn", - "ow there are difficulties with voip or connecting ", - "to clients connected to a nat ed network from the ", - "internet wild i e machines with static real ips an", - "y help pointers would be helpful cheers rgrds bern", - "ard bernard tyers national centre for sensor resea", - "rch p = = = = e bernard tyers ", - " w l n= ______________________________", - "_________________ iiu mailing list iiu \r\n\r\n" - ), - concat!( - "Message-ID: \r\nSubject: can someone", - " explain\r\n\r\nwhat type of operating system solaris is", - " as ive never seen or used it i dont know wheather", - " to get a server from sun or from dell i would pre", - "fer a linux based server and sun seems to be the o", - "ne for that but im not sure if solaris is a distro", - " of linux or a completely different operating syst", - "em can someone explain kiall mac innes irish linux", - " users group ilug for un subscription info", - "rmation list maintainer listmaster \r\n\r\n" - ), - concat!( - "Message-ID: \r\nSubject: folks my fi", - "rst time posting\r\n\r\nhave a bit of unix experience bu", - "t am new to linux just got a new pc at home dell b", - "ox with windows xp added a second hard disk for li", - "nux partitioned the disk and have installed suse N", - "UMBER = from cd which went fine except it did", - "n t pick up my monitor i have a dell branded eNUMB", - "ERfpp = lcd flat panel monitor and a nvidia g", - "eforce= ti= video card both of which are", - " probably too new to feature in suse s default set", - " i downloaded a driver from the nvidia website and", - " installed it using rpm then i ran sax= as wa", - "s recommended in some postings i found on the net ", - "but it still doesn t feature my video card in the ", - "available list what next another problem i have a ", - "dell branded keyboard and if i hit caps lock twice", - " the whole machine crashes in linux not windows ev", - "en the on off switch is inactive leaving me to rea", - "ch for the power cable instead if anyone can help ", - "me in any way with these probs i d be really grate", - "ful i ve searched the net but have run out of idea", - "s or should i be going for a different version of ", - "linux such as redhat opinions welcome thanks a lot", - " peter irish linux users group ilug for un", - " subscription information list maintainer listmast", - "er \r\n\r\n" - ), - concat!( - "Message-ID: \r\nSubject: has anyone\r\n", - "\r\nseen heard of used some package that would let a ", - "random person go to a webpage create a mailing lis", - "t then administer that list also of course let ppl", - " sign up for the lists and manage their subscripti", - "ons similar to the old but i d like to have it", - " running on my server not someone elses chris ", - "\r\n\r\n" - ), - concat!( - "Message-ID: \r\nSubject: hi thank yo", - "u for the useful replies\r\n\r\ni have found some intere", - "sting tutorials in the ibm developer connection UR", - "L and registration is needed i will post the s", - "ame message on the web application security list a", - "s suggested by someone for now i thing i will use ", - "md= for password checking i will use the appr", - "oach described in secure programmin fo linux and u", - "nix how to i will separate the authentication modu", - "le so i can change its implementation at anytime t", - "hank you again mario torre please avoid sending me", - " word or powerpoint attachments see \r\n\r\n" - ), - concat!( - "Message-ID: \r\nSubject: hehe sorry\r\n", - "\r\nbut if you hit caps lock twice the computer crash", - "es theres one ive never heard before have you trye", - "d dell support yet i think dell computers prefer r", - "edhat dell provide some computers pre loaded with ", - "red hat i dont know for sure tho so get someone el", - "ses opnion as well as mine original message from i", - "lug admin mailto ilug admin on behalf of p", - "eter staunton sent = august = = NUM", - "BER to ilug subject ilug newbie seeks advice s", - "use = = folks my first time posting have", - " a bit of unix experience but am new to linux just", - " got a new pc at home dell box with windows xp add", - "ed a second hard disk for linux partitioned the di", - "sk and have installed suse = = from cd w", - "hich went fine except it didn t pick up my monitor", - " i have a dell branded e=fpp = lcd flat ", - "panel monitor and a nvidia geforce= ti= ", - "video card both of which are probably too new to f", - "eature in suse s default set i downloaded a driver", - " from the nvidia website and installed it using rp", - "m then i ran sax= as was recommended in some ", - "postings i found on the net but it still doesn t f", - "eature my video card in the available list what ne", - "xt another problem i have a dell branded keyboard ", - "and if i hit caps lock twice the whole machine cra", - "shes in linux not windows even the on off switch i", - "s inactive leaving me to reach for the power cable", - " instead if anyone can help me in any way with the", - "se probs i d be really grateful i ve searched the ", - "net but have run out of ideas or should i be going", - " for a different version of linux such as redhat o", - "pinions welcome thanks a lot peter irish linux use", - "rs group ilug for un subscription informat", - "ion list maintainer listmaster irish linux use", - "rs group ilug for un subscription informat", - "ion list maintainer listmaster \r\n\r\n" - ), - concat!( - "Message-ID: \r\nSubject: it will fun", - "ction as a router\r\n\r\nif that is what you wish it eve", - "n looks like the modem s embedded os is some kind ", - "of linux being that it has interesting interfaces ", - "like eth= i don t use it as a router though i", - " just have it do the absolute minimum dsl stuff an", - "d do all the really fun stuff like pppoe on my lin", - "ux box also the manual tells you what the default ", - "password is don t forget to run pppoe over the alc", - "atel speedtouch =i as in my case you have to ", - "have a bridge configured in the router modem s sof", - "tware this lists your vci values etc also does any", - "one know if the high end speedtouch with = et", - "hernet ports can act as a full router or do i stil", - "l need to run a pppoe stack on the linux box regar", - "ds vin irish linux users group ilug for un", - " subscription information list maintainer listmast", - "er irish linux users group ilug for un", - " subscription information list maintainer listmast", - "er \r\n\r\n" - ), - concat!( - "Message-ID: \r\nSubject: all is it ", - "just me\r\n\r\nor has there been a massive increase in t", - "he amount of email being falsely bounced around th", - "e place i ve already received email from a number ", - "of people i don t know asking why i am sending the", - "m email these can be explained by servers from rus", - "sia and elsewhere coupled with the false emails i ", - "received myself it s really starting to annoy me a", - "m i the only one seeing an increase in recent week", - "s martin martin whelan déise design tel NUMBE", - "R = our core product déiseditor allows organ", - "isations to publish information to their web site ", - "in a fast and cost effective manner there is no ne", - "ed for a full time web developer as the site can b", - "e easily updated by the organisations own staff in", - "stant updates to keep site information fresh sites", - " which are updated regularly bring users back visi", - "t for a demonstration déiseditor managing you", - "r information ____________________________________", - "___________ iiu mailing list iiu ,0\r\n" - ), -]; - -const TEST: [&str; 3] = [ - concat!( - "Subject: save up to = on life insurance\r\n\r\nwhy ", - "spend more than you have to life quote savings ens", - "uring your family s financial security is very imp", - "ortant life quote savings makes buying life insura", - "nce simple and affordable we provide free access t", - "o the very best companies and the lowest rates lif", - "e quote savings is fast easy and saves you money l", - "et us help you get started with the best values in", - " the country on new coverage you can save hundreds", - " or even thousands of dollars by requesting a free", - " quote from lifequote savings our service will tak", - "e you less than = minutes to complete shop an", - "d compare save up to = on all types of life i", - "nsurance hyperlink click here for your free quote ", - "protecting your family is the best investment you ", - "ll ever make if you are in receipt of this email i", - "n error and or wish to be removed from our list hy", - "perlink please click here and type remove if you r", - "eside in any state which prohibits e mail solicita", - "tions for insurance please disregard this email\r\n" - ), - concat!( - "Subject: can someone explain\r\n\r\nwhat type of operati", - "ng system solaris is as ive never seen or used it ", - "i dont know wheather to get a server from sun or f", - "rom dell i would prefer a linux based server and s", - "un seems to be the one for that but im not sure if", - " solaris is a distro of linux or a completely diff", - "erent operating system can someone explain kiall m", - "ac innes irish linux users group ilug for ", - "un subscription information list maintainer listma", - "ster \r\n" - ), - concat!( - "Subject: classifier test\r\n\r\nthis is a novel text tha", - "t the sgd classifier has never seen before, it s", - "hould be classified as ham or non-ham\r\n" - ), -]; diff --git a/tests/src/jmap/mail/antispam.rs b/tests/src/jmap/mail/antispam.rs deleted file mode 100644 index d4f4511c..00000000 --- a/tests/src/jmap/mail/antispam.rs +++ /dev/null @@ -1,175 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use std::sync::Arc; - -use email::mailbox::{DRAFTS_ID, INBOX_ID, JUNK_ID}; -use store::write::now; -use types::{id::Id, keyword::Keyword}; - -use crate::{imap::antispam::*, jmap::JMAPTest}; - -pub async fn test(test: &mut TestServer) { - println!("Running Email Spam classifier tests..."); - let account = test.account("jdoe@example.com"); - let client = account.jmap_client().await; - let account_id = account.id().document_id(); - - // Make sure there are no training samples - spam_delete_samples(¶ms.server).await; - assert_eq!(spam_training_samples(¶ms.server).await.total_count, 0); - - // Import samples - let mut spam_ids = vec![]; - let mut ham_ids = vec![]; - for (idx, samples) in [&SPAM, &HAM].into_iter().enumerate() { - let is_spam = idx == 0; - for (num, sample) in samples.iter().enumerate() { - let mut mailbox_ids = vec![]; - let mut keywords = vec![]; - - if num == 0 { - if is_spam { - mailbox_ids.push(Id::from(JUNK_ID).to_string()); - keywords.push(Keyword::Junk.to_string()); - } else { - mailbox_ids.push(Id::from(INBOX_ID).to_string()); - keywords.push(Keyword::NotJunk.to_string()); - } - } else { - mailbox_ids.push(Id::from(DRAFTS_ID).to_string()); - } - - let mail_id = client - .email_import( - sample.as_bytes().to_vec(), - &mailbox_ids, - Some(&keywords), - None, - ) - .await - .unwrap() - .take_id(); - if is_spam { - spam_ids.push(mail_id); - } else { - ham_ids.push(mail_id); - } - } - } - let samples = spam_training_samples(¶ms.server).await; - assert_eq!(samples.ham_count, 1); - assert_eq!(samples.spam_count, 1); - - // Train the classifier via JMAP - for (ids, is_spam) in [(&spam_ids, true), (&ham_ids, false)] { - for (idx, id) in ids.iter().skip(1).enumerate() { - // Set keywords and mailboxes - let mut request = client.build(); - let req = request.set_email().update(id); - if idx < 5 || !is_spam { - // Update via keywords - let keyword = if is_spam { - Keyword::Junk - } else { - Keyword::NotJunk - } - .to_string(); - req.keywords([&keyword]); - } else { - // Update via mailbox - let mailbox_id = if is_spam { JUNK_ID } else { INBOX_ID }; - req.mailbox_ids([&Id::from(mailbox_id).to_string()]); - } - - request.send_set_email().await.unwrap().updated(id).unwrap(); - } - } - let samples = spam_training_samples(¶ms.server).await; - assert_eq!(samples.ham_count, 10); - assert_eq!(samples.spam_count, 10); - - // Reclassifying an email should not add a new sample - let mut request = client.build(); - request - .set_email() - .update(&ham_ids[0]) - .keywords([Keyword::Junk.to_string()]); - request - .send_set_email() - .await - .unwrap() - .updated(&ham_ids[0]) - .unwrap(); - let samples = spam_training_samples(¶ms.server).await; - assert_eq!(samples.ham_count, 9); - assert_eq!(samples.spam_count, 11); - assert_eq!(samples.samples.len(), 20); - let hold_for = params - .server - .core - .spam - .classifier - .as_ref() - .unwrap() - .hold_samples_for; - assert!(hold_for > 2 * 86400); - let hold_until = now() + hold_for; - let hold_range = (hold_until - 86400)..=hold_until; - assert!(samples.samples.iter().all(|s| s.account_id == account_id - && s.remove.is_none() - && hold_range.contains(&s.until))); - - // Purging blobs should not remove training samples - params - .server - .store() - .purge_blobs(params.server.blob_store().clone()) - .await - .unwrap(); - let samples = spam_training_samples(¶ms.server).await; - assert_eq!(samples.ham_count, 9); - assert_eq!(samples.spam_count, 11); - assert_eq!(samples.samples.len(), 20); - - // Extend hold period so a new training sample is generated - let old_core = params.server.core.clone(); - let mut new_core = old_core.as_ref().clone(); - new_core.spam.classifier.as_mut().unwrap().hold_samples_for += 2 * 86400; - params.server.inner.shared_core.store(Arc::new(new_core)); - - // Reclassifying an email will now add a new sample - let mut request = client.build(); - request - .set_email() - .update(&ham_ids[0]) - .keywords([Keyword::NotJunk.to_string()]); - request - .send_set_email() - .await - .unwrap() - .updated(&ham_ids[0]) - .unwrap(); - let samples = spam_training_samples(¶ms.server).await; - assert_eq!(samples.ham_count, 10); - assert_eq!(samples.spam_count, 11); - assert_eq!(samples.samples.len(), 21); - - // Blob purge should remove the duplicated sample - params - .server - .store() - .purge_blobs(params.server.blob_store().clone()) - .await - .unwrap(); - let samples = spam_training_samples(¶ms.server).await; - assert_eq!(samples.ham_count, 10); - assert_eq!(samples.spam_count, 10); - assert_eq!(samples.samples.len(), 20); - - test.destroy_all_mailboxes(account).await; - test.assert_is_empty().await;; -} diff --git a/tests/src/jmap/server/enterprise.rs b/tests/src/jmap/server/enterprise.rs index 8b709969..a1abe125 100644 --- a/tests/src/jmap/server/enterprise.rs +++ b/tests/src/jmap/server/enterprise.rs @@ -70,13 +70,6 @@ message = "this should not have happened" "#; -const RAW_MESSAGE: &str = "From: john@example.com -To: john@example.com -Subject: undelete test - -test -"; - pub async fn test(test: &mut TestServer) { // Enable Enterprise println!("Running Enterprise tests..."); @@ -166,35 +159,6 @@ pub async fn test(test: &mut TestServer) { ); } -pub trait EnterpriseCore { - fn enable_enterprise(self) -> Self; -} - -impl EnterpriseCore for Core { - fn enable_enterprise(mut self) -> Self { - self.enterprise = Enterprise { - license: LicenseKey { - valid_to: now() + 3600, - valid_from: now() - 3600, - domain: String::new(), - accounts: 100, - }, - undelete: None, - trace_store: None, - metrics_store: None, - metrics_alerts: vec![], - logo_url: None, - ai_apis: Default::default(), - spam_filter_llm: None, - template_calendar_alarm: None, - template_scheduling_email: None, - template_scheduling_web: None, - } - .into(); - self - } -} - async fn alerts(server: &Server) { // Make sure the required metrics are set to 0 assert_eq!( @@ -384,111 +348,6 @@ async fn metrics(test: &mut TestServer) { ); } -async fn undelete(test: &mut TestServer) { - // Authenticate - let mut imap = ImapConnection::connect(b"_x ").await; - imap.authenticate("jdoe@example.com", "12345").await; - - // Insert test message - imap.send("STATUS INBOX (MESSAGES)").await; - imap.assert_read(Type::Tagged, ResponseType::Ok) - .await - .assert_contains("MESSAGES 0"); - imap.send(&format!("APPEND INBOX {{{}}}", RAW_MESSAGE.len())) - .await; - imap.assert_read(Type::Continuation, ResponseType::Ok).await; - imap.send_untagged(RAW_MESSAGE).await; - imap.assert_read(Type::Tagged, ResponseType::Ok).await; - - // Make sure the message is there - imap.send("STATUS INBOX (MESSAGES)").await; - imap.assert_read(Type::Tagged, ResponseType::Ok) - .await - .assert_contains("MESSAGES 1"); - imap.send("SELECT INBOX").await; - imap.assert_read(Type::Tagged, ResponseType::Ok).await; - - // Fetch message body - imap.send("FETCH 1 BODY[]").await; - imap.assert_read(Type::Tagged, ResponseType::Ok) - .await - .assert_contains("Subject: undelete test"); - - // Delete and expunge message - imap.send("STORE 1 +FLAGS (\\Deleted)").await; - imap.assert_read(Type::Tagged, ResponseType::Ok).await; - imap.send("EXPUNGE").await; - imap.assert_read(Type::Tagged, ResponseType::Ok).await; - - // Logout and reconnect - imap.send("LOGOUT").await; - imap.assert_read(Type::Tagged, ResponseType::Ok).await; - let mut imap = ImapConnection::connect(b"_x ").await; - imap.authenticate("jdoe@example.com", "12345").await; - - // Make sure the message is gone - imap.send("STATUS INBOX (MESSAGES)").await; - imap.assert_read(Type::Tagged, ResponseType::Ok) - .await - .assert_contains("MESSAGES 0"); - - // Query undelete API - let api = ManagementApi::new(8899, "admin", "secret"); - api.get::("/api/store/purge/account/jdoe@example.com") - .await - .unwrap(); - test.wait_for_tasks().await; - tokio::time::sleep(Duration::from_millis(200)).await; - let deleted = api - .get::>("/api/store/undelete/jdoe@example.com") - .await - .unwrap() - .unwrap_data() - .items; - assert_eq!(deleted.len(), 1); - let deleted = deleted.into_iter().next().unwrap(); - match deleted.item { - DeletedItemResponse::Email { from, subject, .. } => { - assert_eq!(subject.as_ref(), "undelete test"); - assert_eq!(from.as_ref(), "john@example.com"); - } - other => { - panic!("Unexpected deleted item response: {:?}", other); - } - } - - // Undelete - let result = api - .post::>( - "/api/store/undelete/jdoe@example.com", - &vec![UndeleteRequest { - hash: deleted.hash, - collection: "email".to_string(), - time: deleted.deleted_at, - cancel_deletion: deleted.expires_at.into(), - }], - ) - .await - .unwrap() - .unwrap_data(); - assert_eq!(result, vec![UndeleteResponse::Success]); - - // Make sure the message is back - imap.send("STATUS INBOX (MESSAGES)").await; - imap.assert_read(Type::Tagged, ResponseType::Ok) - .await - .assert_contains("MESSAGES 1"); - - imap.send("SELECT INBOX").await; - imap.assert_read(Type::Tagged, ResponseType::Ok).await; - - // Fetch message body - imap.send("FETCH 1 BODY[]").await; - imap.assert_read(Type::Tagged, ResponseType::Ok) - .await - .assert_contains("Subject: undelete test"); -} - pub async fn insert_test_metrics(core: Arc) { let store = core.storage.data.clone(); store.purge_metrics(Duration::from_secs(0)).await.unwrap(); diff --git a/tests/src/system/antispam.rs b/tests/src/system/antispam.rs new file mode 100644 index 00000000..95badd76 --- /dev/null +++ b/tests/src/system/antispam.rs @@ -0,0 +1,814 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::utils::{jmap::JmapUtils, server::TestServer}; +use email::mailbox::{DRAFTS_ID, INBOX_ID, JUNK_ID}; +use registry::{ + schema::{ + enums::{Permission, TaskSpamFilterMaintenanceType, TaskStoreMaintenanceType}, + prelude::{ObjectType, Property}, + structs::{ + Permissions, PermissionsList, SpamTrainingSample, Task, TaskSpamFilterMaintenance, + TaskStatus, TaskStoreMaintenance, + }, + }, + types::map::Map, +}; +use serde_json::json; +use store::write::now; +use types::{id::Id, keyword::Keyword}; + +pub async fn test(test: &mut TestServer) { + println!("Running Email Spam classifier tests..."); + + // Create test accounts + let admin = test.account("admin@example.org"); + let account = test + .create_user_account( + "admin@example.org", + "jdoe@example.org", + "this is a very strong password", + &[], + ) + .await; + let other_account = test + .create_user_account( + "admin@example.org", + "jane@example.org", + "this is a very strong password", + &[], + ) + .await; + let client = account.jmap_client().await; + let account_id = account.id().document_id(); + + // Make sure there are no spam training samples + admin + .registry_destroy_all(ObjectType::SpamTrainingSample) + .await; + assert!( + admin + .registry_query( + ObjectType::SpamTrainingSample, + Vec::<(&str, &str)>::new(), + Vec::<&str>::new(), + ) + .await + .ids() + .next() + .is_none() + ); + + // Import samples + let mut spam_ids = vec![]; + let mut ham_ids = vec![]; + for (idx, samples) in [&SPAM, &HAM].into_iter().enumerate() { + let is_spam = idx == 0; + for (num, sample) in samples.iter().enumerate() { + let mut mailbox_ids = vec![]; + let mut keywords = vec![]; + + if num == 0 { + if is_spam { + mailbox_ids.push(Id::from(JUNK_ID).to_string()); + keywords.push(Keyword::Junk.to_string()); + } else { + mailbox_ids.push(Id::from(INBOX_ID).to_string()); + keywords.push(Keyword::NotJunk.to_string()); + } + } else { + mailbox_ids.push(Id::from(DRAFTS_ID).to_string()); + } + + let mail_id = client + .email_import( + sample.as_bytes().to_vec(), + &mailbox_ids, + Some(&keywords), + None, + ) + .await + .unwrap() + .take_id(); + if is_spam { + spam_ids.push(mail_id); + } else { + ham_ids.push(mail_id); + } + } + } + let samples = account.spam_training_samples().await; + assert_eq!(samples.iter().filter(|x| !x.1.is_spam).count(), 1); + assert_eq!(samples.iter().filter(|x| x.1.is_spam).count(), 1); + + // Other users should no see the training samples + let samples = other_account.spam_training_samples().await; + assert_eq!(samples.iter().filter(|x| !x.1.is_spam).count(), 0); + assert_eq!(samples.iter().filter(|x| x.1.is_spam).count(), 0); + + // The admin user should see all training samples + let samples = admin.spam_training_samples().await; + assert_eq!(samples.iter().filter(|x| !x.1.is_spam).count(), 1); + assert_eq!(samples.iter().filter(|x| x.1.is_spam).count(), 1); + + // Train the classifier via JMAP + for (ids, is_spam) in [(&spam_ids, true), (&ham_ids, false)] { + for (idx, id) in ids.iter().skip(1).enumerate() { + // Set keywords and mailboxes + let mut request = client.build(); + let req = request.set_email().update(id); + if idx < 5 || !is_spam { + // Update via keywords + let keyword = if is_spam { + Keyword::Junk + } else { + Keyword::NotJunk + } + .to_string(); + req.keywords([&keyword]); + } else { + // Update via mailbox + let mailbox_id = if is_spam { JUNK_ID } else { INBOX_ID }; + req.mailbox_ids([&Id::from(mailbox_id).to_string()]); + } + + request.send_set_email().await.unwrap().updated(id).unwrap(); + } + } + let samples = account.spam_training_samples().await; + assert_eq!(samples.iter().filter(|x| !x.1.is_spam).count(), 10); + assert_eq!(samples.iter().filter(|x| x.1.is_spam).count(), 10); + + // Make sure the email details are available in the sample + assert_eq!(samples[0].1.subject, "save up to = on life insurance"); + assert_eq!(samples[0].1.from, "spammy@mcspamface.net"); + + // Reclassifying an email should not add a new sample + let mut request = client.build(); + request + .set_email() + .update(&ham_ids[0]) + .keywords([Keyword::Junk.to_string()]); + request + .send_set_email() + .await + .unwrap() + .updated(&ham_ids[0]) + .unwrap(); + + admin + .registry_create_object(Task::SpamFilterMaintenance(TaskSpamFilterMaintenance { + maintenance_type: TaskSpamFilterMaintenanceType::Train, + status: TaskStatus::now(), + })) + .await; + test.wait_for_tasks().await; + + let samples = account.spam_training_samples().await; + assert_eq!(samples.len(), 20); + assert_eq!(samples.iter().filter(|x| !x.1.is_spam).count(), 9); + assert_eq!(samples.iter().filter(|x| x.1.is_spam).count(), 11); + let hold_for = test + .server + .core + .spam + .classifier + .as_ref() + .unwrap() + .hold_samples_for; + assert!( + hold_for > 2 * 86400, + "hold for {} should be greater than 2 days", + hold_for + ); + let hold_until = now() + hold_for; + let hold_range = (hold_until - 86400)..=hold_until; + + assert!(samples.iter().all(|(_, s)| { + s.blob_id.class.account_id() == account_id + && !s.delete_after_use + && hold_range.contains(&(s.expires_at.timestamp() as u64)) + })); + + // Purging blobs should not remove training samples + admin + .registry_create_object(Task::StoreMaintenance(TaskStoreMaintenance { + maintenance_type: TaskStoreMaintenanceType::PurgeBlob, + shard_index: None, + status: TaskStatus::now(), + })) + .await; + test.wait_for_tasks().await; + let samples = account.spam_training_samples().await; + assert_eq!(samples.iter().filter(|x| !x.1.is_spam).count(), 9); + assert_eq!(samples.iter().filter(|x| x.1.is_spam).count(), 11); + assert_eq!(samples.len(), 20); + + // Adding a training sample without permissions should fail + assert_eq!( + account + .registry_create_many( + ObjectType::SpamTrainingSample, + [json!({ + Property::BlobId: samples[0].1.blob_id.clone(), + })], + ) + .await + .method_response() + .text_field("type"), + "forbidden" + ); + + // Update permissions and try again + admin + .registry_update_object( + ObjectType::Account, + account.id(), + json!({ + Property::Permissions: Permissions::Merge(PermissionsList { + disabled_permissions: Map::default(), + enabled_permissions: Map::new(vec![Permission::SysSpamTrainingSampleCreate]), + }) + }), + ) + .await; + let sample_id = account + .registry_create_many( + ObjectType::SpamTrainingSample, + [json!({ + Property::BlobId: samples[0].1.blob_id.clone(), + Property::IsSpam: true, + })], + ) + .await + .created_id(0); + let sample = account.registry_get::(sample_id).await; + assert_eq!(sample.subject, "save up to = on life insurance"); + assert_eq!(sample.from, "spammy@mcspamface.net"); + let samples = account.spam_training_samples().await; + assert_eq!(samples.iter().filter(|x| !x.1.is_spam).count(), 9); + assert_eq!(samples.iter().filter(|x| x.1.is_spam).count(), 12); + assert_eq!(samples.len(), 21); + + // Delete account + test.destroy_all_mailboxes(&account).await; + account + .registry_destroy_all(ObjectType::SpamTrainingSample) + .await; + test.assert_is_empty().await; + + admin.destroy_account(account).await; + admin.destroy_account(other_account).await; + test.wait_for_tasks().await; +} + +pub const SPAM: [&str; 10] = [ + concat!( + "From: spammy@mcspamface.net\r\n", + "Subject: save up to = on life insurance\r\n\r\n wh", + "y spend more than you have to life quote savings e", + "nsuring your family s financial security is very i", + "mportant life quote savings makes buying life insu", + "rance simple and affordable we provide free access", + " to the very best companies and the lowest rates l", + "ife quote savings is fast easy and saves you money", + " let us help you get started with the best values ", + "in the country on new coverage you can save hundre", + "ds or even thousands of dollars by requesting a fr", + "ee quote from lifequote savings our service will t", + "ake you less than = minutes to complete shop ", + "and compare save up to = on all types of life", + " insurance hyperlink click here for your free quot", + "e protecting your family is the best investment yo", + "u ll ever make if you are in receipt of this email", + " in error and or wish to be removed from our list ", + "hyperlink please click here and type remove if you", + " reside in any state which prohibits e mail solici", + "tations for insurance please disregard this email\r\n", + " \r\n" + ), + concat!( + "Subject: a powerhouse gifting program\r\n\r\nyou don t ", + "want to miss get in with the founders the major pl", + "ayers are on this one for once be where the player", + "s are this is your private invitation experts are ", + "calling this the fastest way to huge cash flow eve", + "r conceived leverage = = into = NUM", + "BER over and over again the question here is you e", + "ither want to be wealthy or you don t which one ar", + "e you i am tossing you a financial lifeline and fo", + "r your sake i hope you grab onto it and hold on ti", + "ght for the ride of your life testimonials hear wh", + "at average people are doing their first few days w", + "e ve received = = in = day and we a", + "re doing that over and over again q s in al i m a ", + "single mother in fl and i ve received = NUMBE", + "R in the last = days d s in fl i was not sure", + " about this when i sent off my = = pledg", + "e but i got back = = the very next day l", + " l in ky i didn t have the money so i found myself", + " a partner to work this with we have received NUMB", + "ER = over the last = days i think i made", + " the right decision don t you k c in fl i pick up ", + "= = my first day and i they gave me free", + " leads and all the training you can too j w in ca ", + "announcing we will close your sales for you and he", + "lp you get a fax blast immediately upon your entry", + " you make the money free leads training don t wait", + " call now fax back to = = = = ", + "or call = = = = name__________", + "________________________phone_____________________", + "______________________ fax________________________", + "_____________email________________________________", + "____________ best time to call____________________", + "_____time zone____________________________________", + "____ this message is sent in compliance of the new", + " e mail bill per section = paragraph a =", + " c of s = further transmissions by the sender", + " of this email may be stopped at no cost to you by", + " sending a reply to this email address with the wo", + "rd remove in the subject line errors omissions and", + " exceptions excluded this is not spam i have compi", + "led this list from our replicate database relative", + " to seattle marketing group the gigt or turbo team", + " for the sole purpose of these communications your", + " continued inclusion is only by your gracious perm", + "ission if you wish to not receive this mail from m", + "e please send an email to tesrewinter with rem", + "ove in the subject and you will be deleted immedia", + "tely\r\n\r\n" + ), + concat!( + "Subject: help wanted \r\n\r\nwe are a = year old f", + "ortune = company that is growing at a tremend", + "ous rate we are looking for individuals who want t", + "o work from home this is an opportunity to make an", + " excellent income no experience is required we wil", + "l train you so if you are looking to be employed f", + "rom home with a career that has vast opportunities", + " then go we are looking for energetic and self", + " motivated people if that is you than click on the", + " link and fill out the form and one of our employe", + "ment specialist will contact you to be removed fro", + "m our link simple go to \r\n\r\n" + ), + concat!( + "Subject: tired of the bull out there\r\n\r\n want to st", + "op losing money want a real money maker receive NU", + "MBER = = = today experts are callin", + "g this the fastest way to huge cash flow ever conc", + "eived a powerhouse gifting program you don t want ", + "to miss we work as a team this is your private inv", + "itation get in with the founders this is where the", + " big boys play the major players are on this one f", + "or once be where the players are this is a system ", + "that will drive = = s to your doorstep i", + "n a short period of time leverage = = in", + "to = = over and over again the question ", + "here is you either want to be wealthy or you don t", + " which one are you i am tossing you a financial li", + "feline and for your sake i hope you grab onto it a", + "nd hold on tight for the ride of your life testimo", + "nials hear what average people are doing their fir", + "st few days we ve received = = in =", + " day and we are doing that over and over again q s", + " in al i m a single mother in fl and i ve received", + " = = in the last = days d s in fl i", + " was not sure about this when i sent off my =", + " = pledge but i got back = = the ve", + "ry next day l l in ky i didn t have the money so i", + " found myself a partner to work this with we have ", + "received = = over the last = days i", + " think i made the right decision don t you k c in ", + "fl i pick up = = my first day and i they", + " gave me free leads and all the training you can t", + "oo j w in ca this will be the most important call ", + "you make this year free leads training announcing ", + "we will close your sales for you and help you get ", + "a fax blast immediately upon your entry you make t", + "he money free leads training don t wait call now N", + "UMBER = = = print and fax to =", + " = = = or send an email requesting ", + "more information to successleads please includ", + "e your name and telephone number receive = NU", + "MBER free leads just for responding a = NUMBE", + "R value name___________________________________ ph", + "one___________________________________ fax________", + "_____________________________ email_______________", + "____________________ this message is sent in compl", + "iance of the new e mail bill per section = pa", + "ragraph a = c of s = further transmissio", + "ns by the sender of this email may be stopped at n", + "o cost to you by sending a reply to this email add", + "ress with the word remove in the subject line erro", + "rs omissions and exceptions excluded this is not s", + "pam i have compiled this list from our replicate d", + "atabase relative to seattle marketing group the gi", + "gt or turbo team for the sole purpose of these com", + "munications your continued inclusion is only by yo", + "ur gracious permission if you wish to not receive ", + "this mail from me please send an email to tesrewin", + "ter with remove in the subject and you will be", + " deleted immediately\r\n\r\n" + ), + concat!( + "Subject: cellular phone accessories \r\n\r\n all at bel", + "ow wholesale prices http = = = NUMB", + "ER = sites merchant sales hands free ear buds", + " = = phone holsters = = booste", + "r antennas only = = phone cases = N", + "UMBER car chargers = = face plates as lo", + "w as = = lithium ion batteries as low as", + " = = http = = = = NU", + "MBER sites merchant sales click below for accessor", + "ies on all nokia motorola lg nextel samsung qualco", + "mm ericsson audiovox phones at below wholesale pri", + "ces http = = = = = sites ", + "merchant sales if you need assistance please call ", + "us = = = to be removed from future ", + "mailings please send your remove request to remove", + " me now = thank you and have a super day\r\n", + " \r\n" + ), + concat!( + "Subject: conferencing made easy\r\n\r\n only = cen", + "ts per minute including long distance no setup fee", + "s no contracts or monthly fees call anytime from a", + "nywhere to anywhere connects up to = particip", + "ants simplicity in set up and administration opera", + "tor help available = = the highest quali", + "ty service for the lowest rate in the industry fil", + "l out the form below to find out how you can lower", + " your phone bill every month required input field ", + "name web address company name state business phone", + " home phone email address type of business to be r", + "emoved from our distribution lists please hyperlin", + "k click here\r\n\r\n" + ), + concat!( + "Subject: dear friend\r\n\r\n i am mrs sese seko widow o", + "f late president mobutu sese seko of zaire now kno", + "wn as democratic republic of congo drc i am moved ", + "to write you this letter this was in confidence co", + "nsidering my presentcircumstance and situation i e", + "scaped along with my husband and two of our sons g", + "eorge kongolo and basher out of democratic republi", + "c of congo drc to abidjan cote d ivoire where my f", + "amily and i settled while we later moved to settle", + "d in morroco where my husband later died of cancer", + " disease however due to this situation we decided ", + "to changed most of my husband s billions of dollar", + "s deposited in swiss bank and other countries into", + " other forms of money coded for safe purpose becau", + "se the new head of state of dr mr laurent kabila h", + "as made arrangement with the swiss government and ", + "other european countries to freeze all my late hus", + "band s treasures deposited in some european countr", + "ies hence my children and i decided laying low in ", + "africa to study the situation till when things get", + "s better like now that president kabila is dead an", + "d the son taking over joseph kabila one of my late", + " husband s chateaux in southern france was confisc", + "ated by the french government and as such i had to", + " change my identity so that my investment will not", + " be traced and confiscated i have deposited the su", + "m eighteen million united state dollars us = ", + "= = = with a security company for s", + "afekeeping the funds are security coded to prevent", + " them from knowing the content what i want you to ", + "do is to indicate your interest that you will assi", + "st us by receiving the money on our behalf acknowl", + "edge this message so that i can introduce you to m", + "y son kongolo who has the out modalities for the c", + "laim of the said funds i want you to assist in inv", + "esting this money but i will not want my identity ", + "revealed i will also want to buy properties and st", + "ock in multi national companies and to engage in o", + "ther safe and non speculative investments may i at", + " this point emphasise the high level of confidenti", + "ality which this business demands and hope you wil", + "l not betray the trust and confidence which i repo", + "se in you in conclusion if you want to assist us m", + "y son shall put you in the picture of the business", + " tell you where the funds are currently being main", + "tained and also discuss other modalities including", + " remunerationfor your services for this reason kin", + "dly furnish us your contact information that is yo", + "ur personal telephone and fax number for confident", + "ial regards mrs m sese seko\r\n\r\n" + ), + concat!( + "Subject: lowest rates available for term life insu", + "rance\r\n\r\n take a moment and fill out our online for", + "m to see the low rate you qualify for save up to N", + "UMBER from regular rates smokers accepted repr", + "esenting quality nationwide carriers act now to ea", + "sily remove your address from the list go to p", + "lease allow = = hours for removal\r\n\r\n" + ), + concat!( + "Subject: central bank of nigeria foreign remittanc", + "e \r\n\r\n dept tinubu square lagos nigeria email smith", + "_j =th of august = attn president ce", + "o strictly private business proposal i am mr johns", + "on s abu the bills and exchange director at the fo", + "reignremittance department of the central bank of ", + "nigeria i am writingyou this letter to ask for you", + "r support and cooperation to carrying thisbusiness", + " opportunity in my department we discovered abando", + "ned the sumof us = = = = thirt", + "y seven million four hundred thousand unitedstates", + " dollars in an account that belong to one of our f", + "oreign customers an american late engr john creek ", + "junior an oil merchant with the federal government", + " of nigeria who died along with his entire family ", + "of a wifeand two children in kenya airbus a= ", + "= flight kq= in november= since we ", + "heard of his death we have been expecting his next", + " of kin tocome over and put claims for his money a", + "s the heir because we cannotrelease the fund from ", + "his account unless someone applies for claims asth", + "e next of kin to the deceased as indicated in our ", + "banking guidelines unfortunately neither their fam", + "ily member nor distant relative hasappeared to cla", + "im the said fund upon this discovery i and other o", + "fficialsin my department have agreed to make busin", + "ess with you release the totalamount into your acc", + "ount as the heir of the fund since no one came for", + "it or discovered either maintained account with ou", + "r bank other wisethe fund will be returned to the ", + "bank treasury as unclaimed fund we have agreed tha", + "t our ratio of sharing will be as stated thus NUMB", + "ER for you as foreign partner and = for us th", + "e officials in my department upon the successful c", + "ompletion of this transfer my colleague and i will", + "come to your country and mind our share it is from", + " our = we intendto import computer accessorie", + "s into my country as way of recycling thefund to c", + "ommence this transaction we require you to immedia", + "tely indicateyour interest by calling me or sendin", + "g me a fax immediately on the abovetelefax and enc", + "lose your private contact telephone fax full namea", + "nd address and your designated banking co ordinate", + "s to enable us fileletter of claim to the appropri", + "ate department for necessary approvalsbefore the t", + "ransfer can be made note also this transaction mus", + "t be kept strictly confidential becauseof its natu", + "re nb please remember to give me your phone and fa", + "x no mr johnson smith abu irish linux users group ", + "ilug for un subscription information list ", + "maintainer listmaster \r\n\r\n" + ), + concat!( + "Subject: dear stuart\r\n\r\n are you tired of searching", + " for love in all the wrong places find love now at", + " browse through thousands of personals in ", + "your area join for free search e mail chat use", + " to meet cool guys and hot girls go = on ", + "= or use our private chat rooms click on the ", + "link to get started find love now you have rec", + "eived this email because you have registerd with e", + "mailrewardz or subscribed through one of our marke", + "ting partners if you have received this message in", + " error or wish to stop receiving these great offer", + "s please click the remove link above to unsubscrib", + "e from these mailings please click here \r\n\r\n" + ), +]; + +pub const HAM: [&str; 10] = [ + concat!( + "Message-ID: \r\nSubject: i have been", + " trying to research via sa mirrors and search engi", + "nes\r\n\r\nif a canned script exists giving clients acce", + "ss to their user_prefs options via a web based cgi", + " interface numerous isps provide this feature to c", + "lients but so far i can find nothing our configura", + "tion uses amavis postfix and clamav for virus filt", + "ering and procmail with spamassassin for spam filt", + "ering i would prefer not to have to write a script", + " myself but will appreciate any suggestions this U", + "RL email is sponsored by osdn tired of that same o", + "ld cell phone get a new here for free ________", + "_______________________________________ spamassass", + "in talk mailing list spamassassin talk \r\n\r\n" + ), + concat!( + "Message-ID: mid2@foobar.org\r\nSubject: hello\r\n\r\nhave y", + "ou seen and discussed this article and his approac", + "h thank you hell there are no rules here we re", + " trying to accomplish something thomas alva edison", + " this email is sponsored by osdn tired of that", + " same old cell phone get a new here for free _", + "______________________________________________ spa", + "massassin devel mailing list spamassassin devel UR", + "L \r\n\r\n" + ), + concat!( + "Message-ID: \r\nSubject: hi all apol", + "ogies for the possible silly question\r\n\r\ni don t thi", + "nk it is but but is eircom s adsl service nat ed a", + "nd what implications would that have for voip i kn", + "ow there are difficulties with voip or connecting ", + "to clients connected to a nat ed network from the ", + "internet wild i e machines with static real ips an", + "y help pointers would be helpful cheers rgrds bern", + "ard bernard tyers national centre for sensor resea", + "rch p = = = = e bernard tyers ", + " w l n= ______________________________", + "_________________ iiu mailing list iiu \r\n\r\n" + ), + concat!( + "Message-ID: \r\nSubject: can someone", + " explain\r\n\r\nwhat type of operating system solaris is", + " as ive never seen or used it i dont know wheather", + " to get a server from sun or from dell i would pre", + "fer a linux based server and sun seems to be the o", + "ne for that but im not sure if solaris is a distro", + " of linux or a completely different operating syst", + "em can someone explain kiall mac innes irish linux", + " users group ilug for un subscription info", + "rmation list maintainer listmaster \r\n\r\n" + ), + concat!( + "Message-ID: \r\nSubject: folks my fi", + "rst time posting\r\n\r\nhave a bit of unix experience bu", + "t am new to linux just got a new pc at home dell b", + "ox with windows xp added a second hard disk for li", + "nux partitioned the disk and have installed suse N", + "UMBER = from cd which went fine except it did", + "n t pick up my monitor i have a dell branded eNUMB", + "ERfpp = lcd flat panel monitor and a nvidia g", + "eforce= ti= video card both of which are", + " probably too new to feature in suse s default set", + " i downloaded a driver from the nvidia website and", + " installed it using rpm then i ran sax= as wa", + "s recommended in some postings i found on the net ", + "but it still doesn t feature my video card in the ", + "available list what next another problem i have a ", + "dell branded keyboard and if i hit caps lock twice", + " the whole machine crashes in linux not windows ev", + "en the on off switch is inactive leaving me to rea", + "ch for the power cable instead if anyone can help ", + "me in any way with these probs i d be really grate", + "ful i ve searched the net but have run out of idea", + "s or should i be going for a different version of ", + "linux such as redhat opinions welcome thanks a lot", + " peter irish linux users group ilug for un", + " subscription information list maintainer listmast", + "er \r\n\r\n" + ), + concat!( + "Message-ID: \r\nSubject: has anyone\r\n", + "\r\nseen heard of used some package that would let a ", + "random person go to a webpage create a mailing lis", + "t then administer that list also of course let ppl", + " sign up for the lists and manage their subscripti", + "ons similar to the old but i d like to have it", + " running on my server not someone elses chris ", + "\r\n\r\n" + ), + concat!( + "Message-ID: \r\nSubject: hi thank yo", + "u for the useful replies\r\n\r\ni have found some intere", + "sting tutorials in the ibm developer connection UR", + "L and registration is needed i will post the s", + "ame message on the web application security list a", + "s suggested by someone for now i thing i will use ", + "md= for password checking i will use the appr", + "oach described in secure programmin fo linux and u", + "nix how to i will separate the authentication modu", + "le so i can change its implementation at anytime t", + "hank you again mario torre please avoid sending me", + " word or powerpoint attachments see \r\n\r\n" + ), + concat!( + "Message-ID: \r\nSubject: hehe sorry\r\n", + "\r\nbut if you hit caps lock twice the computer crash", + "es theres one ive never heard before have you trye", + "d dell support yet i think dell computers prefer r", + "edhat dell provide some computers pre loaded with ", + "red hat i dont know for sure tho so get someone el", + "ses opnion as well as mine original message from i", + "lug admin mailto ilug admin on behalf of p", + "eter staunton sent = august = = NUM", + "BER to ilug subject ilug newbie seeks advice s", + "use = = folks my first time posting have", + " a bit of unix experience but am new to linux just", + " got a new pc at home dell box with windows xp add", + "ed a second hard disk for linux partitioned the di", + "sk and have installed suse = = from cd w", + "hich went fine except it didn t pick up my monitor", + " i have a dell branded e=fpp = lcd flat ", + "panel monitor and a nvidia geforce= ti= ", + "video card both of which are probably too new to f", + "eature in suse s default set i downloaded a driver", + " from the nvidia website and installed it using rp", + "m then i ran sax= as was recommended in some ", + "postings i found on the net but it still doesn t f", + "eature my video card in the available list what ne", + "xt another problem i have a dell branded keyboard ", + "and if i hit caps lock twice the whole machine cra", + "shes in linux not windows even the on off switch i", + "s inactive leaving me to reach for the power cable", + " instead if anyone can help me in any way with the", + "se probs i d be really grateful i ve searched the ", + "net but have run out of ideas or should i be going", + " for a different version of linux such as redhat o", + "pinions welcome thanks a lot peter irish linux use", + "rs group ilug for un subscription informat", + "ion list maintainer listmaster irish linux use", + "rs group ilug for un subscription informat", + "ion list maintainer listmaster \r\n\r\n" + ), + concat!( + "Message-ID: \r\nSubject: it will fun", + "ction as a router\r\n\r\nif that is what you wish it eve", + "n looks like the modem s embedded os is some kind ", + "of linux being that it has interesting interfaces ", + "like eth= i don t use it as a router though i", + " just have it do the absolute minimum dsl stuff an", + "d do all the really fun stuff like pppoe on my lin", + "ux box also the manual tells you what the default ", + "password is don t forget to run pppoe over the alc", + "atel speedtouch =i as in my case you have to ", + "have a bridge configured in the router modem s sof", + "tware this lists your vci values etc also does any", + "one know if the high end speedtouch with = et", + "hernet ports can act as a full router or do i stil", + "l need to run a pppoe stack on the linux box regar", + "ds vin irish linux users group ilug for un", + " subscription information list maintainer listmast", + "er irish linux users group ilug for un", + " subscription information list maintainer listmast", + "er \r\n\r\n" + ), + concat!( + "Message-ID: \r\nSubject: all is it ", + "just me\r\n\r\nor has there been a massive increase in t", + "he amount of email being falsely bounced around th", + "e place i ve already received email from a number ", + "of people i don t know asking why i am sending the", + "m email these can be explained by servers from rus", + "sia and elsewhere coupled with the false emails i ", + "received myself it s really starting to annoy me a", + "m i the only one seeing an increase in recent week", + "s martin martin whelan déise design tel NUMBE", + "R = our core product déiseditor allows organ", + "isations to publish information to their web site ", + "in a fast and cost effective manner there is no ne", + "ed for a full time web developer as the site can b", + "e easily updated by the organisations own staff in", + "stant updates to keep site information fresh sites", + " which are updated regularly bring users back visi", + "t for a demonstration déiseditor managing you", + "r information ____________________________________", + "___________ iiu mailing list iiu ,0\r\n" + ), +]; + +pub const TEST: [&str; 3] = [ + concat!( + "From: spammy@mcspamface.net\r\n", + "Subject: save up to = on life insurance\r\n\r\nwhy ", + "spend more than you have to life quote savings ens", + "uring your family s financial security is very imp", + "ortant life quote savings makes buying life insura", + "nce simple and affordable we provide free access t", + "o the very best companies and the lowest rates lif", + "e quote savings is fast easy and saves you money l", + "et us help you get started with the best values in", + " the country on new coverage you can save hundreds", + " or even thousands of dollars by requesting a free", + " quote from lifequote savings our service will tak", + "e you less than = minutes to complete shop an", + "d compare save up to = on all types of life i", + "nsurance hyperlink click here for your free quote ", + "protecting your family is the best investment you ", + "ll ever make if you are in receipt of this email i", + "n error and or wish to be removed from our list hy", + "perlink please click here and type remove if you r", + "eside in any state which prohibits e mail solicita", + "tions for insurance please disregard this email\r\n" + ), + concat!( + "Subject: can someone explain\r\n\r\nwhat type of operati", + "ng system solaris is as ive never seen or used it ", + "i dont know wheather to get a server from sun or f", + "rom dell i would prefer a linux based server and s", + "un seems to be the one for that but im not sure if", + " solaris is a distro of linux or a completely diff", + "erent operating system can someone explain kiall m", + "ac innes irish linux users group ilug for ", + "un subscription information list maintainer listma", + "ster \r\n" + ), + concat!( + "Subject: classifier test\r\n\r\nthis is a novel text tha", + "t the sgd classifier has never seen before, it s", + "hould be classified as ham or non-ham\r\n" + ), +]; diff --git a/tests/src/system/archiving.rs b/tests/src/system/archiving.rs new file mode 100644 index 00000000..7b960c34 --- /dev/null +++ b/tests/src/system/archiving.rs @@ -0,0 +1,365 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: LicenseRef-SEL + * + * This file is subject to the Stalwart Enterprise License Agreement (SEL) and + * is NOT open source software. + * + */ + +use crate::utils::{ + imap::{AssertResult, ImapConnection, Type}, + jmap::JmapUtils, + server::TestServer, +}; +use imap_proto::ResponseType; +use jmap_proto::error::set::SetErrorType; +use registry::{ + schema::{ + enums::{AccountType, ArchivedItemStatus, TaskStoreMaintenanceType}, + prelude::{ObjectType, Property}, + structs::{ + Account, ArchivedItem, Credential, DataRetention, PasswordCredential, Task, TaskStatus, + TaskStoreMaintenance, + }, + }, + types::list::List, +}; +use serde_json::json; +use types::id::Id; + +pub async fn test(test: &mut TestServer) { + // Add test settings + let admin = test.account("admin@example.org"); + admin + .registry_update_setting( + DataRetention { + archive_deleted_accounts_for: Some(3600u64.into()), + archive_deleted_items_for: Some(1u64.into()), + ..Default::default() + }, + &[ + Property::ArchiveDeletedAccountsFor, + Property::ArchiveDeletedItemsFor, + ], + ) + .await; + admin.reload_settings().await; + + // Create test account + let john = test + .create_user_account( + "admin@example.org", + "jdoe@example.org", + "this is a very strong password", + &[], + ) + .await; + let jane = test + .create_user_account( + "admin@example.org", + "jane@example.org", + "this is a very strong password", + &[], + ) + .await; + let mut john_imap = john.imap_client().await; + let mut jane_imap = jane.imap_client().await; + + for (account, imap) in [(&john, &mut john_imap), (&jane, &mut jane_imap)] { + let message = RAW_MESSAGE.replace("NAME", account.name()); + + // Insert test message + imap.send("STATUS INBOX (MESSAGES)").await; + imap.assert_read(Type::Tagged, ResponseType::Ok) + .await + .assert_contains("MESSAGES 0"); + imap.send(&format!("APPEND INBOX {{{}}}", message.len())) + .await; + imap.assert_read(Type::Continuation, ResponseType::Ok).await; + imap.send_untagged(&message).await; + imap.assert_read(Type::Tagged, ResponseType::Ok).await; + + // Make sure the message is there + imap.send("STATUS INBOX (MESSAGES)").await; + imap.assert_read(Type::Tagged, ResponseType::Ok) + .await + .assert_contains("MESSAGES 1"); + imap.send("SELECT INBOX").await; + imap.assert_read(Type::Tagged, ResponseType::Ok).await; + + // Fetch message body + imap.send("FETCH 1 BODY[]").await; + imap.assert_read(Type::Tagged, ResponseType::Ok) + .await + .assert_contains(&format!("Subject: undelete test for {}", account.name())); + + // Delete and expunge message + imap.send("STORE 1 +FLAGS (\\Deleted)").await; + imap.assert_read(Type::Tagged, ResponseType::Ok).await; + imap.send("EXPUNGE").await; + imap.assert_read(Type::Tagged, ResponseType::Ok).await; + + // Logout and reconnect + imap.send("LOGOUT").await; + imap.assert_read(Type::Tagged, ResponseType::Ok).await; + *imap = ImapConnection::connect(b"_x ").await; + imap.authenticate(account.name(), account.secret()).await; + + // Make sure the message is gone + imap.send("STATUS INBOX (MESSAGES)").await; + imap.assert_read(Type::Tagged, ResponseType::Ok) + .await + .assert_contains("MESSAGES 0"); + } + + // Expunge messages + admin + .registry_create_object(Task::StoreMaintenance(TaskStoreMaintenance { + maintenance_type: TaskStoreMaintenanceType::PurgeAccounts, + shard_index: None, + status: TaskStatus::now(), + })) + .await; + test.wait_for_tasks().await; + + // Fetch archived items + let mut john_archive_id = Id::singleton(); + let mut jane_archive_id = Id::singleton(); + for (account, archive_id) in [(&john, &mut john_archive_id), (&jane, &mut jane_archive_id)] { + let ids = account + .registry_query( + ObjectType::ArchivedItem, + Vec::<(&str, &str)>::new(), + Vec::<&str>::new(), + ) + .await + .object_ids() + .collect::>(); + assert_eq!(ids.len(), 1); + *archive_id = ids[0]; + + let response = account + .registry_get_many(ObjectType::ArchivedItem, Vec::<&str>::new()) + .await; + let archives = response.list(); + assert_eq!(archives.len(), 1); + assert_eq!(archives[0].object_id(), *archive_id); + + let archive = account.registry_get::(*archive_id).await; + if let ArchivedItem::Email(archive) = archive { + assert_eq!( + archive.subject, + format!("undelete test for {}", account.name()) + ); + assert_eq!(archive.from, format!("{}@example.org", account.name())); + assert_eq!( + archive.blob_id.class.account_id(), + account.id().document_id() + ); + assert!(archive.size > 0); + } else { + panic!("Unexpected archived item type: {:?}", archive); + } + } + + // John should not be able to get, update or destroy Jane's archived item and vice versa + assert_eq!( + john.registry_get_many(ObjectType::ArchivedItem, [jane_archive_id]) + .await + .not_found() + .count(), + 1 + ); + john.registry_update_object_expect_err( + ObjectType::ArchivedItem, + jane_archive_id, + json!({ + Property::Status: ArchivedItemStatus::RequestRestore, + }), + ) + .await + .assert_type(SetErrorType::NotFound); + john.registry_destroy_object_expect_err(ObjectType::ArchivedItem, jane_archive_id) + .await + .assert_type(SetErrorType::NotFound); + + // Admin should be able to see both archived items + assert_eq!( + admin + .registry_query( + ObjectType::ArchivedItem, + Vec::<(&str, &str)>::new(), + Vec::<&str>::new() + ) + .await + .object_ids() + .collect::>(), + vec![john_archive_id, jane_archive_id] + ); + assert_eq!( + admin + .registry_query( + ObjectType::ArchivedItem, + [(Property::AccountId, jane.id().to_string())], + Vec::<&str>::new() + ) + .await + .object_ids() + .collect::>(), + vec![jane_archive_id] + ); + + // Request restore for John's archived item + john.registry_update_object( + ObjectType::ArchivedItem, + john_archive_id, + json!({ + Property::Status: ArchivedItemStatus::RequestRestore, + }), + ) + .await; + test.wait_for_tasks().await; + + // Make sure the message is back + john_imap.send("STATUS INBOX (MESSAGES)").await; + john_imap + .assert_read(Type::Tagged, ResponseType::Ok) + .await + .assert_contains("MESSAGES 1"); + + john_imap.send("SELECT INBOX").await; + john_imap.assert_read(Type::Tagged, ResponseType::Ok).await; + + // Fetch message body + john_imap.send("FETCH 1 BODY[]").await; + john_imap + .assert_read(Type::Tagged, ResponseType::Ok) + .await + .assert_contains(&format!("Subject: undelete test for {}", john.name())); + + // Jane's archived item should be deleted on the next purge + admin + .registry_create_object(Task::StoreMaintenance(TaskStoreMaintenance { + maintenance_type: TaskStoreMaintenanceType::PurgeBlob, + shard_index: None, + status: TaskStatus::now(), + })) + .await; + test.wait_for_tasks().await; + assert_eq!( + jane.registry_get_many(ObjectType::ArchivedItem, [jane_archive_id]) + .await + .not_found() + .count(), + 1 + ); + + // Delete John's account + john_imap.send("LOGOUT").await; + john_imap.assert_read(Type::Tagged, ResponseType::Ok).await; + let domain_id = admin.find_or_create_domain("example.org").await; + admin + .registry_destroy(ObjectType::Account, [john.id()]) + .await + .assert_destroyed(&[john.id()]); + assert!( + test.server + .rcpt_id_from_email("jdoe@example.org") + .await + .unwrap() + .is_none() + ); + assert!( + test.server + .try_account(john.id().document_id()) + .await + .unwrap() + .is_none() + ); + + // Make sure the deletion task is created + let task_ids = admin + .registry_query( + ObjectType::Task, + Vec::<(&str, &str)>::new(), + Vec::<&str>::new(), + ) + .await + .object_ids() + .collect::>(); + assert_eq!( + task_ids.len(), + 1, + "Expected exactly one task, found {:?}", + task_ids + ); + let task = admin.registry_get::(task_ids[0]).await; + if let Task::DestroyAccount(task) = task { + assert_eq!(task.account_id, john.id()); + assert_eq!(task.account_domain_id, domain_id); + assert_eq!(task.account_name, "jdoe"); + assert_eq!(task.account_type, AccountType::User) + } else { + panic!("Unexpected task type: {:?}", task); + } + + // Delete task to trigger restore + admin + .registry_destroy(ObjectType::Task, [task_ids[0]]) + .await + .assert_destroyed(&[task_ids[0]]); + test.wait_for_tasks().await; + + // Make sure the account is back and set a new password + let _john_account = admin.registry_get::(john.id()).await; + admin + .registry_update_object( + ObjectType::Account, + john.id(), + json!({ + Property::Credentials: List::from_iter([Credential::Password(PasswordCredential { + secret: "brand new secret".to_string(), + ..Default::default() + })]), + }), + ) + .await; + + // Authenticate with the new password and fetch the message again + let mut john_imap = ImapConnection::connect(b"_x ").await; + john_imap + .authenticate("jdoe@example.org", "brand new secret") + .await; + john_imap.send("FETCH 1 BODY[]").await; + john_imap + .assert_read(Type::Tagged, ResponseType::Ok) + .await + .assert_contains(&format!("Subject: undelete test for {}", john.name())); + + // Restore settings + admin + .registry_update_setting( + DataRetention::default(), + &[ + Property::ArchiveDeletedAccountsFor, + Property::ArchiveDeletedItemsFor, + ], + ) + .await; + admin.reload_settings().await; + + // Delete accounts + admin.destroy_account(john).await; + admin.destroy_account(jane).await; + + test.assert_is_empty().await; +} + +const RAW_MESSAGE: &str = "From: NAME@example.org +To: NAME@example.org +Subject: undelete test for NAME + +test +"; diff --git a/tests/src/system/authorization.rs b/tests/src/system/authorization.rs index e0eb1da2..975724e3 100644 --- a/tests/src/system/authorization.rs +++ b/tests/src/system/authorization.rs @@ -4,6 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::utils::{jmap::JmapUtils, server::TestServer}; +use ahash::AHashMap; use common::auth::{BuildAccessToken, permissions::DefaultPermissions}; use jmap_proto::error::set::SetErrorType; use registry::{ @@ -18,10 +20,9 @@ use registry::{ types::{EnumImpl, list::List, map::Map}, }; use serde_json::json; +use std::str::FromStr; use types::id::Id; -use crate::utils::{jmap::JmapUtils, server::TestServer}; - pub async fn test(test: &mut TestServer) { println!("Running authorization tests..."); @@ -225,5 +226,161 @@ pub async fn test(test: &mut TestServer) { .assert_destroyed(&[role_id]); } + // Create test data for John and Jane + let john = test + .create_user_account( + "admin@example.org", + "john@example.org", + "this is john's secret", + &[], + ) + .await; + let jane = test + .create_user_account( + "admin@example.org", + "jane@example.org", + "this is jane's secret", + &[], + ) + .await; + let mut john_ids = AHashMap::new(); + let mut jane_ids = AHashMap::new(); + for (account, ids) in [(&john, &mut john_ids), (&jane, &mut jane_ids)] { + let pk_id = account + .registry_create_many( + ObjectType::PublicKey, + [json!({ + Property::Description:"This is a public key", + Property::Key: SMIME_CERTIFICATE, + })], + ) + .await + .created(0) + .object_id(); + ids.insert(ObjectType::PublicKey, pk_id); + + let masked_id = account + .registry_create_many( + ObjectType::MaskedEmail, + [json!({ + Property::EmailDomain: "example.org", + })], + ) + .await + .created(0) + .object_id(); + ids.insert(ObjectType::MaskedEmail, masked_id); + } + + // John should not be able to see Jane's objects and vice versa + for (account, own_ids, other_ids) in + [(&john, &john_ids, &jane_ids), (&jane, &jane_ids, &john_ids)] + { + for (object_type, id) in own_ids { + assert_eq!( + account + .registry_query(*object_type, Vec::<(&str, &str)>::new(), Vec::<&str>::new()) + .await + .object_ids() + .collect::>(), + vec![*id] + ); + assert_eq!( + account + .registry_get_many(*object_type, Vec::<&str>::new()) + .await + .list() + .len(), + 1 + ); + } + + for (object_type, id) in other_ids { + assert_eq!( + account + .registry_get_many(*object_type, [*id]) + .await + .not_found() + .map(|id| Id::from_str(id).unwrap()) + .collect::>(), + vec![*id] + ); + + account + .registry_update_object_expect_err( + *object_type, + *id, + json!({ + Property::Description: "Hacked description" + }), + ) + .await + .assert_type(SetErrorType::NotFound); + + account + .registry_destroy_object_expect_err(*object_type, *id) + .await + .assert_type(SetErrorType::NotFound); + } + } + + // Admin should see all objects + for object_type in [ObjectType::PublicKey, ObjectType::MaskedEmail] { + let objects = admin + .registry_query(object_type, Vec::<(&str, &str)>::new(), Vec::<&str>::new()) + .await + .object_ids() + .collect::>(); + assert_eq!(objects.len(), 2); + assert!( + objects.contains(&john_ids[&object_type]) && objects.contains(&jane_ids[&object_type]), + ); + + // Filter by account id should work + let objects = admin + .registry_query( + object_type, + [(Property::AccountId, john.id().to_string())], + Vec::<&str>::new(), + ) + .await + .object_ids() + .collect::>(); + assert_eq!(objects, vec![john_ids[&object_type]]); + } + + // Destroy test data + for (account, ids) in [(&john, &john_ids), (&jane, &jane_ids)] { + for (object_type, id) in ids { + account + .registry_destroy(*object_type, [*id]) + .await + .assert_destroyed(&[*id]); + } + } + test.assert_is_empty().await; } + +const SMIME_CERTIFICATE: &str = "-----BEGIN CERTIFICATE----- +MIIDbjCCAlagAwIBAgIUZ4K0WXNSS8H0cUcZavD9EYqqTAswDQYJKoZIhvcNAQEN +BQAwLTErMCkGA1UEAxMiU2FtcGxlIExBTVBTIENlcnRpZmljYXRlIEF1dGhvcml0 +eTAgFw0xOTExMjAwNjU0MThaGA8yMDUyMDkyNzA2NTQxOFowGTEXMBUGA1UEAxMO +QWxpY2UgTG92ZWxhY2UwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDD +7q35ZdG2JAzzJGNZDZ9sV7AKh0hlRfoFjTZN5m4RegQAYSyag43ouWi1xRN0avf0 +UTYrwjK04qRdV7GzCACoEKq/xiNUOsjfJXzbCublN3fZMOXDshKKBqThlK75SjA9 +Czxg7ejGoiY/iidk0e91neK30SCCaBTJlfR2ZDrPk73IPMeksxoTatfF9hw9dDA+ +/Hi1yptN/aG0Q/s9icFrxr6y2zQXsjuQPmjMZgj10aD9cazWVgRYCgflhmA0V1uQ +l1wobYU8DAVxVn+GgabqyjGQMoythIK0Gn5+ofwxXXUM/zbU+g6+1ISdoXxRRFtq +2GzbIqkAHZZQm+BbnFrhAgMBAAGjgZcwgZQwDAYDVR0TAQH/BAIwADAeBgNVHREE +FzAVgRNhbGljZUBzbWltZS5leGFtcGxlMBMGA1UdJQQMMAoGCCsGAQUFBwMEMA8G +A1UdDwEB/wQFAwMHoAAwHQYDVR0OBBYEFKwuVFqk/VUYry7oZkQ40SXR1wB5MB8G +A1UdIwQYMBaAFLdSTXPAiD2yw3paDPOU9/eAonfbMA0GCSqGSIb3DQEBDQUAA4IB +AQB76o4Yz7yrVSFcpXqLrcGtdI4q93aKCXECCCzNQLp4yesh6brqaZHNJtwYcJ5T +qbUym9hJ70iJE4jGNN+yAZR1ltte0HFKYIBKM4EJumG++2hqbUaLz4tl06BHaQPC +v/9NiNY7q9R9c/B6s1YzHhwqkWht2a+AtgJ4BkpG+g+MmZMQV/Ao7RwLFKJ9OlMW +LBmEXFcpIJN0HpPasT0nEl/MmotSu+8RnClAi3yFfyTKb+8rD7VxuyXetqDZ6dU/ +9/iqD/SZS7OQIjywtd343mACz3B1RlFxMHSA6dQAf2btGumqR0KiAp3KkYRAePoa +JqYkB7Zad06ngFl0G0FHON+7 +-----END CERTIFICATE----- +"; diff --git a/tests/src/jmap/mail/crypto.rs b/tests/src/system/crypto.rs similarity index 51% rename from tests/src/jmap/mail/crypto.rs rename to tests/src/system/crypto.rs index cd896267..ca49b921 100644 --- a/tests/src/jmap/mail/crypto.rs +++ b/tests/src/system/crypto.rs @@ -4,13 +4,23 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::jmap::{JMAPTest, ManagementApi, mail::delivery::SmtpConnection}; -use mail_parser::{MessageParser, MimeHeaders}; -use std::path::PathBuf; -use store::{ - Deserialize, Serialize, - write::{Archive, Archiver}, +use crate::utils::{jmap::JmapUtils, server::TestServer, smtp::SmtpConnection}; +use common::{ + auth::{ + ACCOUNT_FLAG_ENCRYPT_ALGO_AES128, ACCOUNT_FLAG_ENCRYPT_ALGO_AES256, + ACCOUNT_FLAG_ENCRYPT_METHOD_PGP, ACCOUNT_FLAG_ENCRYPT_METHOD_SMIME, + }, + storage::encryption::{EncryptionMethod, parse_public_key}, }; +use email::message::crypto::EncryptMessage; +use mail_parser::{MessageParser, MimeHeaders}; +use registry::schema::{ + prelude::{ObjectType, Property}, + structs::{EncryptionAtRest, EncryptionSettings, PublicKey}, +}; +use serde_json::json; +use std::path::PathBuf; +use types::id::Id; pub async fn test(test: &mut TestServer) { println!("Running Encryption-at-rest tests..."); @@ -20,57 +30,85 @@ pub async fn test(test: &mut TestServer) { import_certs_and_encrypt().await; // Create test account - let account = test.account("jdoe@example.com"); + let account = test + .create_user_account( + "admin@example.org", + "jdoe@example.org", + "this is a very strong password", + &[], + ) + .await; let client = account.jmap_client().await; - // Build API - let api = ManagementApi::new(8899, "jdoe@example.com", "12345"); - - // Try importing using multiple methods and symmetric algos - for (file_name, method, num_certs) in [ - ("cert_smime.pem", EncryptionMethod::SMIME, 3), - ("cert_pgp.pem", EncryptionMethod::PGP, 1), - ] { + // Import all certs + let mut cert_ids = Vec::new(); + let mut certs_parsed = Vec::new(); + for cert_file in ["cert_smime.pem", "cert_pgp.pem"] { let certs = std::fs::read_to_string( PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("resources") .join("crypto") - .join(file_name), + .join(cert_file), ) .unwrap(); - for algo in [Algorithm::Aes128, Algorithm::Aes256] { - let request = match method { - EncryptionMethod::PGP => EncryptionType::PGP { - algo, - certs: certs.clone(), - allow_spam_training: true, - }, - EncryptionMethod::SMIME => EncryptionType::SMIME { - algo, - certs: certs.clone(), - allow_spam_training: true, - }, - }; + let params = parse_public_key(&PublicKey { + description: cert_file.to_string(), + key: certs.clone(), + ..Default::default() + }) + .unwrap() + .unwrap(); + certs_parsed.push(params.certs); - assert_eq!( - api.post::("/api/account/crypto", &request) - .await - .unwrap() - .unwrap_data(), - num_certs - ); - } + let cert_id = account + .registry_create_many( + ObjectType::PublicKey, + [json!({ + Property::Description: "This is a public key", + Property::Key: certs + })], + ) + .await + .created(0) + .object_id(); + + cert_ids.push(cert_id); } + // Update encryption at rest settings + account + .registry_update_object( + ObjectType::AccountSettings, + Id::singleton(), + json!({ + Property::EncryptionAtRest: EncryptionAtRest::Aes256(EncryptionSettings { + allow_spam_training: true, + encrypt_on_append: true, + public_key: cert_ids[1], + }) + }), + ) + .await; + assert_eq!( + test.server + .account(account.id().document_id()) + .await + .unwrap() + .encryption_key + .as_ref() + .unwrap(), + &certs_parsed[1] + ); + // Send a new message, which should be encrypted let mut lmtp = SmtpConnection::connect().await; lmtp.ingest( - "bill@example.com", - &["jdoe@example.com"], + "bill@example.org", + &["jdoe@example.org"], concat!( - "From: bill@example.com\r\n", - "To: jdoe@example.com\r\n", + "From: bill@example.org\r\n", + "To: jdoe@example.org\r\n", "Subject: TPS Report (should be encrypted)\r\n", "\r\n", "I'm going to need those TPS reports ASAP. ", @@ -81,11 +119,11 @@ pub async fn test(test: &mut TestServer) { // Send an encrypted message lmtp.ingest( - "bill@example.com", - &["jdoe@example.com"], + "bill@example.org", + &["jdoe@example.org"], concat!( - "From: bill@example.com\r\n", - "To: jdoe@example.com\r\n", + "From: bill@example.org\r\n", + "To: jdoe@example.org\r\n", "Subject: TPS Report (already encrypted)\r\n", "Content-Type: application/pkcs7-mime; name=\"smime.p7m\"; smime-type=enveloped-data\r\n", "\r\n", @@ -104,21 +142,23 @@ pub async fn test(test: &mut TestServer) { .await; // Disable encryption - assert_eq!( - api.post::>("/api/account/crypto", &EncryptionType::Disabled) - .await - .unwrap() - .unwrap_data(), - None - ); + account + .registry_update_object( + ObjectType::AccountSettings, + Id::singleton(), + json!({ + Property::EncryptionAtRest: EncryptionAtRest::Disabled + }), + ) + .await; // Send a new message, which should NOT be encrypted lmtp.ingest( - "bill@example.com", - &["jdoe@example.com"], + "bill@example.org", + &["jdoe@example.org"], concat!( - "From: bill@example.com\r\n", - "To: jdoe@example.com\r\n", + "From: bill@example.org\r\n", + "To: jdoe@example.org\r\n", "Subject: TPS Report (plain text)\r\n", "\r\n", "I'm going to need those TPS reports ASAP. ", @@ -156,69 +196,70 @@ pub async fn test(test: &mut TestServer) { panic!("Unexpected message: {:#?}", message) } } + + test.account("admin@example.org") + .destroy_account(account) + .await; + test.assert_is_empty().await; } pub async fn import_certs_and_encrypt() { - for (name, method, expected_certs) in [ - ("cert_pgp.pem", EncryptionMethod::PGP, 1), - //("cert_pgp.der", EncryptionMethod::PGP, 1), - ("cert_smime.pem", EncryptionMethod::SMIME, 3), - ("cert_smime.der", EncryptionMethod::SMIME, 1), + for (name, method) in [ + ("cert_pgp.pem", EncryptionMethod::PGP), + //("cert_pgp.der", EncryptionMethod::PGP), + ("cert_smime.pem", EncryptionMethod::SMIME), + //("cert_smime.der", EncryptionMethod::SMIME), ] { - let mut certs = try_parse_certs( - method, - std::fs::read( - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("resources") - .join("crypto") - .join(name), + let pk = PublicKey { + description: name.to_string(), + key: String::from_utf8( + std::fs::read( + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("resources") + .join("crypto") + .join(name), + ) + .unwrap(), ) .unwrap(), - ) - .expect(name); - - assert_eq!(certs.len(), expected_certs); - - if method == EncryptionMethod::PGP && certs.len() == 2 { - // PGP library won't encrypt using EC - let mut certs_ = certs.to_vec(); - certs_.pop(); - certs = certs_.into(); - } - - let mut params = EncryptionParams { - certs, - flags: method.flags(), + ..Default::default() }; - for algo in [Algorithm::Aes128, Algorithm::Aes256] { + let params = parse_public_key(&pk).unwrap().unwrap(); + assert_eq!(params.method, method); + + for mut flags in [ + ACCOUNT_FLAG_ENCRYPT_ALGO_AES128, + ACCOUNT_FLAG_ENCRYPT_ALGO_AES256, + ] { let message = MessageParser::new() .parse(b"Subject: test\r\ntest\r\n") .unwrap(); assert!(!message.is_encrypted()); - params.flags = algo.flags() | method.flags(); - let arch = - Archive::deserialize_owned(Archiver::new(params.clone()).serialize().unwrap()) - .unwrap(); - message - .encrypt(arch.unarchive::().unwrap()) - .await - .unwrap(); + flags |= match method { + EncryptionMethod::PGP => ACCOUNT_FLAG_ENCRYPT_METHOD_PGP, + EncryptionMethod::SMIME => ACCOUNT_FLAG_ENCRYPT_METHOD_SMIME, + }; + message.encrypt(¶ms.certs, flags).await.unwrap(); } } // S/MIME and PGP should not be allowed mixed assert!( - try_parse_certs( - EncryptionMethod::PGP, - std::fs::read( - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("resources") - .join("crypto") - .join("cert_mixed.pem"), + parse_public_key(&PublicKey { + description: "err".into(), + key: String::from_utf8( + std::fs::read( + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("resources") + .join("crypto") + .join("cert_mixed.pem"), + ) + .unwrap() ) .unwrap(), - ) + ..Default::default() + }) .is_err() ); } diff --git a/tests/src/system/delivery.rs b/tests/src/system/delivery.rs index 8dbf2896..363683a9 100644 --- a/tests/src/system/delivery.rs +++ b/tests/src/system/delivery.rs @@ -5,7 +5,7 @@ */ use crate::utils::{ - account::Account, imap::AssertResult, server::TestServer, smtp::SmtpConnection, + account::Account, imap::AssertResult, jmap::JmapUtils, server::TestServer, smtp::SmtpConnection, }; use common::{Server, auth::BuildAccessToken}; use email::{ @@ -15,19 +15,24 @@ use email::{ }; use groupware::DavResourceName; use jmap::blob::download::BlobDownload; +use jmap_proto::error::set::SetErrorType; use registry::{ schema::{ - prelude::ObjectType, - structs::{EmailAlias, MailingList, SpamTag, SpamTagScore, SpamTrainingSample}, + enums::StorageQuota, + prelude::{ObjectType, Property}, + structs::{ + EmailAlias, Expression, MailingList, MtaExtensions, SpamTag, SpamTagScore, + SpamTrainingSample, + }, }, - types::{float::Float, list::List, map::Map}, + types::{EnumImpl, datetime::UTCDateTime, float::Float, list::List, map::Map}, }; use serde_json::json; use std::time::Duration; use store::{ ValueKey, roaring::RoaringBitmap, - write::{AlignedBytes, Archive}, + write::{AlignedBytes, Archive, now}, }; use types::{ blob::{BlobClass, BlobId}, @@ -48,6 +53,23 @@ pub async fn test(test: &mut TestServer) { tag: "GTUBE_TEST".to_string(), })) .await; + admin + .registry_update_setting( + MtaExtensions { + expn: Expression { + else_: "true".to_string(), + ..Default::default() + }, + vrfy: Expression { + else_: "true".to_string(), + ..Default::default() + }, + ..Default::default() + }, + &[Property::Expn, Property::Vrfy], + ) + .await; + admin.reload_settings().await; // Create a domain name and a test account let john = test @@ -74,6 +96,15 @@ pub async fn test(test: &mut TestServer) { &[], ) .await; + admin + .registry_update_object( + ObjectType::Account, + john.id(), + json!({ + Property::Quotas: {StorageQuota::MaxMaskedAddresses.as_str(): 2} + }), + ) + .await; // Create a mailing list let domain_id = admin.find_or_create_domain("example.org").await; @@ -98,7 +129,6 @@ pub async fn test(test: &mut TestServer) { // Delivering to individuals let mut lmtp = SmtpConnection::connect().await; - lmtp.ingest( "bill@example.org", &["jdoe@example.org"], @@ -140,10 +170,69 @@ pub async fn test(test: &mut TestServer) { .is_none() ); - // Test spam filtering + // Masked email tests + john.registry_create_many( + ObjectType::MaskedEmail, + [json!({ + Property::EmailDomain: "invalid.org" + })], + ) + .await + .not_created(0) + .to_set_error() + .assert_type(SetErrorType::Forbidden) + .assert_description_contains("The specified domain is not valid for this account."); + + let response = john + .registry_create_many( + ObjectType::MaskedEmail, + [json!({ + Property::EmailDomain: "example.org", + Property::EmailPrefix: "secretive", + Property::ExpiresAt: UTCDateTime::from_timestamp((now() + 1) as i64) + })], + ) + .await; + let masked = response.created(0); + let masked_prefix_id = masked.object_id(); + let masked_prefix_email = masked.text_field("email").to_string(); + assert!( + masked_prefix_email.starts_with("secretive") + && masked_prefix_email.ends_with("@example.org"), + "Unexpected masked email: {masked_prefix_email}" + ); + + let response = john + .registry_create_many( + ObjectType::MaskedEmail, + [json!({ + Property::EmailDomain: "example.org", + })], + ) + .await; + let masked = response.created(0); + let masked_random_id = masked.object_id(); + let masked_random_email = masked.text_field("email").to_string(); + assert!( + masked_random_email.contains(".") && masked_random_email.ends_with("@example.org"), + "Unexpected masked email: {masked_random_email}" + ); + + john.registry_create_many( + ObjectType::MaskedEmail, + [json!({ + Property::EmailDomain: "example.org", + })], + ) + .await + .not_created(0) + .to_set_error() + .assert_type(SetErrorType::OverQuota); + + // Test spam filtering using masked email lmtp.ingest( "bill@example.org", - &["john.doe@example.org"], + &[masked_prefix_email.as_str()], concat!( "From: bill@example.org\r\n", "To: john.doe@example.org\r\n", @@ -180,7 +269,7 @@ pub async fn test(test: &mut TestServer) { .await; assert_eq!(john.spam_training_samples().await, vec![]); - // CardDAV spam override + // CardDAV spam override, using masked email let dav_client = john.webdav_client(); dav_client .request( @@ -201,7 +290,7 @@ END:VCARD .with_status(hyper::StatusCode::CREATED); lmtp.ingest( "dmarc-bill@example.org", - &["john.doe@example.org"], + &[masked_random_email.as_str()], concat!( "From: dmarc-bill@example.org\r\n", "To: john.doe@example.org\r\n", @@ -238,8 +327,8 @@ END:VCARD ) .await; let samples = john.spam_training_samples().await; - assert_eq!(samples.iter().filter(|x| x.1.is_spam).count(), 1); - assert_eq!(samples.iter().filter(|x| !x.1.is_spam).count(), 0); + assert_eq!(samples.iter().filter(|x| !x.1.is_spam).count(), 1); + assert_eq!(samples.iter().filter(|x| x.1.is_spam).count(), 0); // Test trusted reply override john.jmap_client() @@ -312,8 +401,8 @@ END:VCARD ) .await; let samples = john.spam_training_samples().await; - assert_eq!(samples.iter().filter(|x| x.1.is_spam).count(), 2); - assert_eq!(samples.iter().filter(|x| !x.1.is_spam).count(), 0); + assert_eq!(samples.iter().filter(|x| !x.1.is_spam).count(), 2); + assert_eq!(samples.iter().filter(|x| x.1.is_spam).count(), 0); // EXPN and VRFY lmtp.expn("members@example.org", 2) @@ -326,6 +415,8 @@ END:VCARD lmtp.vrfy("jdoe@example.org", 2).await; lmtp.vrfy("members@example.org", 5).await; lmtp.vrfy("non_existant@example.org", 5).await; + lmtp.vrfy(masked_random_email.as_str(), 2).await; + lmtp.vrfy(masked_prefix_email.as_str(), 5).await; // Should have expired // Delivering to a mailing list lmtp.ingest( @@ -359,14 +450,6 @@ END:VCARD ); } - let todos = "todo"; - /* - - MaskedEmail (receiving, expiring, not accessing other users' masked addresses) - - SpamSamples, can't access from other accounts but admin can using filter - - Catchall? Subaddressing? - - Review other code points for more testing ideas - */ - // Removing members from the mailing list and chunked ingest admin .registry_update_object( @@ -446,11 +529,12 @@ END:VCARD let metadata = message_metadata(&test.server, account_id, document_id).await; let partial_message = test .server - .store() + .blob_store() .get_blob(metadata.blob_hash.0.as_ref(), 0..usize::MAX) .await .unwrap() .unwrap(); + assert_ne!(metadata.blob_body_offset, 0); let expected_full_message = String::from_utf8( ChainedBytes::new(metadata.raw_headers.as_ref()) @@ -491,9 +575,20 @@ END:VCARD } // Remove test data + john.registry_destroy( + ObjectType::MaskedEmail, + [masked_prefix_id, masked_random_id], + ) + .await + .assert_destroyed(&[masked_prefix_id, masked_random_id]); for account in [&john, &jane, &bill] { test.destroy_all_mailboxes(account).await; } + admin.registry_destroy_all(ObjectType::MailingList).await; + admin + .registry_destroy_all(ObjectType::SpamTrainingSample) + .await; + admin.registry_destroy_all(ObjectType::SpamTag).await; test.assert_is_empty().await; for account in [john, jane, bill] { diff --git a/tests/src/system/directory.rs b/tests/src/system/directory.rs index f15ad963..1525a405 100644 --- a/tests/src/system/directory.rs +++ b/tests/src/system/directory.rs @@ -4,16 +4,19 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::utils::server::TestServer; -use common::auth::{ACCOUNT_IS_USER, EmailAddress, EmailCache}; +use crate::utils::{jmap::JmapUtils, server::TestServer}; +use common::{ + auth::{ACCOUNT_IS_USER, EmailAddress, EmailCache}, + network::RcptResolution, +}; use jmap_proto::error::set::SetErrorType; use registry::{ schema::{ enums::{AccountType, StorageQuota}, prelude::{ObjectType, Property}, structs::{ - Account, Credential, Domain, EmailAlias, GroupAccount, MailingList, PasswordCredential, - UserAccount, + Account, Credential, Domain, EmailAlias, Expression, ExpressionMatch, GroupAccount, + MailingList, PasswordCredential, SubAddressing, SubAddressingCustom, UserAccount, }, }, types::{EnumImpl, list::List, map::Map}, @@ -32,6 +35,7 @@ pub async fn test(test: &TestServer) { aliases: Map::new(vec!["beispiel.de".to_string()]), is_enabled: true, catch_all_address: Some("catchy@example.com".to_string()), + sub_addressing: SubAddressing::Enabled, ..Default::default() }) .await; @@ -227,6 +231,35 @@ pub async fn test(test: &TestServer) { let account_cache = test.server.account(account_id.document_id()).await.unwrap(); assert!(account_cache.id_member_of.as_ref().is_empty()); + // Create a masked email + let john = + crate::utils::account::Account::new("johndoe@example.com", "hello world", &[], account_id); + let response = john + .registry_create_many( + ObjectType::MaskedEmail, + [json!({ + Property::EmailPrefix: "test", + })], + ) + .await; + let masked = response.created(0); + let masked_id = masked.object_id(); + let masked_email = masked.text_field("email").to_string(); + assert_eq!( + test.server + .account_id_from_email("johndoe@example.com", true) + .await + .unwrap(), + Some(account_id.document_id()) + ); + assert_eq!( + test.server + .account_id_from_email(&masked_email, true) + .await + .unwrap(), + Some(account_id.document_id()) + ); + // Create a mailing list let list_id = account .registry_create_object(MailingList { @@ -327,6 +360,108 @@ pub async fn test(test: &TestServer) { None ); + // MTA rcpt resolve + let domain_2_id = account + .registry_create_object(Domain { + name: "another-example.com".to_string(), + is_enabled: true, + sub_addressing: SubAddressing::Custom(SubAddressingCustom { + custom_rule: Expression { + else_: "false".to_string(), + match_: List::from_iter([ExpressionMatch { + if_: "matches('^([^.]+)\\.([^.]+)', rcpt)".to_string(), + then: "$1".to_string(), + }]), + }, + }), + ..Default::default() + }) + .await; + let account_2_id = account + .registry_create_object(Account::User(UserAccount { + name: "subaddresser".to_string(), + domain_id: domain_2_id, + ..Default::default() + })) + .await; + assert_eq!( + test.server.rcpt_resolve("unknown", 0).await.unwrap(), + RcptResolution::UnknownDomain + ); + assert_eq!( + test.server + .rcpt_resolve("unknown@unknown.org", 0) + .await + .unwrap(), + RcptResolution::UnknownDomain + ); + assert_eq!( + test.server + .rcpt_resolve("jdoe@example.com", 0) + .await + .unwrap(), + RcptResolution::Accept + ); + assert_eq!( + test.server + .rcpt_resolve("johndoe@beispiel.de", 0) + .await + .unwrap(), + RcptResolution::Accept + ); + assert_eq!( + test.server + .rcpt_resolve("sales@example.com", 0) + .await + .unwrap(), + RcptResolution::Accept + ); + assert_eq!( + test.server + .rcpt_resolve("jdoe+promotions@example.com", 0) + .await + .unwrap(), + RcptResolution::Rewrite("jdoe@example.com".into()) + ); + assert_eq!( + test.server + .rcpt_resolve("newsletter@example.com", 0) + .await + .unwrap(), + RcptResolution::Expand(Arc::from(Box::from_iter([ + "jdoe@example.com".into(), + "sales@example.com".into() + ]))) + ); + assert_eq!( + test.server + .rcpt_resolve("unknown@example.com", 0) + .await + .unwrap(), + RcptResolution::Rewrite("catchy@example.com".into()) + ); + assert_eq!( + test.server + .rcpt_resolve("subaddresser.ignoreme@another-example.com", 0) + .await + .unwrap(), + RcptResolution::Rewrite("subaddresser@another-example.com".into()) + ); + assert_eq!( + test.server + .rcpt_resolve("unknown@another-example.com", 0) + .await + .unwrap(), + RcptResolution::UnknownRecipient + ); + assert_eq!( + test.server + .rcpt_resolve(masked_email.as_str(), 0) + .await + .unwrap(), + RcptResolution::Rewrite("johndoe@example.com".into()) + ); + // Query tests assert_eq!( account @@ -354,30 +489,21 @@ pub async fn test(test: &TestServer) { ); // Delete everything - assert_eq!( - account - .registry_destroy(ObjectType::MailingList, [list_id]) - .await - .destroyed_ids() - .collect::>(), - vec![list_id] - ); - assert_eq!( - account - .registry_destroy(ObjectType::Account, [group_id, account_id]) - .await - .destroyed_ids() - .collect::>(), - vec![group_id, account_id] - ); - assert_eq!( - account - .registry_destroy(ObjectType::Domain, [domain_id]) - .await - .destroyed_ids() - .collect::>(), - vec![domain_id] - ); + john.registry_destroy(ObjectType::MaskedEmail, [masked_id]) + .await + .assert_destroyed(&[masked_id]); + account + .registry_destroy(ObjectType::MailingList, [list_id]) + .await + .assert_destroyed(&[list_id]); + account + .registry_destroy(ObjectType::Account, [group_id, account_id, account_2_id]) + .await + .assert_destroyed(&[group_id, account_id, account_2_id]); + account + .registry_destroy(ObjectType::Domain, [domain_id, domain_2_id]) + .await + .assert_destroyed(&[domain_id, domain_2_id]); assert!( test.server .try_list(list_id.document_id()) @@ -400,6 +526,13 @@ pub async fn test(test: &TestServer) { .is_none() ); assert!(test.server.domain("example.com").await.unwrap().is_none()); + assert!( + test.server + .domain("another-example.com") + .await + .unwrap() + .is_none() + ); test.assert_is_empty().await; } diff --git a/tests/src/system/mod.rs b/tests/src/system/mod.rs index a9d9e019..be1cc52f 100644 --- a/tests/src/system/mod.rs +++ b/tests/src/system/mod.rs @@ -4,8 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +pub mod antispam; +pub mod archiving; pub mod authentication; pub mod authorization; +pub mod crypto; pub mod delivery; pub mod directory; pub mod oidc; @@ -15,7 +18,7 @@ pub mod security; pub mod tenant; use crate::utils::server::TestServerBuilder; -use registry::schema::structs::{Imap, SpamClassifier}; +use registry::schema::structs::{Expression, Imap, MtaStageAuth}; #[tokio::test(flavor = "multi_thread")] pub async fn system_tests() { @@ -28,8 +31,11 @@ pub async fn system_tests() { ..Default::default() }) .await - .with_object(SpamClassifier { - hold_samples_for: 1u64.into(), + .with_object(MtaStageAuth { + require: Expression { + else_: "false".to_string(), + ..Default::default() + }, ..Default::default() }) .await @@ -50,7 +56,7 @@ pub async fn system_tests() { .await; test.insert_account(admin); - let todo = "test permissions on account filtered objects"; + let todo = "test tasks retries and other types"; //directory::test(&test).await; //authentication::test(&test).await; @@ -60,5 +66,8 @@ pub async fn system_tests() { //security::test(&mut test).await; //quota::test(&mut test).await; //purge::test(&mut test).await; - delivery::test(&mut test).await; + //delivery::test(&mut test).await; + //crypto::test(&mut test).await; + //antispam::test(&mut test).await; + archiving::test(&mut test).await; } diff --git a/tests/src/system/purge.rs b/tests/src/system/purge.rs index 06da060c..150a761c 100644 --- a/tests/src/system/purge.rs +++ b/tests/src/system/purge.rs @@ -18,7 +18,10 @@ use imap_proto::ResponseType; use registry::schema::{ enums::{TaskAccountMaintenanceType, TaskStoreMaintenanceType}, prelude::Property, - structs::{DataRetention, Task, TaskAccountMaintenance, TaskStatus, TaskStoreMaintenance}, + structs::{ + DataRetention, SpamClassifier, Task, TaskAccountMaintenance, TaskStatus, + TaskStoreMaintenance, + }, }; use store::{IterateParams, LogKey, U32_LEN, U64_LEN, write::key::DeserializeBigEndian}; use types::id::Id; @@ -41,6 +44,15 @@ pub async fn test(test: &mut TestServer) { &[Property::MaxChangesHistory, Property::ExpungeTrashAfter], ) .await; + admin + .registry_update_setting( + SpamClassifier { + hold_samples_for: 1u64.into(), + ..Default::default() + }, + &[Property::HoldSamplesFor], + ) + .await; admin.reload_settings().await; // Create test account @@ -193,6 +205,11 @@ pub async fn test(test: &mut TestServer) { admin.destroy_account(account).await; test.wait_for_tasks().await; test.assert_is_empty().await; + + // Reset settings + admin + .registry_update_setting(SpamClassifier::default(), &[Property::HoldSamplesFor]) + .await; } async fn get_changes(server: &Server) -> (AHashSet<(u64, u8)>, bool) { diff --git a/tests/src/system/quota.rs b/tests/src/system/quota.rs index 0ace0aab..eb0d045d 100644 --- a/tests/src/system/quota.rs +++ b/tests/src/system/quota.rs @@ -16,8 +16,8 @@ use registry::{ enums::{Permission, StorageQuota, TaskAccountMaintenanceType}, prelude::{ObjectType, Property}, structs::{ - self, Credential, Expression, Jmap, MtaStageAuth, PasswordCredential, PermissionsList, - Task, TaskAccountMaintenance, TaskStatus, UserAccount, + self, Credential, Jmap, PasswordCredential, PermissionsList, Task, + TaskAccountMaintenance, TaskStatus, UserAccount, }, }, types::{EnumImpl, list::List, map::Map}, @@ -47,18 +47,6 @@ pub async fn test(test: &mut TestServer) { ], ) .await; - admin - .registry_update_setting( - MtaStageAuth { - require: Expression { - else_: "false".to_string(), - ..Default::default() - }, - ..Default::default() - }, - &[Property::Require], - ) - .await; admin.reload_settings().await; // Create test accounts diff --git a/tests/src/system/tenant.rs b/tests/src/system/tenant.rs index 77b3f522..0c886b37 100644 --- a/tests/src/system/tenant.rs +++ b/tests/src/system/tenant.rs @@ -297,15 +297,16 @@ pub async fn test(test: &mut TestServer) { .build(); assert!(!user_x_at.has_permission(Permission::FetchAnyBlob)); assert!(!user_x_at.has_permission(Permission::Impersonate)); + admin_x .registry_update_object( ObjectType::Account, user_x_id, json!({ - Property::Permissions: Permissions::Merge( - PermissionsList { disabled_permissions: Map::default(), - enabled_permissions: Map::new(vec![Permission::FetchAnyBlob, Permission::Impersonate]) - }) + Property::Permissions: Permissions::Merge(PermissionsList { + disabled_permissions: Map::default(), + enabled_permissions: Map::new(vec![Permission::FetchAnyBlob, Permission::Impersonate]), + }) }), ) .await; diff --git a/tests/src/utils/account.rs b/tests/src/utils/account.rs index cc9c5653..169cf11e 100644 --- a/tests/src/utils/account.rs +++ b/tests/src/utils/account.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::utils::{server::TestServer, webdav::DummyWebDavClient}; +use crate::utils::{imap::ImapConnection, server::TestServer, webdav::DummyWebDavClient}; use ahash::AHashMap; use jmap_client::client::{Client, Credentials}; use registry::{ @@ -190,6 +190,12 @@ impl Account { ) } + pub async fn imap_client(&self) -> ImapConnection { + let mut imap = ImapConnection::connect(b"_x ").await; + imap.authenticate(self.name(), self.secret()).await; + imap + } + pub async fn jmap_client(&self) -> Client { let mut client = Client::new() .credentials(Credentials::basic(self.name(), self.secret())) diff --git a/tests/src/utils/cleanup.rs b/tests/src/utils/cleanup.rs index 7c827a1c..3e900840 100644 --- a/tests/src/utils/cleanup.rs +++ b/tests/src/utils/cleanup.rs @@ -15,7 +15,6 @@ use store::{ }; use trc::AddContext; use types::blob_hash::{BLOB_HASH_LEN, BlobHash}; -use utils::codec::leb128::Leb128Reader; pub async fn store_destroy(store: &Store) { store_destroy_sql_indexes(store).await; @@ -311,7 +310,7 @@ pub async fn store_assert_is_empty(store: &Store, blob_store: BlobStore, include if include_registry && is_allowed_registry_type(object_id) { return Ok(true); } - let item_id = key.read_leb128::().unwrap().0; + let item_id = key.deserialize_be_u64(U16_LEN).unwrap(); println!( "Found registry item for object type {:?} and id {}", diff --git a/tests/src/utils/jmap.rs b/tests/src/utils/jmap.rs index 72d9e918..9dbe7b75 100644 --- a/tests/src/utils/jmap.rs +++ b/tests/src/utils/jmap.rs @@ -646,6 +646,8 @@ pub trait JmapUtils { self.text_field("description") } + fn to_set_error(&self) -> JmapSetError; + fn with_property(self, field: impl Display, value: impl Into) -> Self; fn text_field(&self, field: &str) -> &str; @@ -668,6 +670,10 @@ impl JmapUtils for Value { .unwrap_or_else(|| panic!("Missing {field} in object: {self:?}")) } + fn to_set_error(&self) -> JmapSetError { + serde_json::from_str(&self.to_string()).expect("Failed to deserialize set error") + } + fn assert_is_equal(&self, expected: Value) { if self != &expected { panic!( diff --git a/tests/src/utils/registry.rs b/tests/src/utils/registry.rs index 5fc0da63..66b8d148 100644 --- a/tests/src/utils/registry.rs +++ b/tests/src/utils/registry.rs @@ -6,7 +6,7 @@ use crate::utils::{ account::Account, - jmap::{JmapResponse, JmapSetError}, + jmap::{JmapResponse, JmapSetError, JmapUtils}, }; use registry::{ schema::{ @@ -41,6 +41,16 @@ impl Account { .await } + pub async fn registry_create_many( + &self, + object_type: ObjectType, + items: impl IntoIterator, + ) -> JmapResponse { + let name = object_type.as_str(); + self.jmap_create_account(self, format!("x:{name}"), items, Vec::<(&str, &str)>::new()) + .await + } + pub async fn registry_get(&self, id: Id) -> T { let name = T::OBJECT.as_str(); @@ -149,12 +159,10 @@ impl Account { } pub async fn registry_create_object_expect_err(&self, item: T) -> JmapSetError { - let v = self - .registry_create([item]) + self.registry_create([item]) .await .not_created(0) - .to_string(); - serde_json::from_str(&v).expect("Failed to deserialize set error") + .to_set_error() } pub async fn registry_update_object(&self, object: ObjectType, id: Id, item: Value) { @@ -192,12 +200,10 @@ impl Account { id: Id, item: Value, ) -> JmapSetError { - let v = self - .registry_update(object, [(id, item)]) + self.registry_update(object, [(id, item)]) .await .not_updated(&id.to_string()) - .to_string(); - serde_json::from_str(&v).expect("Failed to deserialize set error") + .to_set_error() } pub async fn registry_destroy_object_expect_err( @@ -205,12 +211,10 @@ impl Account { object: ObjectType, id: Id, ) -> JmapSetError { - let v = self - .registry_destroy(object, [id]) + self.registry_destroy(object, [id]) .await .not_destroyed(&id.to_string()) - .to_string(); - serde_json::from_str(&v).expect("Failed to deserialize set error") + .to_set_error() } pub async fn destroy_account(&self, account: Account) { diff --git a/tests/src/utils/webdav.rs b/tests/src/utils/webdav.rs index 3bac9672..b7dc94d0 100644 --- a/tests/src/utils/webdav.rs +++ b/tests/src/utils/webdav.rs @@ -4,8 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{borrow::Cow, time::Duration}; - use ahash::{AHashMap, AHashSet}; use base64::{Engine, engine::general_purpose::STANDARD}; use dav_proto::{ @@ -16,6 +14,7 @@ use dav_proto::{ use groupware::DavResourceName; use hyper::{HeaderMap, Method, StatusCode, header::AUTHORIZATION}; use quick_xml::{Reader, events::Event}; +use std::{borrow::Cow, time::Duration}; use store::rand::{Rng, distr::Alphanumeric, rng}; #[allow(dead_code)] @@ -611,8 +610,7 @@ impl DavResponse { pub fn with_body(self, expect_body: impl AsRef) -> Self { let expect_body = expect_body.as_ref(); - if self.body.is_ok() { - let body = self.body.as_ref().unwrap(); + if let Ok(body) = &self.body { if body != expect_body { self.dump_response(); assert_eq!(body, &expect_body); @@ -625,8 +623,7 @@ impl DavResponse { } pub fn with_empty_body(self) -> Self { - if self.body.is_ok() { - let body = self.body.as_ref().unwrap(); + if let Ok(body) = &self.body { if !body.is_empty() { self.dump_response(); panic!("Expected empty body but got {body:?}"); @@ -639,8 +636,8 @@ impl DavResponse { } pub fn expect_body(&self) -> &str { - if self.body.is_ok() { - self.body.as_ref().unwrap() + if let Ok(body) = &self.body { + body } else { self.dump_response(); panic!("Expected body but no body was returned.")