diff --git a/Cargo.lock b/Cargo.lock index c2eebee9..899cd522 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1423,6 +1423,7 @@ dependencies = [ "async-trait", "deadpool", "futures", + "jmap_proto", "ldap3", "lru-cache", "mail-builder", diff --git a/crates/directory/Cargo.toml b/crates/directory/Cargo.toml index 6f9d5d26..c3a86da5 100644 --- a/crates/directory/Cargo.toml +++ b/crates/directory/Cargo.toml @@ -7,6 +7,7 @@ resolver = "2" [dependencies] utils = { path = "../utils" } store = { path = "../store" } +jmap_proto = { path = "../jmap-proto" } smtp-proto = { git = "https://github.com/stalwartlabs/smtp-proto" } mail-parser = { git = "https://github.com/stalwartlabs/mail-parser", features = ["full_encoding", "serde_support", "ludicrous_mode"] } mail-send = { git = "https://github.com/stalwartlabs/mail-send", default-features = false, features = ["cram-md5", "skip-ehlo"] } diff --git a/crates/directory/src/imap/client.rs b/crates/directory/src/backend/imap/client.rs similarity index 99% rename from crates/directory/src/imap/client.rs rename to crates/directory/src/backend/imap/client.rs index a2daedc6..0ff98369 100644 --- a/crates/directory/src/imap/client.rs +++ b/crates/directory/src/backend/imap/client.rs @@ -199,7 +199,7 @@ mod test { use smtp_proto::{AUTH_OAUTHBEARER, AUTH_PLAIN, AUTH_XOAUTH, AUTH_XOAUTH2}; use std::time::Duration; - use crate::imap::ImapClient; + use crate::backend::imap::ImapClient; #[ignore] #[tokio::test] diff --git a/crates/directory/src/imap/config.rs b/crates/directory/src/backend/imap/config.rs similarity index 95% rename from crates/directory/src/imap/config.rs rename to crates/directory/src/backend/imap/config.rs index 5f4f20f1..9859a25d 100644 --- a/crates/directory/src/imap/config.rs +++ b/crates/directory/src/backend/imap/config.rs @@ -26,9 +26,9 @@ use std::sync::Arc; use mail_send::smtp::tls::build_tls_connector; use utils::config::{utils::AsKey, Config}; -use crate::{cache::CachedDirectory, config::build_pool, imap::ImapConnectionManager, Directory}; +use crate::{cache::CachedDirectory, config::build_pool, Directory}; -use super::ImapDirectory; +use super::{ImapConnectionManager, ImapDirectory}; impl ImapDirectory { pub fn from_config( diff --git a/crates/directory/src/backend/imap/lookup.rs b/crates/directory/src/backend/imap/lookup.rs new file mode 100644 index 00000000..2208884e --- /dev/null +++ b/crates/directory/src/backend/imap/lookup.rs @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use mail_send::Credentials; +use smtp_proto::{AUTH_CRAM_MD5, AUTH_LOGIN, AUTH_OAUTHBEARER, AUTH_PLAIN, AUTH_XOAUTH2}; +use store::Store; + +use crate::{Directory, DirectoryError, Principal, QueryBy, QueryType}; + +use super::{ImapDirectory, ImapError}; + +#[async_trait::async_trait] +impl Directory for ImapDirectory { + async fn query(&self, query: QueryBy<'_>) -> crate::Result> { + if let QueryType::Credentials(credentials) = query.t { + let mut client = self.pool.get().await?; + let mechanism = match credentials { + Credentials::Plain { .. } + if (client.mechanisms & (AUTH_PLAIN | AUTH_LOGIN | AUTH_CRAM_MD5)) != 0 => + { + if client.mechanisms & AUTH_CRAM_MD5 != 0 { + AUTH_CRAM_MD5 + } else if client.mechanisms & AUTH_PLAIN != 0 { + AUTH_PLAIN + } else { + AUTH_LOGIN + } + } + Credentials::OAuthBearer { .. } if client.mechanisms & AUTH_OAUTHBEARER != 0 => { + AUTH_OAUTHBEARER + } + Credentials::XOauth2 { .. } if client.mechanisms & AUTH_XOAUTH2 != 0 => { + AUTH_XOAUTH2 + } + _ => { + tracing::warn!( + context = "remote", + event = "error", + protocol = "imap", + "IMAP server does not offer any supported auth mechanisms.", + ); + return Ok(None); + } + }; + + match client.authenticate(mechanism, credentials).await { + Ok(_) => { + client.is_valid = false; + Ok(Some(Principal::default())) + } + Err(err) => match &err { + ImapError::AuthenticationFailed => Ok(None), + _ => Err(err.into()), + }, + } + } else { + Err(DirectoryError::unsupported("imap", "query")) + } + } + + async fn email_to_ids(&self, _address: &str, _store: &Store) -> crate::Result> { + Err(DirectoryError::unsupported("imap", "email_to_ids")) + } + + async fn rcpt(&self, _address: &str) -> crate::Result { + Err(DirectoryError::unsupported("imap", "rcpt")) + } + + async fn vrfy(&self, _address: &str) -> crate::Result> { + Err(DirectoryError::unsupported("imap", "vrfy")) + } + + async fn expn(&self, _address: &str) -> crate::Result> { + Err(DirectoryError::unsupported("imap", "expn")) + } + + async fn is_local_domain(&self, domain: &str) -> crate::Result { + Ok(self.domains.contains(domain)) + } +} diff --git a/crates/directory/src/imap/mod.rs b/crates/directory/src/backend/imap/mod.rs similarity index 100% rename from crates/directory/src/imap/mod.rs rename to crates/directory/src/backend/imap/mod.rs diff --git a/crates/directory/src/imap/pool.rs b/crates/directory/src/backend/imap/pool.rs similarity index 100% rename from crates/directory/src/imap/pool.rs rename to crates/directory/src/backend/imap/pool.rs diff --git a/crates/directory/src/imap/tls.rs b/crates/directory/src/backend/imap/tls.rs similarity index 100% rename from crates/directory/src/imap/tls.rs rename to crates/directory/src/backend/imap/tls.rs diff --git a/crates/directory/src/backend/internal/manage.rs b/crates/directory/src/backend/internal/manage.rs new file mode 100644 index 00000000..71e03247 --- /dev/null +++ b/crates/directory/src/backend/internal/manage.rs @@ -0,0 +1,229 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use jmap_proto::types::collection::Collection; +use store::{ + write::{assert::HashedValue, BatchBuilder, DirectoryValue, ValueClass}, + Serialize, Store, ValueKey, +}; + +use crate::{Principal, Type}; + +#[async_trait::async_trait] +pub trait ManageDirectory { + async fn delete_account_by_name(&self, name: &str) -> store::Result; + async fn delete_account_by_id(&self, id: u32) -> store::Result; + async fn rename_account(&self, name: &str, new_name: String) -> store::Result; + async fn get_account_id(&self, name: &str) -> store::Result>; + async fn get_or_create_account_id(&self, name: &str) -> crate::Result; +} + +#[async_trait::async_trait] +impl ManageDirectory for Store { + async fn get_account_id(&self, name: &str) -> store::Result> { + self.get_value::(ValueKey::from(ValueClass::Directory( + DirectoryValue::NameToId(name.as_bytes().to_vec()), + ))) + .await + } + + // Used by all directories except internal + async fn get_or_create_account_id(&self, name: &str) -> crate::Result { + let mut try_count = 0; + + loop { + // Try to obtain ID + if let Some(account_id) = self.get_account_id(name).await? { + return Ok(account_id); + } + + // Assign new ID + let account_id = self + .assign_document_id(u32::MAX, Collection::Principal) + .await?; + + // Write account ID + let name_key = + ValueClass::Directory(DirectoryValue::NameToId(name.as_bytes().to_vec())); + let mut batch = BatchBuilder::new(); + batch + .with_account_id(u32::MAX) + .with_collection(Collection::Principal) + .create_document(account_id) + .assert_value(name_key.clone(), ()) + .set(name_key, account_id.serialize()) + .set( + ValueClass::Directory(DirectoryValue::Principal(account_id)), + Principal { + id: account_id, + typ: Type::Individual, + name: name.to_string(), + ..Default::default() + } + .serialize(), + ); + + match self.write(batch.build()).await { + Ok(_) => { + return Ok(account_id); + } + Err(store::Error::AssertValueFailed) if try_count < 3 => { + try_count += 1; + continue; + } + Err(err) => { + tracing::error!(event = "error", + context = "store", + error = ?err, + "Failed to generate account id"); + return Err(err.into()); + } + } + } + } + + async fn delete_account_by_name(&self, name: &str) -> store::Result { + if let Some(account_id) = self.get_account_id(name).await? { + self.delete_account_by_id(account_id).await + } else { + Ok(false) + } + } + + async fn delete_account_by_id(&self, account_id: u32) -> store::Result { + let principal = if let Some(principal) = self + .get_value::(ValueKey::from(ValueClass::Directory( + DirectoryValue::Principal(account_id), + ))) + .await? + { + principal + } else { + return Ok(false); + }; + + // Unlink all account's blobs + self.blob_hash_unlink_account(account_id).await?; + + // Revoke ACLs + self.acl_revoke_all(account_id).await?; + + // Delete account + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .clear(DirectoryValue::NameToId(principal.name.as_bytes().to_vec())) + .clear(DirectoryValue::Principal(account_id)) + .clear(DirectoryValue::UsedQuota(account_id)); + + for email in principal.emails { + batch.clear(DirectoryValue::EmailToId(email.as_bytes().to_vec())); + } + + self.write(batch.build()).await?; + + // Delete account data + self.purge_account(account_id).await?; + + Ok(true) + } + + async fn rename_account(&self, name: &str, new_name: String) -> store::Result { + if let Some(account_id) = self.get_account_id(name).await? { + if let Some(mut principal) = self + .get_value::>(ValueKey::from(ValueClass::Directory( + DirectoryValue::Principal(account_id), + ))) + .await? + { + if principal.inner.name != name { + return Ok(false); + } + principal.inner.name = new_name.clone(); + + let mut batch = BatchBuilder::new(); + batch + .assert_value( + ValueClass::Directory(DirectoryValue::Principal(account_id)), + &principal, + ) + .set( + ValueClass::Directory(DirectoryValue::Principal(account_id)), + principal.inner.serialize(), + ) + .clear(ValueClass::Directory(DirectoryValue::NameToId( + name.as_bytes().to_vec(), + ))) + .set( + ValueClass::Directory(DirectoryValue::NameToId(new_name.into_bytes())), + account_id.serialize(), + ); + self.write(batch.build()).await?; + + return Ok(true); + } + } + + Ok(false) + } +} + +/* +pub async fn try_get_account_id(store: &Store, name: &str) -> crate::Result> { + store + .get_value::(NamedKey::Name(name)) + .await + .map_err(|err| { + tracing::error!(event = "error", + context = "store", + account_name = name, + error = ?err, + "Failed to retrieve account id"); + MethodError::ServerPartialFail + }) +} + + + +pub async fn map_member_of(store: &Store, names: Vec) -> crate::Result> { + let mut ids = Vec::with_capacity(names.len()); + for name in names { + ids.push(self.get_account_id(&name).await?); + } + Ok(ids) +} + +pub async fn get_account_name(store: &Store, account_id: u32) -> crate::Result> { + store + .get_value::(NamedKey::Id::<&[u8]>(account_id)) + .await + .map_err(|err| { + tracing::error!(event = "error", + context = "store", + account_id = account_id, + error = ?err, + "Failed to retrieve account name"); + MethodError::ServerPartialFail + }) +} +*/ diff --git a/crates/directory/src/backend/internal/mod.rs b/crates/directory/src/backend/internal/mod.rs new file mode 100644 index 00000000..73f47163 --- /dev/null +++ b/crates/directory/src/backend/internal/mod.rs @@ -0,0 +1,225 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +pub mod manage; + +use std::slice::Iter; + +use mail_send::Credentials; +use store::{ + write::{key::KeySerializer, DirectoryValue, ValueClass}, + Deserialize, Serialize, Store, ValueKey, U32_LEN, +}; +use utils::codec::leb128::Leb128Iterator; + +use crate::{Principal, QueryBy, QueryType, Type}; + +use self::manage::ManageDirectory; + +pub struct InternalDirectory { + pub store: Store, +} + +impl<'x> QueryBy<'x> { + pub fn name(name: &'x str) -> Self { + Self { + t: QueryType::Name(name), + store: None, + } + } + + pub fn id(id: u32) -> Self { + Self { + t: QueryType::Id(id), + store: None, + } + } + + pub fn credentials(credentials: &'x Credentials) -> Self { + Self { + t: QueryType::Credentials(credentials), + store: None, + } + } + + pub fn with_store(mut self, store: &'x Store) -> Self { + self.store = Some(store); + self + } + + pub fn has_store(&self) -> bool { + self.store.is_some() + } + + pub async fn account_name(&self, account_id: u32) -> crate::Result> { + self.store + .unwrap() + .get_value::(ValueKey::from(ValueClass::Directory( + DirectoryValue::Principal(account_id), + ))) + .await + .map_err(Into::into) + .map(|v| { + if let Some(v) = v { + Some(v.name) + } else { + tracing::debug!( + context = "directory", + event = "not_found", + account = account_id, + "Principal not found for account id" + ); + + None + } + }) + } + + pub async fn account_id(&self, name: &str) -> crate::Result { + self.store + .unwrap() + .get_or_create_account_id(name) + .await + .map_err(Into::into) + } + + pub async fn account_ids>( + &self, + items: impl Iterator, + ) -> crate::Result> { + let mut ids = Vec::new(); + for item in items { + ids.push(self.account_id(item.as_ref()).await?); + } + Ok(ids) + } +} + +impl Serialize for Principal { + fn serialize(self) -> Vec { + let mut serializer = KeySerializer::new( + U32_LEN * 3 + + 2 + + self.name.len() + + self.emails.iter().map(|s| s.len()).sum::() + + self.secrets.iter().map(|s| s.len()).sum::() + + self.member_of.len() * U32_LEN + + self.description.as_ref().map(|s| s.len()).unwrap_or(0), + ) + .write(1u8) + .write_leb128(self.id) + .write(self.typ as u8) + .write_leb128(self.quota) + .write_leb128(self.name.len()) + .write(self.name.as_bytes()) + .write_leb128(self.description.as_ref().map_or(0, |s| s.len())) + .write(self.description.as_deref().unwrap_or_default().as_bytes()); + + for list in [&self.secrets, &self.emails] { + serializer = serializer.write_leb128(list.len()); + for value in list { + serializer = serializer.write_leb128(value.len()).write(value.as_bytes()); + } + } + + serializer = serializer.write_leb128(self.member_of.len()); + for id in &self.member_of { + serializer = serializer.write_leb128(*id); + } + + serializer.finalize() + } +} + +impl Deserialize for Principal { + fn deserialize(bytes: &[u8]) -> store::Result { + deserialize(bytes) + .ok_or_else(|| store::Error::InternalError("Failed to deserialize principal".into())) + } +} + +fn deserialize(bytes: &[u8]) -> Option { + let mut bytes = bytes.iter(); + if bytes.next()? != &1 { + return None; + } + + Principal { + id: bytes.next_leb128()?, + typ: Type::from_u8(*bytes.next()?), + quota: bytes.next_leb128()?, + name: deserialize_string(&mut bytes)?, + description: deserialize_string(&mut bytes).map(|v| { + if !v.is_empty() { + Some(v) + } else { + None + } + })?, + secrets: deserialize_string_list(&mut bytes)?, + emails: deserialize_string_list(&mut bytes)?, + member_of: deserialize_u32_list(&mut bytes)?, + } + .into() +} + +fn deserialize_string(bytes: &mut Iter<'_, u8>) -> Option { + let len = bytes.next_leb128()?; + let mut string = Vec::with_capacity(len); + for _ in 0..len { + string.push(*bytes.next()?); + } + String::from_utf8(string).ok() +} + +fn deserialize_string_list(bytes: &mut Iter<'_, u8>) -> Option> { + let len = bytes.next_leb128()?; + let mut list = Vec::with_capacity(len); + for _ in 0..len { + list.push(deserialize_string(bytes)?); + } + Some(list) +} + +fn deserialize_u32_list(bytes: &mut Iter<'_, u8>) -> Option> { + let len = bytes.next_leb128()?; + let mut list = Vec::with_capacity(len); + for _ in 0..len { + list.push(bytes.next_leb128()?); + } + Some(list) +} + +impl Type { + pub fn from_u8(value: u8) -> Self { + match value { + 0 => Type::Individual, + 1 => Type::Group, + 2 => Type::Resource, + 3 => Type::Location, + 4 => Type::Superuser, + 5 => Type::List, + _ => Type::Other, + } + } +} diff --git a/crates/directory/src/ldap/config.rs b/crates/directory/src/backend/ldap/config.rs similarity index 93% rename from crates/directory/src/ldap/config.rs rename to crates/directory/src/backend/ldap/config.rs index 821e18c9..4181aa6e 100644 --- a/crates/directory/src/ldap/config.rs +++ b/crates/directory/src/backend/ldap/config.rs @@ -64,16 +64,18 @@ impl LdapDirectory { filter_verify: LdapFilter::from_config(config, (&prefix, "filter.verify"))?, filter_expand: LdapFilter::from_config(config, (&prefix, "filter.expand"))?, filter_domains: LdapFilter::from_config(config, (&prefix, "filter.domains"))?, - obj_user: config - .value_require((&prefix, "object-classes.user"))? - .to_string(), - obj_group: config - .value_require((&prefix, "object-classes.group"))? - .to_string(), attr_name: config .values((&prefix, "attributes.name")) .map(|(_, v)| v.to_string()) .collect(), + attr_groups: config + .values((&prefix, "attributes.groups")) + .map(|(_, v)| v.to_string()) + .collect(), + attr_type: config + .values((&prefix, "attributes.type")) + .map(|(_, v)| v.to_string()) + .collect(), attr_description: config .values((&prefix, "attributes.description")) .map(|(_, v)| v.to_string()) @@ -82,10 +84,6 @@ impl LdapDirectory { .values((&prefix, "attributes.secret")) .map(|(_, v)| v.to_string()) .collect(), - attr_groups: config - .values((&prefix, "attributes.groups")) - .map(|(_, v)| v.to_string()) - .collect(), attr_email_address: config .values((&prefix, "attributes.email")) .map(|(_, v)| v.to_string()) @@ -94,27 +92,26 @@ impl LdapDirectory { .values((&prefix, "attributes.quota")) .map(|(_, v)| v.to_string()) .collect(), - attrs_principal: vec!["objectClass".to_string()], - attrs_email: config + attr_email_alias: config .values((&prefix, "attributes.email-alias")) .map(|(_, v)| v.to_string()) .collect(), + attrs_principal: vec!["objectClass".to_string()], }; for attr in [ &mappings.attr_name, + &mappings.attr_type, &mappings.attr_description, &mappings.attr_secret, &mappings.attr_quota, &mappings.attr_groups, + &mappings.attr_email_address, + &mappings.attr_email_alias, ] { mappings.attrs_principal.extend(attr.iter().cloned()); } - mappings - .attrs_email - .extend(mappings.attr_email_address.iter().cloned()); - let auth_bind = if config.property_or_static::((&prefix, "auth-bind.enable"), "false")? { LdapFilter::from_config(config, (&prefix, "auth-bind.dn"))?.into() diff --git a/crates/directory/src/backend/ldap/lookup.rs b/crates/directory/src/backend/ldap/lookup.rs new file mode 100644 index 00000000..d392931f --- /dev/null +++ b/crates/directory/src/backend/ldap/lookup.rs @@ -0,0 +1,431 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use ldap3::{Ldap, LdapConnAsync, LdapError, Scope, SearchEntry}; +use mail_send::Credentials; +use store::Store; + +use crate::{ + backend::internal::manage::ManageDirectory, Directory, DirectoryError, Principal, QueryBy, + QueryType, Type, +}; + +use super::{LdapDirectory, LdapMappings}; + +#[async_trait::async_trait] +impl Directory for LdapDirectory { + async fn query(&self, by: QueryBy<'_>) -> crate::Result> { + let mut conn = self.pool.get().await?; + let mut account_id = None; + let account_name; + + let principal = match by.t { + QueryType::Name(username) => { + account_name = username.to_string(); + + if let Some(principal) = self + .find_principal(&mut conn, &self.mappings.filter_name.build(username)) + .await? + { + principal + } else { + return Ok(None); + } + } + QueryType::Id(uid) => { + if let Some(username) = by.account_name(uid).await? { + account_name = username; + } else { + return Ok(None); + } + account_id = Some(uid); + + if let Some(principal) = self + .find_principal(&mut conn, &self.mappings.filter_name.build(&account_name)) + .await? + { + principal + } else { + return Ok(None); + } + } + QueryType::Credentials(credentials) => { + let (username, secret) = match credentials { + Credentials::Plain { username, secret } => (username, secret), + Credentials::OAuthBearer { token } => (token, token), + Credentials::XOauth2 { username, secret } => (username, secret), + }; + account_name = username.to_string(); + + if let Some(auth_bind) = &self.auth_bind { + let (conn, mut ldap) = LdapConnAsync::with_settings( + self.pool.manager().settings.clone(), + &self.pool.manager().address, + ) + .await?; + + ldap3::drive!(conn); + + ldap.simple_bind(&auth_bind.build(username), secret).await?; + + match self + .find_principal(&mut ldap, &self.mappings.filter_name.build(username)) + .await + { + Ok(Some(principal)) => principal, + Err(DirectoryError::Ldap(LdapError::LdapResult { result })) + if [49, 50].contains(&result.rc) => + { + return Ok(None); + } + Ok(None) => return Ok(None), + Err(err) => return Err(err), + } + } else if let Some(principal) = self + .find_principal(&mut conn, &self.mappings.filter_name.build(username)) + .await? + { + if principal.principal.verify_secret(secret).await { + principal + } else { + tracing::debug!( + context = "directory", + event = "invalid_password", + protocol = "ldap", + account = username, + "Invalid password for account" + ); + return Ok(None); + } + } else { + return Ok(None); + } + } + }; + let groups = principal.groups; + let mut principal = principal.principal; + + // Obtain account ID if not available + if let Some(account_id) = account_id { + principal.id = account_id; + } else if by.has_store() { + principal.id = by.account_id(&account_name).await?; + } + principal.name = account_name; + + // Obtain groups + if by.has_store() && !groups.is_empty() { + principal.member_of = Vec::with_capacity(groups.len()); + for group in groups { + if group.contains('=') { + let (rs, _res) = conn + .search( + &group, + Scope::Base, + "objectClass=*", + &self.mappings.attr_name, + ) + .await? + .success()?; + for entry in rs { + 'outer: for (attr, value) in SearchEntry::construct(entry).attrs { + if self.mappings.attr_name.contains(&attr) { + if let Some(group) = value.first() { + if !group.is_empty() { + principal.member_of.push(by.account_id(group).await?); + break 'outer; + } + } + } + } + } + } else { + principal.member_of.push(by.account_id(&group).await?); + } + } + } + + Ok(Some(principal)) + } + + async fn email_to_ids(&self, address: &str, store: &Store) -> crate::Result> { + let mut rs = self + .pool + .get() + .await? + .search( + &self.mappings.base_dn, + Scope::Subtree, + &self + .mappings + .filter_email + .build(self.opt.subaddressing.to_subaddress(address).as_ref()), + &self.mappings.attr_name, + ) + .await? + .success() + .map(|(rs, _res)| rs)?; + + if rs.is_empty() { + if let Some(address) = self.opt.catch_all.to_catch_all(address) { + rs = self + .pool + .get() + .await? + .search( + &self.mappings.base_dn, + Scope::Subtree, + &self.mappings.filter_email.build(address.as_ref()), + &self.mappings.attr_name, + ) + .await? + .success() + .map(|(rs, _res)| rs)?; + } else { + return Ok(Vec::new()); + } + } + + let mut ids = Vec::with_capacity(rs.len()); + for entry in rs { + let entry = SearchEntry::construct(entry); + 'outer: for attr in &self.mappings.attr_name { + if let Some(name) = entry.attrs.get(attr).and_then(|v| v.first()) { + if !name.is_empty() { + ids.push(store.get_or_create_account_id(name).await?); + break 'outer; + } + } + } + } + + Ok(ids) + } + + async fn rcpt(&self, address: &str) -> crate::Result { + match self + .pool + .get() + .await? + .streaming_search( + &self.mappings.base_dn, + Scope::Subtree, + &self + .mappings + .filter_email + .build(self.opt.subaddressing.to_subaddress(address).as_ref()), + &self.mappings.attr_email_address, + ) + .await? + .next() + .await + { + Ok(Some(_)) => Ok(true), + Ok(None) => { + if let Some(address) = self.opt.catch_all.to_catch_all(address) { + self.pool + .get() + .await? + .streaming_search( + &self.mappings.base_dn, + Scope::Subtree, + &self.mappings.filter_email.build(address.as_ref()), + &self.mappings.attr_email_address, + ) + .await? + .next() + .await + .map(|entry| entry.is_some()) + .map_err(|e| e.into()) + } else { + Ok(false) + } + } + + Err(e) => Err(e.into()), + } + } + + async fn vrfy(&self, address: &str) -> crate::Result> { + let mut stream = self + .pool + .get() + .await? + .streaming_search( + &self.mappings.base_dn, + Scope::Subtree, + &self + .mappings + .filter_verify + .build(self.opt.subaddressing.to_subaddress(address).as_ref()), + &self.mappings.attr_email_address, + ) + .await?; + + let mut emails = Vec::new(); + while let Some(entry) = stream.next().await? { + let entry = SearchEntry::construct(entry); + for attr in &self.mappings.attr_email_address { + if let Some(values) = entry.attrs.get(attr) { + for email in values { + if !email.is_empty() { + emails.push(email.to_string()); + } + } + } + } + } + + Ok(emails) + } + + async fn expn(&self, address: &str) -> crate::Result> { + let mut stream = self + .pool + .get() + .await? + .streaming_search( + &self.mappings.base_dn, + Scope::Subtree, + &self + .mappings + .filter_expand + .build(self.opt.subaddressing.to_subaddress(address).as_ref()), + &self.mappings.attr_email_address, + ) + .await?; + + let mut emails = Vec::new(); + while let Some(entry) = stream.next().await? { + let entry = SearchEntry::construct(entry); + for attr in &self.mappings.attr_email_address { + if let Some(values) = entry.attrs.get(attr) { + for email in values { + if !email.is_empty() { + emails.push(email.to_string()); + } + } + } + } + } + + Ok(emails) + } + + async fn is_local_domain(&self, domain: &str) -> crate::Result { + self.pool + .get() + .await? + .streaming_search( + &self.mappings.base_dn, + Scope::Subtree, + &self.mappings.filter_domains.build(domain), + Vec::::new(), + ) + .await? + .next() + .await + .map(|entry| entry.is_some()) + .map_err(|e| e.into()) + } +} + +impl LdapDirectory { + async fn find_principal( + &self, + conn: &mut Ldap, + filter: &str, + ) -> crate::Result> { + conn.search( + &self.mappings.base_dn, + Scope::Subtree, + filter, + &self.mappings.attrs_principal, + ) + .await? + .success() + .map(|(rs, _)| { + rs.into_iter().next().map(|entry| { + self.mappings + .entry_to_principal(SearchEntry::construct(entry)) + }) + }) + .map_err(Into::into) + } +} + +struct PrincipalWithGroups { + principal: Principal, + groups: Vec, +} + +impl LdapMappings { + fn entry_to_principal(&self, entry: SearchEntry) -> PrincipalWithGroups { + let mut groups = Vec::new(); + let mut principal = Principal::default(); + + for (attr, value) in entry.attrs { + if self.attr_name.contains(&attr) { + principal.name = value.into_iter().next().unwrap_or_default(); + } else if self.attr_secret.contains(&attr) { + principal.secrets.extend(value); + } else if self.attr_email_address.contains(&attr) { + for value in value { + if principal.emails.is_empty() { + principal.emails.push(value); + } else { + principal.emails.insert(0, value); + } + } + } else if self.attr_email_alias.contains(&attr) { + principal.emails.extend(value); + } else if let Some(idx) = self.attr_description.iter().position(|a| a == &attr) { + if principal.description.is_none() || idx == 0 { + principal.description = value.into_iter().next(); + } + } else if self.attr_groups.contains(&attr) { + groups.extend(value); + } else if self.attr_quota.contains(&attr) { + if let Ok(quota) = value.into_iter().next().unwrap_or_default().parse() { + principal.quota = quota; + } + } else if self.attr_type.contains(&attr) { + for value in value { + match value.to_ascii_lowercase().as_str() { + "admin" | "administrator" | "root" | "superuser" => { + principal.typ = Type::Superuser + } + "posixaccount" | "individual" | "person" | "inetorgperson" => { + principal.typ = Type::Individual + } + "posixgroup" | "group" => principal.typ = Type::Group, + _ => continue, + } + break; + } + } + } + + PrincipalWithGroups { principal, groups } + } +} diff --git a/crates/directory/src/ldap/mod.rs b/crates/directory/src/backend/ldap/mod.rs similarity index 97% rename from crates/directory/src/ldap/mod.rs rename to crates/directory/src/backend/ldap/mod.rs index 8f0fe882..e8c57cbb 100644 --- a/crates/directory/src/ldap/mod.rs +++ b/crates/directory/src/backend/ldap/mod.rs @@ -45,16 +45,15 @@ pub struct LdapMappings { filter_verify: LdapFilter, filter_expand: LdapFilter, filter_domains: LdapFilter, - obj_user: String, - obj_group: String, attr_name: Vec, + attr_type: Vec, + attr_groups: Vec, attr_description: Vec, attr_secret: Vec, - attr_groups: Vec, attr_email_address: Vec, + attr_email_alias: Vec, attr_quota: Vec, attrs_principal: Vec, - attrs_email: Vec, } #[derive(Debug, Default)] diff --git a/crates/directory/src/ldap/pool.rs b/crates/directory/src/backend/ldap/pool.rs similarity index 100% rename from crates/directory/src/ldap/pool.rs rename to crates/directory/src/backend/ldap/pool.rs diff --git a/crates/directory/src/backend/memory/config.rs b/crates/directory/src/backend/memory/config.rs new file mode 100644 index 00000000..cbcd692b --- /dev/null +++ b/crates/directory/src/backend/memory/config.rs @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::sync::Arc; + +use utils::config::{utils::AsKey, Config}; + +use crate::{Directory, DirectoryOptions, Principal, Type}; + +use super::{EmailType, MemoryDirectory}; + +impl MemoryDirectory { + pub fn from_config( + config: &Config, + prefix: impl AsKey, + ) -> utils::config::Result> { + let prefix = prefix.as_key(); + let mut directory = MemoryDirectory { + opt: DirectoryOptions::from_config(config, prefix.clone())?, + ..Default::default() + }; + + for lookup_id in config.sub_keys((prefix.as_str(), "principals")) { + let name = config + .value_require((prefix.as_str(), "principals", lookup_id, "name"))? + .to_string(); + let typ = + match config.value_require((prefix.as_str(), "principals", lookup_id, "name"))? { + "individual" => Type::Individual, + "admin" => Type::Superuser, + "group" => Type::Group, + _ => Type::Other, + }; + + // Obtain id + let next_user_id = directory.names_to_ids.len() as u32; + let id = *directory + .names_to_ids + .entry(name.to_string()) + .or_insert(next_user_id); + + // Obtain group ids + let mut member_of = Vec::new(); + for (_, group) in config.values((prefix.as_str(), "principals", lookup_id, "member-of")) + { + let next_group_id = directory.names_to_ids.len() as u32; + member_of.push( + *directory + .names_to_ids + .entry(group.to_string()) + .or_insert(next_group_id), + ); + } + + // Parse email addresses + let mut emails = Vec::new(); + for (pos, (_, email)) in config + .values((prefix.as_str(), "principals", lookup_id, "email")) + .enumerate() + { + directory + .emails_to_ids + .entry(email.to_string()) + .or_default() + .push(if pos > 0 { + EmailType::Alias(id) + } else { + EmailType::Primary(id) + }); + + if let Some((_, domain)) = email.rsplit_once('@') { + directory.domains.insert(domain.to_lowercase()); + } + + emails.push(email.to_lowercase()); + } + + // Parse mailing lists + for (_, email) in + config.values((prefix.as_str(), "principals", lookup_id, "email-list")) + { + directory + .emails_to_ids + .entry(email.to_lowercase()) + .or_default() + .push(EmailType::List(id)); + if let Some((_, domain)) = email.rsplit_once('@') { + directory.domains.insert(domain.to_lowercase()); + } + } + + directory.principals.push(Principal { + name: name.clone(), + secrets: config + .values((prefix.as_str(), "principals", lookup_id, "secret")) + .map(|(_, v)| v.to_string()) + .collect(), + typ, + description: config + .value((prefix.as_str(), "principals", lookup_id, "description")) + .map(|v| v.to_string()), + quota: config + .property((prefix.as_str(), "principals", lookup_id, "quota"))? + .unwrap_or(0), + member_of, + id, + emails, + }); + } + + Ok(Arc::new(directory)) + } +} diff --git a/crates/directory/src/memory/lookup.rs b/crates/directory/src/backend/memory/lookup.rs similarity index 55% rename from crates/directory/src/memory/lookup.rs rename to crates/directory/src/backend/memory/lookup.rs index 54c230f1..65bb4fec 100644 --- a/crates/directory/src/memory/lookup.rs +++ b/crates/directory/src/backend/memory/lookup.rs @@ -22,65 +22,68 @@ */ use mail_send::Credentials; +use store::Store; -use crate::{Directory, Principal}; +use crate::{Directory, Principal, QueryBy, QueryType}; use super::{EmailType, MemoryDirectory}; #[async_trait::async_trait] impl Directory for MemoryDirectory { - async fn authenticate( - &self, - credentials: &Credentials, - ) -> crate::Result> { - let (username, secret) = match credentials { - Credentials::Plain { username, secret } => (username, secret), - Credentials::OAuthBearer { token } => (token, token), - Credentials::XOauth2 { username, secret } => (username, secret), - }; - match self.principals.get(username) { - Some(principal) if principal.verify_secret(secret).await => Ok(Some(principal.clone())), - _ => Ok(None), - } - } - - async fn principal(&self, name: &str) -> crate::Result> { - Ok(self.principals.get(name).cloned()) - } - - async fn emails_by_name(&self, name: &str) -> crate::Result> { - let mut result = Vec::new(); - if let Some(emails) = self.names_to_email.get(name) { - for email in emails { - match email { - EmailType::Primary(email) | EmailType::Alias(email) => { - result.push(email.clone()) + async fn query(&self, by: QueryBy<'_>) -> crate::Result> { + match by.t { + QueryType::Name(name) => { + for principal in &self.principals { + if principal.name == name { + return Ok(Some(principal.clone())); + } + } + } + QueryType::Id(uid) => { + for principal in &self.principals { + if principal.id == uid { + return Ok(Some(principal.clone())); + } + } + } + QueryType::Credentials(credentials) => { + let (username, secret) = match credentials { + Credentials::Plain { username, secret } => (username, secret), + Credentials::OAuthBearer { token } => (token, token), + Credentials::XOauth2 { username, secret } => (username, secret), + }; + + for principal in &self.principals { + if &principal.name == username { + return if principal.verify_secret(secret).await { + Ok(Some(principal.clone())) + } else { + Ok(None) + }; } - _ => {} } } } - - Ok(result) + Ok(None) } - async fn names_by_email(&self, address: &str) -> crate::Result> { + async fn email_to_ids(&self, address: &str, _: &Store) -> crate::Result> { Ok(self - .emails_to_names + .emails_to_ids .get(self.opt.subaddressing.to_subaddress(address).as_ref()) .or_else(|| { self.opt .catch_all .to_catch_all(address) - .and_then(|address| self.emails_to_names.get(address.as_ref())) + .and_then(|address| self.emails_to_ids.get(address.as_ref())) }) .map(|names| { names .iter() .map(|t| match t { - EmailType::Primary(name) - | EmailType::Alias(name) - | EmailType::List(name) => name.to_string(), + EmailType::Primary(uid) | EmailType::Alias(uid) | EmailType::List(uid) => { + *uid + } }) .collect::>() }) @@ -89,21 +92,21 @@ impl Directory for MemoryDirectory { async fn rcpt(&self, address: &str) -> crate::Result { Ok(self - .emails_to_names + .emails_to_ids .contains_key(self.opt.subaddressing.to_subaddress(address).as_ref()) || self .opt .catch_all .to_catch_all(address) .map_or(false, |address| { - self.emails_to_names.contains_key(address.as_ref()) + self.emails_to_ids.contains_key(address.as_ref()) })) } async fn vrfy(&self, address: &str) -> crate::Result> { let mut result = Vec::new(); let address = self.opt.subaddressing.to_subaddress(address); - for (key, value) in &self.emails_to_names { + for (key, value) in &self.emails_to_ids { if key.contains(address.as_ref()) && value.iter().any(|t| matches!(t, EmailType::Primary(_))) { @@ -116,13 +119,16 @@ impl Directory for MemoryDirectory { async fn expn(&self, address: &str) -> crate::Result> { let mut result = Vec::new(); let address = self.opt.subaddressing.to_subaddress(address); - for (key, value) in &self.emails_to_names { + for (key, value) in &self.emails_to_ids { if key == address.as_ref() { for item in value { - if let EmailType::List(name) = item { - for addr in self.names_to_email.get(name).unwrap() { - if let EmailType::Primary(addr) = addr { - result.push(addr.clone()) + if let EmailType::List(uid) = item { + for principal in &self.principals { + if principal.id == *uid { + if let Some(addr) = principal.emails.first() { + result.push(addr.clone()) + } + break; } } } diff --git a/crates/directory/src/memory/mod.rs b/crates/directory/src/backend/memory/mod.rs similarity index 85% rename from crates/directory/src/memory/mod.rs rename to crates/directory/src/backend/memory/mod.rs index 916333a8..dded7716 100644 --- a/crates/directory/src/memory/mod.rs +++ b/crates/directory/src/backend/memory/mod.rs @@ -30,16 +30,16 @@ pub mod lookup; #[derive(Default, Debug)] pub struct MemoryDirectory { - principals: AHashMap, - emails_to_names: AHashMap>, - names_to_email: AHashMap>, + principals: Vec, + emails_to_ids: AHashMap>, + names_to_ids: AHashMap, domains: AHashSet, opt: DirectoryOptions, } #[derive(Debug)] enum EmailType { - Primary(String), - Alias(String), - List(String), + Primary(u32), + Alias(u32), + List(u32), } diff --git a/crates/directory/src/backend/mod.rs b/crates/directory/src/backend/mod.rs new file mode 100644 index 00000000..f0388b1b --- /dev/null +++ b/crates/directory/src/backend/mod.rs @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +pub mod imap; +pub mod internal; +pub mod ldap; +pub mod memory; +pub mod smtp; +pub mod sql; diff --git a/crates/directory/src/smtp/config.rs b/crates/directory/src/backend/smtp/config.rs similarity index 95% rename from crates/directory/src/smtp/config.rs rename to crates/directory/src/backend/smtp/config.rs index c0fd3835..6accb4b1 100644 --- a/crates/directory/src/smtp/config.rs +++ b/crates/directory/src/backend/smtp/config.rs @@ -26,9 +26,9 @@ use std::sync::Arc; use mail_send::{smtp::tls::build_tls_connector, SmtpClientBuilder}; use utils::config::{utils::AsKey, Config}; -use crate::{cache::CachedDirectory, config::build_pool, smtp::SmtpConnectionManager, Directory}; +use crate::{cache::CachedDirectory, config::build_pool, Directory}; -use super::SmtpDirectory; +use super::{SmtpConnectionManager, SmtpDirectory}; impl SmtpDirectory { pub fn from_config( diff --git a/crates/directory/src/smtp/lookup.rs b/crates/directory/src/backend/smtp/lookup.rs similarity index 84% rename from crates/directory/src/smtp/lookup.rs rename to crates/directory/src/backend/smtp/lookup.rs index 8ff24380..80330abd 100644 --- a/crates/directory/src/smtp/lookup.rs +++ b/crates/directory/src/backend/smtp/lookup.rs @@ -23,30 +23,24 @@ use mail_send::{smtp::AssertReply, Credentials}; use smtp_proto::Severity; +use store::Store; -use crate::{Directory, DirectoryError, Principal}; +use crate::{Directory, DirectoryError, Principal, QueryBy, QueryType}; use super::{SmtpClient, SmtpDirectory}; #[async_trait::async_trait] impl Directory for SmtpDirectory { - async fn authenticate( - &self, - credentials: &Credentials, - ) -> crate::Result> { - self.pool.get().await?.authenticate(credentials).await + async fn query(&self, query: QueryBy<'_>) -> crate::Result> { + if let QueryType::Credentials(credentials) = query.t { + self.pool.get().await?.authenticate(credentials).await + } else { + Err(DirectoryError::unsupported("smtp", "query")) + } } - async fn principal(&self, _name: &str) -> crate::Result> { - Err(DirectoryError::unsupported("smtp", "principal")) - } - - async fn emails_by_name(&self, _: &str) -> crate::Result> { - Err(DirectoryError::unsupported("smtp", "emails_by_name")) - } - - async fn names_by_email(&self, _address: &str) -> crate::Result> { - Err(DirectoryError::unsupported("smtp", "names_by_email")) + async fn email_to_ids(&self, _address: &str, _store: &Store) -> crate::Result> { + Err(DirectoryError::unsupported("smtp", "email_to_ids")) } async fn rcpt(&self, address: &str) -> crate::Result { diff --git a/crates/directory/src/smtp/mod.rs b/crates/directory/src/backend/smtp/mod.rs similarity index 100% rename from crates/directory/src/smtp/mod.rs rename to crates/directory/src/backend/smtp/mod.rs diff --git a/crates/directory/src/smtp/pool.rs b/crates/directory/src/backend/smtp/pool.rs similarity index 100% rename from crates/directory/src/smtp/pool.rs rename to crates/directory/src/backend/smtp/pool.rs diff --git a/crates/directory/src/sql/config.rs b/crates/directory/src/backend/sql/config.rs similarity index 95% rename from crates/directory/src/sql/config.rs rename to crates/directory/src/backend/sql/config.rs index fb758692..2215eb0a 100644 --- a/crates/directory/src/sql/config.rs +++ b/crates/directory/src/backend/sql/config.rs @@ -47,10 +47,6 @@ impl SqlDirectory { .clone(); let mut mappings = SqlMappings { - column_name: config - .value((&prefix, "columns.name")) - .unwrap_or_default() - .to_string(), column_description: config .value((&prefix, "columns.description")) .unwrap_or_default() diff --git a/crates/directory/src/sql/lookup.rs b/crates/directory/src/backend/sql/lookup.rs similarity index 51% rename from crates/directory/src/sql/lookup.rs rename to crates/directory/src/backend/sql/lookup.rs index 0a23bb51..51706cad 100644 --- a/crates/directory/src/sql/lookup.rs +++ b/crates/directory/src/backend/sql/lookup.rs @@ -22,73 +22,124 @@ */ use mail_send::Credentials; -use store::{NamedRows, Rows, Value}; +use store::{NamedRows, Rows, Store, Value}; -use crate::{Directory, Principal, Type}; +use crate::{ + backend::internal::manage::ManageDirectory, Directory, Principal, QueryBy, QueryType, Type, +}; use super::{SqlDirectory, SqlMappings}; #[async_trait::async_trait] impl Directory for SqlDirectory { - async fn authenticate( - &self, - credentials: &Credentials, - ) -> crate::Result> { - let (username, secret) = match credentials { - Credentials::Plain { username, secret } => (username, secret), - Credentials::OAuthBearer { token } => (token, token), - Credentials::XOauth2 { username, secret } => (username, secret), + async fn query(&self, by: QueryBy<'_>) -> crate::Result> { + let mut account_id = None; + let account_name; + let mut secret = None; + + let result = match by.t { + QueryType::Name(username) => { + account_name = username.to_string(); + + self.store + .query::(&self.mappings.query_name, vec![username.into()]) + .await? + } + QueryType::Id(uid) => { + if let Some(username) = by.account_name(uid).await? { + account_name = username; + } else { + return Ok(None); + } + account_id = Some(uid); + + self.store + .query::( + &self.mappings.query_name, + vec![account_name.clone().into()], + ) + .await? + } + QueryType::Credentials(credentials) => { + let (username, secret_) = match credentials { + Credentials::Plain { username, secret } => (username, secret), + Credentials::OAuthBearer { token } => (token, token), + Credentials::XOauth2 { username, secret } => (username, secret), + }; + account_name = username.to_string(); + secret = secret_.into(); + + self.store + .query::(&self.mappings.query_name, vec![username.into()]) + .await? + } }; - match self.principal(username).await { - Ok(Some(principal)) if principal.verify_secret(secret).await => Ok(Some(principal)), - Ok(_) => Ok(None), - Err(err) => Err(err), + if result.rows.is_empty() { + return Ok(None); } - } - async fn principal(&self, name: &str) -> crate::Result> { - let result = self - .store - .query::(&self.mappings.query_name, vec![name.into()]) - .await?; - if !result.rows.is_empty() { - // Map row to principal - let mut principal = self.mappings.row_to_principal(result)?; + // Map row to principal + let mut principal = self.mappings.row_to_principal(result)?; + // Validate password + if let Some(secret) = secret { + if !principal.verify_secret(secret).await { + tracing::debug!( + context = "directory", + event = "invalid_password", + protocol = "sql", + account = account_name, + "Invalid password for account" + ); + return Ok(None); + } + } + + // Obtain account ID if not available + if let Some(account_id) = account_id { + principal.id = account_id; + } else if by.has_store() { + principal.id = by.account_id(&account_name).await?; + } + principal.name = account_name; + + if by.has_store() { // Obtain members - principal.member_of = self - .store - .query::(&self.mappings.query_members, vec![name.into()]) - .await? - .into(); - - // Check whether the user is a superuser - if let Some(idx) = principal - .member_of - .iter() - .position(|group| group.eq_ignore_ascii_case(&self.opt.superuser_group)) - { - principal.member_of.swap_remove(idx); - principal.typ = Type::Superuser; + if !self.mappings.query_members.is_empty() { + for row in self + .store + .query::( + &self.mappings.query_members, + vec![principal.name.clone().into()], + ) + .await? + .rows + { + if let Some(Value::Text(account_id)) = row.values.first() { + principal.member_of.push(by.account_id(account_id).await?); + } + } } - Ok(Some(principal)) - } else { - Ok(None) + // Obtain emails + if !self.mappings.query_emails.is_empty() { + principal.emails = self + .store + .query::( + &self.mappings.query_emails, + vec![principal.name.clone().into()], + ) + .await? + .into(); + } } + + Ok(Some(principal)) } - async fn emails_by_name(&self, name: &str) -> crate::Result> { - self.store - .query::(&self.mappings.query_emails, vec![name.into()]) - .await - .map(Into::into) - .map_err(Into::into) - } - - async fn names_by_email(&self, address: &str) -> crate::Result> { - let ids = self + async fn email_to_ids(&self, address: &str, store: &Store) -> crate::Result> { + let mut names = self .store .query::( &self.mappings.query_recipients, @@ -101,17 +152,26 @@ impl Directory for SqlDirectory { ) .await?; - if !ids.rows.is_empty() { - Ok(ids.into()) - } else if let Some(address) = self.opt.catch_all.to_catch_all(address) { - self.store - .query::(&self.mappings.query_recipients, vec![address.into()]) - .await - .map(Into::into) - .map_err(Into::into) - } else { - Ok(vec![]) + if names.rows.is_empty() { + if let Some(address) = self.opt.catch_all.to_catch_all(address) { + names = self + .store + .query::(&self.mappings.query_recipients, vec![address.into()]) + .await?; + } else { + return Ok(vec![]); + } } + + let mut ids = Vec::with_capacity(names.rows.len()); + + for row in names.rows { + if let Some(Value::Text(name)) = row.values.first() { + ids.push(store.get_or_create_account_id(name).await?); + } + } + + Ok(ids) } async fn rcpt(&self, address: &str) -> crate::Result { @@ -185,11 +245,10 @@ impl Directory for SqlDirectory { impl SqlMappings { pub fn row_to_principal(&self, rows: NamedRows) -> crate::Result { let mut principal = Principal::default(); + if let Some(row) = rows.rows.into_iter().next() { for (name, value) in rows.names.into_iter().zip(row.values) { - if name.eq_ignore_ascii_case(&self.column_name) { - principal.name = value.into_string(); - } else if name.eq_ignore_ascii_case(&self.column_secret) { + if name.eq_ignore_ascii_case(&self.column_secret) { if let Value::Text(secret) = value { principal.secrets.push(secret.into_owned()); } @@ -197,6 +256,7 @@ impl SqlMappings { match value.to_str().as_ref() { "individual" | "person" | "user" => principal.typ = Type::Individual, "group" => principal.typ = Type::Group, + "admin" | "superuser" | "administrator" => principal.typ = Type::Superuser, _ => (), } } else if name.eq_ignore_ascii_case(&self.column_description) { diff --git a/crates/directory/src/sql/mod.rs b/crates/directory/src/backend/sql/mod.rs similarity index 98% rename from crates/directory/src/sql/mod.rs rename to crates/directory/src/backend/sql/mod.rs index 956ab768..001b5e6f 100644 --- a/crates/directory/src/sql/mod.rs +++ b/crates/directory/src/backend/sql/mod.rs @@ -43,7 +43,6 @@ pub(crate) struct SqlMappings { query_domains: String, query_verify: String, query_expand: String, - column_name: String, column_description: String, column_secret: String, column_quota: String, diff --git a/crates/directory/src/cache/lookup.rs b/crates/directory/src/cache/lookup.rs index 3a2c6088..cc31330d 100644 --- a/crates/directory/src/cache/lookup.rs +++ b/crates/directory/src/cache/lookup.rs @@ -21,31 +21,20 @@ * for more details. */ -use mail_send::Credentials; +use store::Store; -use crate::{Directory, Principal}; +use crate::{Directory, Principal, QueryBy}; use super::CachedDirectory; #[async_trait::async_trait] impl Directory for CachedDirectory { - async fn authenticate( - &self, - credentials: &Credentials, - ) -> crate::Result> { - self.inner.authenticate(credentials).await + async fn query(&self, by: QueryBy<'_>) -> crate::Result> { + self.inner.query(by).await } - async fn principal(&self, name: &str) -> crate::Result> { - self.inner.principal(name).await - } - - async fn emails_by_name(&self, name: &str) -> crate::Result> { - self.inner.emails_by_name(name).await - } - - async fn names_by_email(&self, address: &str) -> crate::Result> { - self.inner.names_by_email(address).await + async fn email_to_ids(&self, address: &str, store: &Store) -> crate::Result> { + self.inner.email_to_ids(address, store).await } async fn rcpt(&self, address: &str) -> crate::Result { diff --git a/crates/directory/src/config.rs b/crates/directory/src/config.rs index a3551185..e62e6d14 100644 --- a/crates/directory/src/config.rs +++ b/crates/directory/src/config.rs @@ -36,8 +36,11 @@ use utils::config::{ use ahash::AHashMap; use crate::{ - imap::ImapDirectory, ldap::LdapDirectory, memory::MemoryDirectory, smtp::SmtpDirectory, - sql::SqlDirectory, AddressMapping, Directories, DirectoryOptions, + backend::{ + imap::ImapDirectory, ldap::LdapDirectory, memory::MemoryDirectory, smtp::SmtpDirectory, + sql::SqlDirectory, + }, + AddressMapping, Directories, DirectoryOptions, }; pub trait ConfigDirectory { @@ -49,6 +52,7 @@ impl ConfigDirectory for Config { let mut config = Directories { directories: AHashMap::new(), }; + for id in self.sub_keys("directory") { // Parse directory let protocol = self.value_require(("directory", id, "type"))?; @@ -78,10 +82,6 @@ impl DirectoryOptions { Ok(DirectoryOptions { catch_all: AddressMapping::from_config(config, (&key, "options.catch-all"))?, subaddressing: AddressMapping::from_config(config, (&key, "options.subaddressing"))?, - superuser_group: config - .value((&key, "options.superuser-group")) - .unwrap_or("superusers") - .to_string(), }) } } diff --git a/crates/directory/src/imap/lookup.rs b/crates/directory/src/imap/lookup.rs deleted file mode 100644 index 49be21ab..00000000 --- a/crates/directory/src/imap/lookup.rs +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use mail_send::Credentials; -use smtp_proto::{AUTH_CRAM_MD5, AUTH_LOGIN, AUTH_OAUTHBEARER, AUTH_PLAIN, AUTH_XOAUTH2}; - -use crate::{Directory, DirectoryError, Principal}; - -use super::{ImapDirectory, ImapError}; - -#[async_trait::async_trait] -impl Directory for ImapDirectory { - async fn authenticate( - &self, - credentials: &Credentials, - ) -> crate::Result> { - let mut client = self.pool.get().await?; - let mechanism = match credentials { - Credentials::Plain { .. } - if (client.mechanisms & (AUTH_PLAIN | AUTH_LOGIN | AUTH_CRAM_MD5)) != 0 => - { - if client.mechanisms & AUTH_CRAM_MD5 != 0 { - AUTH_CRAM_MD5 - } else if client.mechanisms & AUTH_PLAIN != 0 { - AUTH_PLAIN - } else { - AUTH_LOGIN - } - } - Credentials::OAuthBearer { .. } if client.mechanisms & AUTH_OAUTHBEARER != 0 => { - AUTH_OAUTHBEARER - } - Credentials::XOauth2 { .. } if client.mechanisms & AUTH_XOAUTH2 != 0 => AUTH_XOAUTH2, - _ => { - tracing::warn!( - context = "remote", - event = "error", - protocol = "imap", - "IMAP server does not offer any supported auth mechanisms.", - ); - return Ok(None); - } - }; - - match client.authenticate(mechanism, credentials).await { - Ok(_) => { - client.is_valid = false; - Ok(Some(Principal::default())) - } - Err(err) => match &err { - ImapError::AuthenticationFailed => Ok(None), - _ => Err(err.into()), - }, - } - } - - async fn principal(&self, _name: &str) -> crate::Result> { - Err(DirectoryError::unsupported("imap", "principal")) - } - - async fn emails_by_name(&self, _: &str) -> crate::Result> { - Err(DirectoryError::unsupported("imap", "emails_by_name")) - } - - async fn names_by_email(&self, _address: &str) -> crate::Result> { - Err(DirectoryError::unsupported("imap", "names_by_email")) - } - - async fn rcpt(&self, _address: &str) -> crate::Result { - Err(DirectoryError::unsupported("imap", "rcpt")) - } - - async fn vrfy(&self, _address: &str) -> crate::Result> { - Err(DirectoryError::unsupported("imap", "vrfy")) - } - - async fn expn(&self, _address: &str) -> crate::Result> { - Err(DirectoryError::unsupported("imap", "expn")) - } - - async fn is_local_domain(&self, domain: &str) -> crate::Result { - Ok(self.domains.contains(domain)) - } -} diff --git a/crates/directory/src/ldap/lookup.rs b/crates/directory/src/ldap/lookup.rs deleted file mode 100644 index 0cc636f0..00000000 --- a/crates/directory/src/ldap/lookup.rs +++ /dev/null @@ -1,399 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use ldap3::{Ldap, LdapConnAsync, LdapError, ResultEntry, Scope, SearchEntry}; -use mail_send::Credentials; - -use crate::{Directory, DirectoryError, Principal, Type}; - -use super::{LdapDirectory, LdapMappings}; - -#[async_trait::async_trait] -impl Directory for LdapDirectory { - async fn authenticate( - &self, - credentials: &Credentials, - ) -> crate::Result> { - let (username, secret) = match credentials { - Credentials::Plain { username, secret } => (username, secret), - Credentials::OAuthBearer { token } => (token, token), - Credentials::XOauth2 { username, secret } => (username, secret), - }; - - if let Some(auth_bind) = &self.auth_bind { - let (conn, mut ldap) = LdapConnAsync::with_settings( - self.pool.manager().settings.clone(), - &self.pool.manager().address, - ) - .await?; - - ldap3::drive!(conn); - - ldap.simple_bind(&auth_bind.build(username), secret).await?; - - match self - .find_principal(&mut ldap, &self.mappings.filter_name.build(username)) - .await - { - Err(DirectoryError::Ldap(LdapError::LdapResult { result })) - if [49, 50].contains(&result.rc) => - { - Ok(None) - } - result => result, - } - } else { - let mut conn = self.pool.get().await?; - match self - .find_principal(&mut conn, &self.mappings.filter_name.build(username)) - .await - { - Ok(Some(principal)) => { - if principal.verify_secret(secret).await { - Ok(Some(principal)) - } else { - Ok(None) - } - } - result => result, - } - } - } - - async fn principal(&self, name: &str) -> crate::Result> { - let mut conn = self.pool.get().await?; - self.find_principal(&mut conn, &self.mappings.filter_name.build(name)) - .await - } - - async fn emails_by_name(&self, name: &str) -> crate::Result> { - let (rs, _res) = self - .pool - .get() - .await? - .search( - &self.mappings.base_dn, - Scope::Subtree, - &self.mappings.filter_name.build(name), - &self.mappings.attrs_email, - ) - .await? - .success()?; - - let mut emails = Vec::new(); - for entry in rs { - let entry = SearchEntry::construct(entry); - for attr in &self.mappings.attrs_email { - if let Some(values) = entry.attrs.get(attr) { - for email in values { - if !email.is_empty() { - emails.push(email.to_string()); - } - } - } - } - } - - Ok(emails) - } - - async fn names_by_email(&self, address: &str) -> crate::Result> { - let names = self - .pool - .get() - .await? - .search( - &self.mappings.base_dn, - Scope::Subtree, - &self - .mappings - .filter_email - .build(self.opt.subaddressing.to_subaddress(address).as_ref()), - &self.mappings.attr_name, - ) - .await? - .success() - .map(|(rs, _res)| self.extract_names(rs))?; - - if !names.is_empty() { - Ok(names) - } else if let Some(address) = self.opt.catch_all.to_catch_all(address) { - self.pool - .get() - .await? - .search( - &self.mappings.base_dn, - Scope::Subtree, - &self.mappings.filter_email.build(address.as_ref()), - &self.mappings.attr_name, - ) - .await? - .success() - .map(|(rs, _res)| self.extract_names(rs)) - .map_err(|e| e.into()) - } else { - Ok(names) - } - } - - async fn rcpt(&self, address: &str) -> crate::Result { - match self - .pool - .get() - .await? - .streaming_search( - &self.mappings.base_dn, - Scope::Subtree, - &self - .mappings - .filter_email - .build(self.opt.subaddressing.to_subaddress(address).as_ref()), - &self.mappings.attr_email_address, - ) - .await? - .next() - .await - { - Ok(Some(_)) => Ok(true), - Ok(None) => { - if let Some(address) = self.opt.catch_all.to_catch_all(address) { - self.pool - .get() - .await? - .streaming_search( - &self.mappings.base_dn, - Scope::Subtree, - &self.mappings.filter_email.build(address.as_ref()), - &self.mappings.attr_email_address, - ) - .await? - .next() - .await - .map(|entry| entry.is_some()) - .map_err(|e| e.into()) - } else { - Ok(false) - } - } - - Err(e) => Err(e.into()), - } - } - - async fn vrfy(&self, address: &str) -> crate::Result> { - let mut stream = self - .pool - .get() - .await? - .streaming_search( - &self.mappings.base_dn, - Scope::Subtree, - &self - .mappings - .filter_verify - .build(self.opt.subaddressing.to_subaddress(address).as_ref()), - &self.mappings.attr_email_address, - ) - .await?; - - let mut emails = Vec::new(); - while let Some(entry) = stream.next().await? { - let entry = SearchEntry::construct(entry); - for attr in &self.mappings.attr_email_address { - if let Some(values) = entry.attrs.get(attr) { - for email in values { - if !email.is_empty() { - emails.push(email.to_string()); - } - } - } - } - } - - Ok(emails) - } - - async fn expn(&self, address: &str) -> crate::Result> { - let mut stream = self - .pool - .get() - .await? - .streaming_search( - &self.mappings.base_dn, - Scope::Subtree, - &self - .mappings - .filter_expand - .build(self.opt.subaddressing.to_subaddress(address).as_ref()), - &self.mappings.attr_email_address, - ) - .await?; - - let mut emails = Vec::new(); - while let Some(entry) = stream.next().await? { - let entry = SearchEntry::construct(entry); - for attr in &self.mappings.attr_email_address { - if let Some(values) = entry.attrs.get(attr) { - for email in values { - if !email.is_empty() { - emails.push(email.to_string()); - } - } - } - } - } - - Ok(emails) - } - - async fn is_local_domain(&self, domain: &str) -> crate::Result { - self.pool - .get() - .await? - .streaming_search( - &self.mappings.base_dn, - Scope::Subtree, - &self.mappings.filter_domains.build(domain), - Vec::::new(), - ) - .await? - .next() - .await - .map(|entry| entry.is_some()) - .map_err(|e| e.into()) - } -} - -impl LdapDirectory { - async fn find_principal( - &self, - conn: &mut Ldap, - filter: &str, - ) -> crate::Result> { - let (rs, _res) = conn - .search( - &self.mappings.base_dn, - Scope::Subtree, - filter, - &self.mappings.attrs_principal, - ) - .await? - .success()?; - - if let Some(mut principal) = rs.into_iter().next().map(|entry| { - self.mappings - .entry_to_principal(SearchEntry::construct(entry)) - }) { - // Map groups - if !principal.member_of.is_empty() { - let mut names = Vec::with_capacity(principal.member_of.len()); - for group in principal.member_of { - if group.contains('=') { - let (rs, _res) = conn - .search( - &group, - Scope::Base, - "objectClass=*", - &self.mappings.attr_name, - ) - .await? - .success()?; - for entry in rs { - 'outer: for (attr, value) in SearchEntry::construct(entry).attrs { - if self.mappings.attr_name.contains(&attr) { - if let Some(group) = value.first() { - if !group.is_empty() { - if !group - .eq_ignore_ascii_case(&self.opt.superuser_group) - { - names.push(group.to_string()); - } else { - principal.typ = Type::Superuser; - } - break 'outer; - } - } - } - } - } - } else if !group.eq_ignore_ascii_case(&self.opt.superuser_group) { - names.push(group); - } else { - principal.typ = Type::Superuser; - } - } - principal.member_of = names; - } - Ok(Some(principal)) - } else { - Ok(None) - } - } - - fn extract_names(&self, rs: Vec) -> Vec { - let mut names = Vec::with_capacity(rs.len()); - for entry in rs { - let entry = SearchEntry::construct(entry); - 'outer: for attr in &self.mappings.attr_name { - if let Some(value) = entry.attrs.get(attr).and_then(|v| v.first()) { - if !value.is_empty() { - names.push(value.to_string()); - break 'outer; - } - } - } - } - names - } -} - -impl LdapMappings { - pub fn entry_to_principal(&self, entry: SearchEntry) -> Principal { - let mut principal = Principal::default(); - for (attr, value) in entry.attrs { - if self.attr_name.contains(&attr) { - principal.name = value.into_iter().next().unwrap_or_default(); - } else if self.attr_secret.contains(&attr) { - principal.secrets.extend(value); - } else if let Some(idx) = self.attr_description.iter().position(|a| a == &attr) { - if principal.description.is_none() || idx == 0 { - principal.description = value.into_iter().next(); - } - } else if self.attr_groups.contains(&attr) { - principal.member_of.extend(value); - } else if self.attr_quota.contains(&attr) { - if let Ok(quota) = value.into_iter().next().unwrap_or_default().parse() { - principal.quota = quota; - } - } else if attr.eq_ignore_ascii_case("objectClass") { - if value.contains(&self.obj_user) { - principal.typ = Type::Individual; - } else if value.contains(&self.obj_group) { - principal.typ = Type::Group; - } - } - } - - principal - } -} diff --git a/crates/directory/src/lib.rs b/crates/directory/src/lib.rs index cfbe97f1..f5a59032 100644 --- a/crates/directory/src/lib.rs +++ b/crates/directory/src/lib.rs @@ -24,46 +24,46 @@ use std::{borrow::Cow, fmt::Debug, sync::Arc}; use ahash::AHashMap; +use backend::imap::ImapError; use deadpool::managed::PoolError; -use imap::ImapError; use ldap3::LdapError; use mail_send::Credentials; +use store::Store; use utils::config::DynValue; +pub mod backend; pub mod cache; pub mod config; -pub mod imap; -pub mod ldap; -pub mod memory; pub mod secret; -pub mod smtp; -pub mod sql; #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct Principal { + pub id: u32, + pub typ: Type, + pub quota: u32, pub name: String, pub secrets: Vec, - pub typ: Type, + pub emails: Vec, + pub member_of: Vec, pub description: Option, - pub quota: u32, - pub member_of: Vec, } #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] pub enum Type { - Individual, - Group, - Resource, - Location, + Individual = 0, + Group = 1, + Resource = 2, + Location = 3, + Superuser = 4, + List = 5, #[default] - Other, - Superuser, + Other = 6, } #[derive(Debug)] pub enum DirectoryError { Ldap(LdapError), - Sql(store::Error), + Store(store::Error), Imap(ImapError), Smtp(mail_send::Error), Pool(String), @@ -73,16 +73,26 @@ pub enum DirectoryError { #[async_trait::async_trait] pub trait Directory: Sync + Send { - async fn authenticate(&self, credentials: &Credentials) -> Result>; - async fn principal(&self, name: &str) -> Result>; - async fn emails_by_name(&self, name: &str) -> Result>; - async fn names_by_email(&self, email: &str) -> Result>; + async fn query(&self, by: QueryBy<'_>) -> Result>; + async fn email_to_ids(&self, email: &str, store: &Store) -> Result>; + async fn is_local_domain(&self, domain: &str) -> crate::Result; async fn rcpt(&self, address: &str) -> crate::Result; async fn vrfy(&self, address: &str) -> Result>; async fn expn(&self, address: &str) -> Result>; } +pub enum QueryType<'x> { + Name(&'x str), + Id(u32), + Credentials(&'x Credentials), +} + +pub struct QueryBy<'x> { + pub t: QueryType<'x>, + pub store: Option<&'x Store>, +} + impl Principal { pub fn name(&self) -> &str { &self.name @@ -111,6 +121,7 @@ impl Type { Self::Resource => "resource", Self::Location => "location", Self::Other => "other", + Self::List => "list", } } } @@ -119,7 +130,6 @@ impl Type { struct DirectoryOptions { catch_all: AddressMapping, subaddressing: AddressMapping, - superuser_group: String, } #[derive(Debug, Default)] @@ -194,7 +204,7 @@ impl From for DirectoryError { "SQL directory error" ); - DirectoryError::Sql(error) + DirectoryError::Store(error) } } diff --git a/crates/directory/src/memory/config.rs b/crates/directory/src/memory/config.rs deleted file mode 100644 index 9f1d7617..00000000 --- a/crates/directory/src/memory/config.rs +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::sync::Arc; - -use utils::config::{utils::AsKey, Config}; - -use crate::{Directory, DirectoryOptions, Principal, Type}; - -use super::{EmailType, MemoryDirectory}; - -impl MemoryDirectory { - pub fn from_config( - config: &Config, - prefix: impl AsKey, - ) -> utils::config::Result> { - let prefix = prefix.as_key(); - let mut directory = MemoryDirectory { - opt: DirectoryOptions::from_config(config, prefix.clone())?, - ..Default::default() - }; - - for lookup_id in config.sub_keys((prefix.as_str(), "users")) { - let name = config - .value_require((prefix.as_str(), "users", lookup_id, "name"))? - .to_string(); - let mut typ = Type::Individual; - let mut member_of = Vec::new(); - - for (_, group) in config.values((prefix.as_str(), "users", lookup_id, "member-of")) { - if !group.eq_ignore_ascii_case(&directory.opt.superuser_group) { - member_of.push(group.to_string()); - } else { - typ = Type::Superuser; - } - } - - directory.principals.insert( - name.clone(), - Principal { - name: name.clone(), - secrets: config - .values((prefix.as_str(), "users", lookup_id, "secret")) - .map(|(_, v)| v.to_string()) - .collect(), - typ, - description: config - .value((prefix.as_str(), "users", lookup_id, "description")) - .map(|v| v.to_string()), - quota: config - .property((prefix.as_str(), "users", lookup_id, "quota"))? - .unwrap_or(0), - member_of, - }, - ); - - directory.parse_emails(config, (prefix.as_str(), "users", lookup_id), name)?; - } - - for lookup_id in config.sub_keys((prefix.as_str(), "groups")) { - let name = config - .value_require((prefix.as_str(), "groups", lookup_id, "name"))? - .to_string(); - directory.principals.insert( - name.clone(), - Principal { - name: name.clone(), - secrets: vec![], - typ: Type::Group, - description: config - .value((prefix.as_str(), "groups", lookup_id, "description")) - .map(|v| v.to_string()), - quota: config - .property((prefix.as_str(), "groups", lookup_id, "quota"))? - .unwrap_or(0), - member_of: config - .values((prefix.as_str(), "groups", lookup_id, "member-of")) - .map(|(_, v)| v.to_string()) - .collect(), - }, - ); - - directory.parse_emails(config, (prefix.as_str(), "groups", lookup_id), name)?; - } - - Ok(Arc::new(directory)) - } -} - -impl MemoryDirectory { - fn parse_emails( - &mut self, - config: &Config, - prefix: impl AsKey, - name: String, - ) -> utils::config::Result<()> { - let prefix = prefix.as_key(); - let mut emails = Vec::new(); - - for (pos, (_, email)) in config.values((prefix.as_str(), "email")).enumerate() { - self.emails_to_names - .entry(email.to_string()) - .or_default() - .push(if pos > 0 { - EmailType::Alias(name.clone()) - } else { - EmailType::Primary(name.clone()) - }); - - if let Some((_, domain)) = email.rsplit_once('@') { - self.domains.insert(domain.to_lowercase()); - } - - emails.push(if pos > 0 { - EmailType::Alias(email.to_lowercase()) - } else { - EmailType::Primary(email.to_lowercase()) - }); - } - for (_, email) in config.values((prefix.as_str(), "email-list")) { - self.emails_to_names - .entry(email.to_lowercase()) - .or_default() - .push(EmailType::List(name.clone())); - if let Some((_, domain)) = email.rsplit_once('@') { - self.domains.insert(domain.to_lowercase()); - } - emails.push(EmailType::List(email.to_lowercase())); - } - - self.names_to_email.insert(name, emails); - Ok(()) - } -} diff --git a/crates/imap/src/core/mailbox.rs b/crates/imap/src/core/mailbox.rs index a20ae26e..69d45f96 100644 --- a/crates/imap/src/core/mailbox.rs +++ b/crates/imap/src/core/mailbox.rs @@ -1,6 +1,7 @@ use std::{collections::BTreeMap, sync::atomic::Ordering}; use ahash::AHashMap; +use directory::QueryBy; use imap_proto::{protocol::list::Attribute, StatusResponse}; use jmap::{ auth::{acl::EffectiveAcl, AccessToken}, @@ -50,9 +51,11 @@ impl SessionData { session.imap.name_shared, session .jmap - .get_account_name(account_id) + .directory + .query(QueryBy::id(account_id).with_store(&session.jmap.store)) .await .unwrap_or_default() + .map(|p| p.name) .unwrap_or_else(|| Id::from(account_id).to_string()) ) .into(), @@ -316,9 +319,11 @@ impl SessionData { "{}/{}", self.imap.name_shared, self.jmap - .get_account_name(account_id) + .directory + .query(QueryBy::id(account_id).with_store(&self.jmap.store)) .await .unwrap_or_default() + .map(|p| p.name) .unwrap_or_else(|| Id::from(account_id).to_string()) ); match self @@ -401,9 +406,11 @@ impl SessionData { "{}/{}", self.imap.name_shared, self.jmap - .get_account_name(account_id) + .directory + .query(QueryBy::id(account_id).with_store(&self.jmap.store)) .await .unwrap_or_default() + .map(|p| p.name) .unwrap_or_else(|| Id::from(account_id).to_string()) ) .into() diff --git a/crates/imap/src/op/acl.rs b/crates/imap/src/op/acl.rs index 3a8b6074..555527ec 100644 --- a/crates/imap/src/op/acl.rs +++ b/crates/imap/src/op/acl.rs @@ -23,6 +23,7 @@ use std::sync::Arc; +use directory::QueryBy; use imap_proto::{ protocol::acl::{ Arguments, GetAclResponse, ListRightsResponse, ModRightsOp, MyRightsResponse, Rights, @@ -74,9 +75,14 @@ impl Session { { if let Some(account_name) = data .jmap - .get_account_name(id.document_id()) + .directory + .query( + QueryBy::id(id.document_id()) + .with_store(&data.jmap.store), + ) .await .unwrap_or_default() + .map(|p| p.name) { let mut rights = Vec::new(); @@ -245,23 +251,13 @@ impl Session { let (acl_account_id, id) = match data .jmap .directory - .principal(arguments.identifier.as_ref().unwrap()) + .query( + QueryBy::name(arguments.identifier.as_ref().unwrap()) + .with_store(&data.jmap.store), + ) .await { - Ok(Some(principal)) => { - match data.jmap.get_account_id(principal.name()).await { - Ok(account_id) => (account_id, Value::Id(Id::from(account_id))), - Err(_) => { - data.write_bytes( - StatusResponse::database_failure() - .with_tag(arguments.tag) - .into_bytes(), - ) - .await; - return; - } - } - } + Ok(Some(principal)) => (principal.id, Value::Id(Id::from(principal.id))), Ok(None) => { data.write_bytes( StatusResponse::no("Account does not exist") diff --git a/crates/install/src/main.rs b/crates/install/src/main.rs index 8c9c02da..48557674 100644 --- a/crates/install/src/main.rs +++ b/crates/install/src/main.rs @@ -749,7 +749,7 @@ fn create_databases(base_path: &Path, domain: Option<&str>) -> std::io::Result. - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use jmap_proto::types::collection::Collection; -use store::{write::BatchBuilder, Serialize}; - -use crate::{NamedKey, JMAP}; - -impl JMAP { - pub async fn delete_account(&self, account_name: &str, account_id: u32) -> store::Result<()> { - let test = true; - - // Unlink all account's blobs - self.store.blob_hash_unlink_account(account_id).await?; - - // Revoke ACLs - self.store.acl_revoke_all(account_id).await?; - - // Delete account data - self.store.purge_account(account_id).await?; - - // Remove FTS index - self.fts_store.remove_all(account_id).await?; - - // Delete account - let mut batch = BatchBuilder::new(); - batch - .with_account_id(u32::MAX) - .with_collection(Collection::Principal) - .clear(NamedKey::Name(account_name)) - .clear(NamedKey::Id::<&[u8]>(account_id)) - .clear(NamedKey::Quota::<&[u8]>(account_id)); - - self.store.write(batch.build()).await?; - - Ok(()) - } - - pub async fn rename_account( - &self, - new_account_name: &str, - account_name: &str, - account_id: u32, - ) -> store::Result<()> { - let mut batch = BatchBuilder::new(); - batch - .with_account_id(u32::MAX) - .with_collection(Collection::Principal) - .clear(NamedKey::Name(account_name)) - .set( - NamedKey::Id::<&[u8]>(account_id), - new_account_name.serialize(), - ) - .set(NamedKey::Name(new_account_name), account_id.serialize()); - self.store.write(batch.build()).await?; - Ok(()) - } -} diff --git a/crates/jmap/src/api/http.rs b/crates/jmap/src/api/http.rs index 17ee5bbe..b062b4eb 100644 --- a/crates/jmap/src/api/http.rs +++ b/crates/jmap/src/api/http.rs @@ -289,7 +289,14 @@ pub async fn parse_jmap_request( req.method(), ) { ("account", "delete", &Method::GET) => { - return if let Some(account_name) = path.next() { + let todo = true; + /* + + // Remove FTS index + self.fts_store.remove_all(principal.id).await?; + */ + todo!() + /*return if let Some(account_name) = path.next() { if let Ok(Some(account_id)) = jmap.try_get_account_id(account_name).await { match jmap.delete_account(account_name, account_id).await { Ok(_) => JsonResponse::new(Value::String("success".into())) @@ -316,10 +323,11 @@ pub async fn parse_jmap_request( "Expected account name", ) .into_http_response() - }; + };*/ } ("account", "rename", &Method::GET) => { - return if let (Some(account_name), Some(new_account_name)) = + todo!() + /*return if let (Some(account_name), Some(new_account_name)) = (path.next(), path.next()) { match ( @@ -362,7 +370,7 @@ pub async fn parse_jmap_request( "Expected old and new account names", ) .into_http_response() - }; + };*/ } ("blob", "purge", &Method::GET) => { return match jmap.store.purge_blobs(jmap.blob_store.clone()).await { diff --git a/crates/jmap/src/api/mod.rs b/crates/jmap/src/api/mod.rs index 927225d6..74dc6712 100644 --- a/crates/jmap/src/api/mod.rs +++ b/crates/jmap/src/api/mod.rs @@ -30,7 +30,6 @@ use utils::map::vec_map::VecMap; use crate::JMAP; -pub mod admin; pub mod config; pub mod event_source; pub mod http; diff --git a/crates/jmap/src/api/request.rs b/crates/jmap/src/api/request.rs index aec30809..e817e2db 100644 --- a/crates/jmap/src/api/request.rs +++ b/crates/jmap/src/api/request.rs @@ -250,7 +250,7 @@ impl JMAP { set::RequestArguments::Identity => { access_token.assert_is_member(req.account_id)?; - self.identity_set(req, access_token).await?.into() + self.identity_set(req).await?.into() } set::RequestArguments::EmailSubmission(arguments) => { access_token.assert_is_member(req.account_id)?; diff --git a/crates/jmap/src/api/session.rs b/crates/jmap/src/api/session.rs index 305ab023..5797493b 100644 --- a/crates/jmap/src/api/session.rs +++ b/crates/jmap/src/api/session.rs @@ -23,6 +23,7 @@ use std::sync::Arc; +use directory::QueryBy; use jmap_proto::{ error::request::RequestError, request::capability::Capability, @@ -212,9 +213,11 @@ impl JMAP { session.add_account( (*id).into(), - self.get_account_name(*id) + self.directory + .query(QueryBy::id(*id).with_store(&self.store)) .await .unwrap_or_default() + .map(|p| p.name) .unwrap_or_else(|| Id::from(*id).to_string()), is_personal, is_readonly, diff --git a/crates/jmap/src/auth/acl.rs b/crates/jmap/src/auth/acl.rs index 5c3c9cb5..9585021f 100644 --- a/crates/jmap/src/auth/acl.rs +++ b/crates/jmap/src/auth/acl.rs @@ -21,6 +21,7 @@ * for more details. */ +use directory::QueryBy; use jmap_proto::{ error::{method::MethodError, set::SetError}, object::Object, @@ -376,13 +377,14 @@ impl JMAP { if let (Some(Value::Id(id)), Some(Value::UnsignedInt(acl_bits))) = (item.first(), item.last()) { - if let Some(account_name) = self - .get_account_name(id.document_id()) + if let Some(principal) = self + .directory + .query(QueryBy::id(id.document_id()).with_store(&self.store)) .await .unwrap_or_default() { acl_obj.append( - Property::_T(account_name), + Property::_T(principal.name), Bitmap::::from(*acl_bits) .map(|acl_item| Value::Text(acl_item.to_string())) .collect::>(), @@ -450,18 +452,13 @@ impl JMAP { async fn map_acl_accounts(&self, mut acl_set: Vec) -> Result, SetError> { for item in &mut acl_set { if let Value::Text(account_name) = item { - match self.directory.principal(account_name).await { - Ok(Some(_)) => { - *item = Value::Id( - self.get_account_id(account_name) - .await - .map_err(|_| { - SetError::forbidden() - .with_property(Property::Acl) - .with_description("Temporary server failure during lookup") - })? - .into(), - ); + match self + .directory + .query(QueryBy::name(account_name).with_store(&self.store)) + .await + { + Ok(Some(principal)) => { + *item = Value::Id(principal.id.into()); } Ok(None) => { return Err(SetError::invalid_properties() diff --git a/crates/jmap/src/auth/authenticate.rs b/crates/jmap/src/auth/authenticate.rs index 8003b67d..96ad2d58 100644 --- a/crates/jmap/src/auth/authenticate.rs +++ b/crates/jmap/src/auth/authenticate.rs @@ -27,17 +27,14 @@ use std::{ time::Instant, }; +use directory::QueryBy; use hyper::header; -use jmap_proto::{ - error::{method::MethodError, request::RequestError}, - types::collection::Collection, -}; +use jmap_proto::error::request::RequestError; use mail_parser::decoders::base64::base64_decode; use mail_send::Credentials; -use store::{write::BatchBuilder, Serialize}; use utils::{listener::limiter::InFlight, map::ttl_dashmap::TtlMap}; -use crate::{NamedKey, JMAP}; +use crate::JMAP; use super::{rate_limit::RemoteAddress, AccessToken}; @@ -150,85 +147,6 @@ impl JMAP { } } - pub async fn try_get_account_id(&self, name: &str) -> Result, MethodError> { - self.store - .get_value::(NamedKey::Name(name)) - .await - .map_err(|err| { - tracing::error!(event = "error", - context = "store", - account_name = name, - error = ?err, - "Failed to retrieve account id"); - MethodError::ServerPartialFail - }) - } - - pub async fn get_account_id(&self, name: &str) -> Result { - let mut try_count = 0; - - loop { - // Try to obtain ID - if let Some(account_id) = self.try_get_account_id(name).await? { - return Ok(account_id); - } - - // Assign new ID - let account_id = self - .assign_document_id(u32::MAX, Collection::Principal) - .await?; - - // Write account ID - let mut batch = BatchBuilder::new(); - batch - .with_account_id(u32::MAX) - .with_collection(Collection::Principal) - .create_document(account_id) - .assert_value(NamedKey::Name(name), ()) - .set(NamedKey::Name(name), account_id.serialize()) - .set(NamedKey::Id::<&[u8]>(account_id), name.serialize()); - - match self.store.write(batch.build()).await { - Ok(_) => { - return Ok(account_id); - } - Err(store::Error::AssertValueFailed) if try_count < 3 => { - try_count += 1; - continue; - } - Err(err) => { - tracing::error!(event = "error", - context = "store", - error = ?err, - "Failed to generate account id"); - return Err(MethodError::ServerPartialFail); - } - } - } - } - - pub async fn map_member_of(&self, names: Vec) -> Result, MethodError> { - let mut ids = Vec::with_capacity(names.len()); - for name in names { - ids.push(self.get_account_id(&name).await?); - } - Ok(ids) - } - - pub async fn get_account_name(&self, account_id: u32) -> Result, MethodError> { - self.store - .get_value::(NamedKey::Id::<&[u8]>(account_id)) - .await - .map_err(|err| { - tracing::error!(event = "error", - context = "store", - account_id = account_id, - error = ?err, - "Failed to retrieve account name"); - MethodError::ServerPartialFail - }) - } - pub fn build_remote_addr( &self, req: &hyper::Request, @@ -254,60 +172,40 @@ impl JMAP { secret: &str, remote_addr: &RemoteAddress, ) -> Option { - let mut principal = match self + match self .directory - .authenticate(&Credentials::Plain { - username: username.to_string(), - secret: secret.to_string(), - }) - .await - { - Ok(Some(principal)) => principal, - Ok(None) => { - let _ = self.is_auth_allowed_hard(remote_addr); - return None; - } - Err(_) => { - return None; - } - }; - - if !principal.has_name() { - principal.name = username.to_string(); - } - // Obtain groups - if let (Ok(account_id), Ok(member_of)) = ( - self.get_account_id(&principal.name).await, - self.map_member_of(std::mem::take(&mut principal.member_of)) - .await, - ) { - // Create access token - self.update_access_token( - AccessToken::new(principal, account_id).with_member_of(member_of), + .query( + QueryBy::credentials(&Credentials::Plain { + username: username.to_string(), + secret: secret.to_string(), + }) + .with_store(&self.store), ) .await - } else { - None + { + Ok(Some(mut principal)) => { + if !principal.has_name() { + principal.name = username.to_string(); + } + + AccessToken::new(principal).into() + } + Ok(None) => { + let _ = self.is_auth_allowed_hard(remote_addr); + None + } + Err(_) => None, } } pub async fn get_access_token(&self, account_id: u32) -> Option { - let name = self.get_account_name(account_id).await.ok()??; - let mut principal = self.directory.principal(&name).await.ok()??; - - // Obtain groups - if let (Ok(account_id), Ok(member_of)) = ( - self.get_account_id(&principal.name).await, - self.map_member_of(std::mem::take(&mut principal.member_of)) - .await, - ) { - // Create access token - self.update_access_token( - AccessToken::new(principal, account_id).with_member_of(member_of), - ) - .await - } else { - None - } + // Create access token + self.update_access_token(AccessToken::new( + self.directory + .query(QueryBy::id(account_id).with_store(&self.store)) + .await + .ok()??, + )) + .await } } diff --git a/crates/jmap/src/auth/mod.rs b/crates/jmap/src/auth/mod.rs index 76c73d1c..ac30addd 100644 --- a/crates/jmap/src/auth/mod.rs +++ b/crates/jmap/src/auth/mod.rs @@ -56,10 +56,10 @@ pub struct AccessToken { } impl AccessToken { - pub fn new(principal: Principal, primary_id: u32) -> Self { + pub fn new(principal: Principal) -> Self { Self { - primary_id, - member_of: Vec::new(), + primary_id: principal.id, + member_of: principal.member_of, access_to: Vec::new(), name: principal.name, description: principal.description, @@ -68,10 +68,6 @@ impl AccessToken { } } - pub fn with_member_of(self, member_of: Vec) -> Self { - Self { member_of, ..self } - } - pub fn with_access_to(self, access_to: Vec<(u32, Bitmap)>) -> Self { Self { access_to, ..self } } diff --git a/crates/jmap/src/auth/oauth/token.rs b/crates/jmap/src/auth/oauth/token.rs index 43cf9231..fe42e1d6 100644 --- a/crates/jmap/src/auth/oauth/token.rs +++ b/crates/jmap/src/auth/oauth/token.rs @@ -23,6 +23,7 @@ use std::{sync::atomic, time::SystemTime}; +use directory::QueryBy; use hyper::StatusCode; use mail_builder::encoders::base64::base64_encode; use mail_parser::decoders::base64::base64_decode; @@ -179,14 +180,9 @@ impl JMAP { client_id: &str, with_refresh_token: bool, ) -> Result { - let account_name = self - .get_account_name(account_id) - .await - .map_err(|_| "Temporary lookup error")? - .ok_or("Account no longer exists")?; let password_hash = self .directory - .principal(&account_name) + .query(QueryBy::id(account_id).with_store(&self.store)) .await .map_err(|_| "Temporary lookup error")? .ok_or("Account no longer exists")? @@ -302,14 +298,10 @@ impl JMAP { } // Optain password hash - let account_name = self - .get_account_name(account_id) - .await - .map_err(|_| "Temporary lookup error")? - .ok_or("Account no longer exists")?; + let password_hash = self .directory - .principal(&account_name) + .query(QueryBy::id(account_id).with_store(&self.store)) .await .map_err(|_| "Temporary lookup error")? .ok_or("Account no longer exists")? diff --git a/crates/jmap/src/email/copy.rs b/crates/jmap/src/email/copy.rs index 57a488e9..8d4ce52e 100644 --- a/crates/jmap/src/email/copy.rs +++ b/crates/jmap/src/email/copy.rs @@ -48,14 +48,12 @@ use jmap_proto::{ }; use mail_parser::{parsers::fields::thread::thread_name, HeaderName, HeaderValue}; use store::{ - write::{BatchBuilder, F_BITMAP, F_VALUE}, + write::{BatchBuilder, ValueClass, F_BITMAP, F_VALUE}, BlobClass, }; use utils::map::vec_map::VecMap; -use crate::{ - auth::AccessToken, mailbox::UidMailbox, services::housekeeper::Event, Bincode, NamedKey, JMAP, -}; +use crate::{auth::AccessToken, mailbox::UidMailbox, services::housekeeper::Event, Bincode, JMAP}; use super::{ index::{EmailIndexBuilder, TrimTextValue, MAX_SORT_FIELD_LENGTH}, @@ -415,11 +413,7 @@ impl JMAP { .value(Property::Keywords, keywords, F_VALUE | F_BITMAP) .value(Property::Cid, changes.change_id, F_VALUE) .set( - NamedKey::IndexEmail::<&[u8]> { - account_id, - document_id: message_id, - seq: self.generate_snowflake_id()?, - }, + ValueClass::IndexEmail(self.generate_snowflake_id()?), metadata.blob_hash.clone(), ) .custom(EmailIndexBuilder::set(metadata)) diff --git a/crates/jmap/src/email/index.rs b/crates/jmap/src/email/index.rs index 09320b65..c43e7acf 100644 --- a/crates/jmap/src/email/index.rs +++ b/crates/jmap/src/email/index.rs @@ -34,11 +34,13 @@ use nlp::language::Language; use store::{ backend::MAX_TOKEN_LENGTH, fts::{index::FtsDocument, Field}, - write::{BatchBuilder, BlobOp, IntoOperations, F_BITMAP, F_CLEAR, F_INDEX, F_VALUE}, + write::{ + BatchBuilder, BlobOp, DirectoryValue, IntoOperations, F_BITMAP, F_CLEAR, F_INDEX, F_VALUE, + }, BlobHash, }; -use crate::{mailbox::UidMailbox, Bincode, NamedKey}; +use crate::{mailbox::UidMailbox, Bincode}; use super::metadata::MessageMetadata; @@ -90,7 +92,7 @@ impl IndexMessage for BatchBuilder { let account_id = self.last_account_id().unwrap(); self.value(Property::Size, message.raw_message.len() as u32, F_INDEX) .add( - NamedKey::Quota::<&[u8]>(account_id), + DirectoryValue::UsedQuota(account_id), message.raw_message.len() as i64, ); @@ -413,7 +415,7 @@ impl<'x> IntoOperations for EmailIndexBuilder<'x> { batch .value(Property::Size, metadata.size as u32, F_INDEX | options) .add( - NamedKey::Quota::<&[u8]>(account_id), + DirectoryValue::UsedQuota(account_id), if self.set { metadata.size as i64 } else { diff --git a/crates/jmap/src/email/ingest.rs b/crates/jmap/src/email/ingest.rs index 07b43850..5d885efb 100644 --- a/crates/jmap/src/email/ingest.rs +++ b/crates/jmap/src/email/ingest.rs @@ -49,7 +49,7 @@ use crate::{ email::index::{IndexMessage, MAX_ID_LENGTH}, mailbox::UidMailbox, services::housekeeper::Event, - IngestError, NamedKey, JMAP, + IngestError, JMAP, }; use super::{ @@ -317,13 +317,10 @@ impl JMAP { .value(Property::ThreadId, thread_id, F_VALUE | F_BITMAP) .custom(changes) .set( - NamedKey::IndexEmail::<&[u8]> { - account_id: params.account_id, - document_id, - seq: self - .generate_snowflake_id() + ValueClass::IndexEmail( + self.generate_snowflake_id() .map_err(|_| IngestError::Temporary)?, - }, + ), blob_id.hash.clone(), ); self.store.write(batch.build()).await.map_err(|err| { diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index 7d42962a..202e7316 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -61,7 +61,7 @@ use store::{ use crate::{ auth::AccessToken, mailbox::UidMailbox, services::housekeeper::Event, Bincode, IngestError, - NamedKey, JMAP, + JMAP, }; use super::{ @@ -1071,7 +1071,11 @@ impl JMAP { batch .with_account_id(account_id) .with_collection(Collection::Email) - .delete_document(document_id); + .delete_document(document_id) + .set( + ValueClass::IndexEmail(self.generate_snowflake_id()?), + vec![], + ); // Remove last changeId batch.value(Property::Cid, (), F_VALUE | F_CLEAR); @@ -1217,16 +1221,6 @@ impl JMAP { .delete_document(thread_id); } - // Remove message from FTS index - batch.set( - NamedKey::IndexEmail::<&[u8]> { - account_id, - document_id, - seq: self.generate_snowflake_id()?, - }, - vec![], - ); - // Commit batch match self.store.write(batch.build()).await { Ok(_) => (), diff --git a/crates/jmap/src/identity/set.rs b/crates/jmap/src/identity/set.rs index 68a26860..6f2099f4 100644 --- a/crates/jmap/src/identity/set.rs +++ b/crates/jmap/src/identity/set.rs @@ -21,6 +21,7 @@ * for more details. */ +use directory::QueryBy; use jmap_proto::{ error::{method::MethodError, set::SetError}, method::set::{RequestArguments, SetRequest, SetResponse}, @@ -34,13 +35,12 @@ use jmap_proto::{ }; use store::write::{log::ChangeLogBuilder, BatchBuilder, F_CLEAR, F_VALUE}; -use crate::{auth::AccessToken, JMAP}; +use crate::JMAP; impl JMAP { pub async fn identity_set( &self, mut request: SetRequest, - access_token: &AccessToken, ) -> Result { let account_id = request.account_id.document_id(); let mut identity_ids = self @@ -73,17 +73,13 @@ impl JMAP { // Validate email address if let Value::Text(email) = identity.get(&Property::Email) { - let account_name = if access_token.primary_id == account_id { - access_token.name.clone() - } else { - self.get_account_name(account_id).await?.unwrap_or_default() - }; - if !self .directory - .emails_by_name(&account_name) + .query(QueryBy::id(account_id).with_store(&self.store)) .await .unwrap_or_default() + .unwrap_or_default() + .emails .contains(email) { response.not_created.append( diff --git a/crates/jmap/src/lib.rs b/crates/jmap/src/lib.rs index 526ee7b4..b0d70cec 100644 --- a/crates/jmap/src/lib.rs +++ b/crates/jmap/src/lib.rs @@ -31,7 +31,7 @@ use auth::{ AccessToken, }; use dashmap::DashMap; -use directory::{Directories, Directory}; +use directory::{Directories, Directory, QueryBy}; use jmap_proto::{ error::method::MethodError, method::{ @@ -52,12 +52,8 @@ use store::{ parking_lot::Mutex, query::{sort::Pagination, Comparator, Filter, ResultSet, SortedResultSet}, roaring::RoaringBitmap, - write::{ - key::{DeserializeBigEndian, KeySerializer}, - BatchBuilder, BitmapClass, TagValue, ToBitmaps, ValueClass, - }, - BitmapKey, BlobStore, Deserialize, FtsStore, Key, Serialize, Store, Stores, ValueKey, - SUBSPACE_INDEX_VALUES, U32_LEN, U64_LEN, + write::{BatchBuilder, BitmapClass, DirectoryValue, TagValue, ToBitmaps, ValueClass}, + BitmapKey, BlobStore, Deserialize, FtsStore, Serialize, Store, Stores, ValueKey, }; use tokio::sync::mpsc; use utils::{ @@ -603,7 +599,7 @@ impl JMAP { access_token.quota as i64 } else { self.directory - .principal(&access_token.name) + .query(QueryBy::id(account_id).with_store(&self.store)) .await .map_err(|err| { tracing::error!( @@ -621,7 +617,7 @@ impl JMAP { pub async fn get_used_quota(&self, account_id: u32) -> Result { self.store - .get_counter(NamedKey::Quota::<&[u8]>(account_id)) + .get_counter(DirectoryValue::UsedQuota(account_id)) .await .map_err(|err| { tracing::error!( @@ -848,93 +844,3 @@ impl UpdateResults for QueryResponse { } } } - -pub enum NamedKey> { - Name(T), - Id(u32), - Quota(u32), - IndexEmail { - account_id: u32, - document_id: u32, - seq: u64, - }, -} - -impl> From<&NamedKey> for ValueClass { - fn from(key: &NamedKey) -> Self { - match key { - NamedKey::Name(name) => ValueClass::Subspace { - key: name.as_ref().to_vec(), - id: 0, - }, - NamedKey::Id(id) => ValueClass::Subspace { - key: KeySerializer::new(std::mem::size_of::()) - .write_leb128(*id) - .finalize(), - id: 1, - }, - NamedKey::Quota(id) => ValueClass::Subspace { - key: KeySerializer::new(std::mem::size_of::()) - .write_leb128(*id) - .finalize(), - id: 2, - }, - NamedKey::IndexEmail { - account_id, - document_id, - seq, - } => ValueClass::Subspace { - key: KeySerializer::new(std::mem::size_of::() * 4) - .write(*seq) - .write(*account_id) - .write(*document_id) - .finalize(), - id: 3, - }, - } - } -} - -impl> NamedKey { - pub fn deserialize_index_email(bytes: &[u8]) -> store::Result { - let len = bytes.len(); - Ok(NamedKey::IndexEmail { - seq: bytes.deserialize_be_u64(len - U64_LEN - (U32_LEN * 2))?, - account_id: bytes.deserialize_be_u32(len - U32_LEN * 2)?, - document_id: bytes.deserialize_be_u32(len - U32_LEN)?, - }) - } -} - -impl> From> for ValueClass { - fn from(key: NamedKey) -> Self { - (&key).into() - } -} - -impl> From> for ValueKey { - fn from(key: NamedKey) -> Self { - ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: key.into(), - } - } -} - -impl + Sync + Send> Key for NamedKey { - fn serialize(&self, include_subspace: bool) -> Vec { - ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::from(self), - } - .serialize(include_subspace) - } - - fn subspace(&self) -> u8 { - SUBSPACE_INDEX_VALUES - } -} diff --git a/crates/jmap/src/principal/get.rs b/crates/jmap/src/principal/get.rs index 95e52062..1651bcdd 100644 --- a/crates/jmap/src/principal/get.rs +++ b/crates/jmap/src/principal/get.rs @@ -21,6 +21,7 @@ * for more details. */ +use directory::QueryBy; use jmap_proto::{ error::method::MethodError, method::get::{GetRequest, GetResponse, RequestArguments}, @@ -66,18 +67,10 @@ impl JMAP { }; for id in ids { - // Obtain the principal name - let name = if let Some(name) = self.get_account_name(id.document_id()).await? { - name - } else { - response.not_found.push(id.into()); - continue; - }; - // Obtain the principal let principal = if let Some(principal) = self .directory - .principal(&name) + .query(QueryBy::id(id.document_id()).with_store(&self.store)) .await .map_err(|_| MethodError::ServerPartialFail)? { @@ -98,14 +91,10 @@ impl JMAP { .clone() .map(Value::Text) .unwrap_or(Value::Null), - Property::Email => self - .directory - .emails_by_name(&name) - .await - .map_err(|_| MethodError::ServerPartialFail)? - .into_iter() - .next() - .map(Value::Text) + Property::Email => principal + .emails + .first() + .map(|email| Value::Text(email.clone())) .unwrap_or(Value::Null), _ => Value::Null, }; diff --git a/crates/jmap/src/principal/query.rs b/crates/jmap/src/principal/query.rs index cf22142f..7578adea 100644 --- a/crates/jmap/src/principal/query.rs +++ b/crates/jmap/src/principal/query.rs @@ -21,6 +21,7 @@ * for more details. */ +use directory::QueryBy; use jmap_proto::{ error::method::MethodError, method::query::{Filter, QueryRequest, QueryResponse, RequestArguments}, @@ -48,14 +49,13 @@ impl JMAP { Filter::Name(name) => { if let Some(principal) = self .directory - .principal(&name) + .query(QueryBy::name(name.as_str()).with_store(&self.store)) .await .map_err(|_| MethodError::ServerPartialFail)? { - let account_id = self.get_account_id(&principal.name).await?; - if is_set || result_set.results.contains(account_id) { + if is_set || result_set.results.contains(principal.id) { result_set.results = - RoaringBitmap::from_sorted_iter([account_id]).unwrap(); + RoaringBitmap::from_sorted_iter([principal.id]).unwrap(); } else { result_set.results = RoaringBitmap::new(); } @@ -66,13 +66,13 @@ impl JMAP { } Filter::Email(email) => { let mut ids = RoaringBitmap::new(); - for name in self + for id in self .directory - .names_by_email(&email) + .email_to_ids(&email, &self.store) .await .map_err(|_| MethodError::ServerPartialFail)? { - ids.insert(self.get_account_id(&name).await?); + ids.insert(id); } if is_set { result_set.results = ids; diff --git a/crates/jmap/src/services/index.rs b/crates/jmap/src/services/index.rs index 1549c993..51bbd9d2 100644 --- a/crates/jmap/src/services/index.rs +++ b/crates/jmap/src/services/index.rs @@ -24,40 +24,37 @@ use jmap_proto::types::{collection::Collection, property::Property}; use store::{ fts::index::FtsDocument, - write::{BatchBuilder, ValueClass}, - IterateParams, ValueKey, + write::{key::DeserializeBigEndian, BatchBuilder, ValueClass}, + Deserialize, IterateParams, ValueKey, U32_LEN, U64_LEN, }; use crate::{ email::{index::IndexMessageText, metadata::MessageMetadata}, - Bincode, NamedKey, JMAP, + Bincode, JMAP, }; use super::housekeeper::Event; +#[derive(Debug)] +struct IndexEmail { + account_id: u32, + document_id: u32, + seq: u64, +} + impl JMAP { pub async fn fts_index_queued(&self) { let from_key = ValueKey:: { account_id: 0, collection: 0, document_id: 0, - class: NamedKey::IndexEmail::<&[u8]> { - account_id: 0, - document_id: 0, - seq: 0, - } - .into(), + class: ValueClass::IndexEmail(0), }; let to_key = ValueKey:: { account_id: u32::MAX, collection: u8::MAX, document_id: u32::MAX, - class: NamedKey::IndexEmail::<&[u8]> { - account_id: u32::MAX, - document_id: u32::MAX, - seq: u64::MAX, - } - .into(), + class: ValueClass::IndexEmail(u64::MAX), }; // Retrieve entries pending to be indexed @@ -68,10 +65,7 @@ impl JMAP { .iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { - entries.push(( - NamedKey::>::deserialize_index_email(key)?, - value.to_vec(), - )); + entries.push((IndexEmail::deserialize(key)?, value.to_vec())); Ok(true) }, ) @@ -87,124 +81,123 @@ impl JMAP { // Index entries for (key, blob_hash) in entries { - if let NamedKey::IndexEmail { - account_id, - document_id, - .. - } = &key - { - if !blob_hash.is_empty() { - match self - .get_property::>( - *account_id, - Collection::Email, - *document_id, - Property::BodyStructure, - ) - .await + if !blob_hash.is_empty() { + match self + .get_property::>( + key.account_id, + Collection::Email, + key.document_id, + Property::BodyStructure, + ) + .await + { + Ok(Some(metadata)) + if metadata.inner.blob_hash.as_slice() == blob_hash.as_slice() => { - Ok(Some(metadata)) - if metadata.inner.blob_hash.as_slice() == blob_hash.as_slice() => + // Obtain raw message + let raw_message = if let Ok(Some(raw_message)) = + self.get_blob(&metadata.inner.blob_hash, 0..u32::MAX).await { - // Obtain raw message - let raw_message = if let Ok(Some(raw_message)) = - self.get_blob(&metadata.inner.blob_hash, 0..u32::MAX).await - { - raw_message - } else { - tracing::warn!( - context = "fts_index_queued", - event = "error", - account_id = *account_id, - document_id = *document_id, - blob_hash = ?metadata.inner.blob_hash, - "Message blob not found" - ); - continue; - }; - let message = metadata.inner.contents.into_message(&raw_message); - - // Index message - let document = - FtsDocument::with_default_language(self.config.default_language) - .with_account_id(*account_id) - .with_collection(Collection::Email) - .with_document_id(*document_id) - .index_message(&message); - if let Err(err) = self.fts_store.index(document).await { - tracing::error!( - context = "fts_index_queued", - event = "error", - account_id = *account_id, - document_id = *document_id, - reason = ?err, - "Failed to index email in FTS index" - ); - continue; - } - - tracing::debug!( + raw_message + } else { + tracing::warn!( context = "fts_index_queued", - event = "index", - account_id = *account_id, - document_id = *document_id, - "Indexed document in FTS index" + event = "error", + account_id = key.account_id, + document_id = key.document_id, + blob_hash = ?metadata.inner.blob_hash, + "Message blob not found" ); - } + continue; + }; + let message = metadata.inner.contents.into_message(&raw_message); - Err(err) => { + // Index message + let document = + FtsDocument::with_default_language(self.config.default_language) + .with_account_id(key.account_id) + .with_collection(Collection::Email) + .with_document_id(key.document_id) + .index_message(&message); + if let Err(err) = self.fts_store.index(document).await { tracing::error!( context = "fts_index_queued", event = "error", - account_id = *account_id, - document_id = *document_id, + account_id = key.account_id, + document_id = key.document_id, reason = ?err, - "Failed to retrieve email metadata" - ); - break; - } - _ => { - // The message was probably deleted or overwritten - tracing::debug!( - context = "fts_index_queued", - event = "error", - account_id = *account_id, - document_id = *document_id, - "Email metadata not found" + "Failed to index email in FTS index" ); + continue; } + + tracing::debug!( + context = "fts_index_queued", + event = "index", + account_id = key.account_id, + document_id = key.document_id, + "Indexed document in FTS index" + ); } - } else { - if let Err(err) = self - .fts_store - .remove(*account_id, Collection::Email.into(), *document_id) - .await - { + + Err(err) => { tracing::error!( context = "fts_index_queued", event = "error", - account_id = *account_id, - document_id = *document_id, + account_id = key.account_id, + document_id = key.document_id, reason = ?err, - "Failed to remove document from FTS index" + "Failed to retrieve email metadata" + ); + break; + } + _ => { + // The message was probably deleted or overwritten + tracing::debug!( + context = "fts_index_queued", + event = "error", + account_id = key.account_id, + document_id = key.document_id, + "Email metadata not found" ); - continue; } - - tracing::debug!( - context = "fts_index_queued", - event = "delete", - account_id = *account_id, - document_id = *document_id, - "Deleted document from FTS index" - ); } + } else { + if let Err(err) = self + .fts_store + .remove(key.account_id, Collection::Email.into(), key.document_id) + .await + { + tracing::error!( + context = "fts_index_queued", + event = "error", + account_id = key.account_id, + document_id = key.document_id, + reason = ?err, + "Failed to remove document from FTS index" + ); + continue; + } + + tracing::debug!( + context = "fts_index_queued", + event = "delete", + account_id = key.account_id, + document_id = key.document_id, + "Deleted document from FTS index" + ); } // Remove entry from queue if let Err(err) = self .store - .write(BatchBuilder::new().clear(key).build_batch()) + .write( + BatchBuilder::new() + .with_account_id(key.account_id) + .update_document(key.document_id) + .clear(ValueClass::IndexEmail(key.seq)) + .build_batch(), + ) .await { tracing::error!( @@ -222,3 +215,14 @@ impl JMAP { } } } + +impl Deserialize for IndexEmail { + fn deserialize(bytes: &[u8]) -> store::Result { + let len = bytes.len(); + Ok(IndexEmail { + seq: bytes.deserialize_be_u64(len - U64_LEN - (U32_LEN * 2))?, + account_id: bytes.deserialize_be_u32(len - U32_LEN * 2)?, + document_id: bytes.deserialize_be_u32(len - U32_LEN)?, + }) + } +} diff --git a/crates/jmap/src/services/ingest.rs b/crates/jmap/src/services/ingest.rs index 74847632..fd140165 100644 --- a/crates/jmap/src/services/ingest.rs +++ b/crates/jmap/src/services/ingest.rs @@ -21,6 +21,7 @@ * for more details. */ +use directory::QueryBy; use jmap_proto::types::{state::StateChange, type_state::DataType}; use mail_parser::MessageParser; use store::ahash::AHashMap; @@ -46,45 +47,37 @@ impl JMAP { let mut recipients = Vec::with_capacity(message.recipients.len()); let mut deliver_names = AHashMap::with_capacity(message.recipients.len()); for rcpt in &message.recipients { - let names = self + let uids = self .directory - .names_by_email(rcpt) + .email_to_ids(rcpt, &self.store) .await .unwrap_or_default(); - for name in &names { - deliver_names.insert(name.clone(), (DeliveryResult::Success, rcpt)); + for uid in &uids { + deliver_names.insert(*uid, (DeliveryResult::Success, rcpt)); } - recipients.push(names); + recipients.push(uids); } // Deliver to each recipient - for (name, (status, rcpt)) in &mut deliver_names { - // Obtain account id - let uid = match self.get_account_id(name).await { - Ok(uid) => uid, - Err(_) => { - *status = DeliveryResult::TemporaryFailure { - reason: "Transient server failure.".into(), - }; - continue; - } - }; - + for (uid, (status, rcpt)) in &mut deliver_names { // Check if there is an active sieve script - let result = match self.sieve_script_get_active(uid).await { + let result = match self.sieve_script_get_active(*uid).await { Ok(Some(active_script)) => { self.sieve_script_ingest( &raw_message, &message.sender_address, rcpt, - uid, - name, + *uid, active_script, ) .await } Ok(None) => { - let account_quota = match self.directory.principal(name).await { + let account_quota = match self + .directory + .query(QueryBy::id(*uid).with_store(&self.store)) + .await + { Ok(Some(p)) => p.quota as i64, Ok(None) => 0, Err(_) => { @@ -98,7 +91,7 @@ impl JMAP { self.email_ingest(IngestEmail { raw_message: &raw_message, message: MessageParser::new().parse(&raw_message), - account_id: uid, + account_id: *uid, account_quota, mailbox_ids: vec![INBOX_ID], keywords: vec![], @@ -121,7 +114,7 @@ impl JMAP { // Notify state change if ingested_message.change_id != u64::MAX { self.broadcast_state_change( - StateChange::new(uid) + StateChange::new(*uid) .with_change(DataType::EmailDelivery, ingested_message.change_id) .with_change(DataType::Email, ingested_message.change_id) .with_change(DataType::Mailbox, ingested_message.change_id) diff --git a/crates/jmap/src/sieve/ingest.rs b/crates/jmap/src/sieve/ingest.rs index ccb15b18..89e84248 100644 --- a/crates/jmap/src/sieve/ingest.rs +++ b/crates/jmap/src/sieve/ingest.rs @@ -23,6 +23,7 @@ use std::borrow::Cow; +use directory::QueryBy; use jmap_proto::types::{collection::Collection, id::Id, keyword::Keyword, property::Property}; use mail_parser::MessageParser; use sieve::{Envelope, Event, Input, Mailbox, Recipient}; @@ -55,7 +56,6 @@ impl JMAP { envelope_from: &str, envelope_to: &str, account_id: u32, - account_name: &str, mut active_script: ActiveScript, ) -> Result { // Parse message @@ -77,35 +77,26 @@ impl JMAP { // Create Sieve instance let mut instance = self.sieve_runtime.filter_parsed(message); - // Obtain mail from address - let mail_from = if let Some(email) = self - .directory - .emails_by_name(account_name) - .await - .unwrap_or_default() - .into_iter() - .next() - { - email - } else { - envelope_to.to_string() - }; - - // Set account address - instance.set_user_address(&mail_from); - // Set account name and obtain quota - let account_quota = match self.directory.principal(account_name).await { + let (account_quota, mail_from) = match self + .directory + .query(QueryBy::id(account_id).with_store(&self.store)) + .await + { Ok(Some(p)) => { instance.set_user_full_name(p.description().unwrap_or_else(|| p.name())); - p.quota as i64 + (p.quota as i64, p.emails.into_iter().next()) } - Ok(None) => 0, + Ok(None) => (0, None), Err(_) => { return Err(IngestError::Temporary); } }; + // Set account address + let mail_from = mail_from.unwrap_or_else(|| envelope_to.to_string()); + instance.set_user_address(&mail_from); + // Set envelope instance.set_envelope(Envelope::From, envelope_from); instance.set_envelope(Envelope::To, envelope_to); diff --git a/crates/jmap/src/sieve/set.rs b/crates/jmap/src/sieve/set.rs index dbf00cdd..f27818ea 100644 --- a/crates/jmap/src/sieve/set.rs +++ b/crates/jmap/src/sieve/set.rs @@ -46,11 +46,14 @@ use sieve::compiler::ErrorType; use store::{ query::Filter, rand::{distributions::Alphanumeric, thread_rng, Rng}, - write::{assert::HashedValue, log::ChangeLogBuilder, BatchBuilder, BlobOp, F_CLEAR, F_VALUE}, + write::{ + assert::HashedValue, log::ChangeLogBuilder, BatchBuilder, BlobOp, DirectoryValue, F_CLEAR, + F_VALUE, + }, BlobClass, }; -use crate::{auth::AccessToken, NamedKey, JMAP}; +use crate::{auth::AccessToken, JMAP}; struct SetContext<'x> { account_id: u32, @@ -119,7 +122,7 @@ impl JMAP { .with_account_id(account_id) .with_collection(Collection::SieveScript) .create_document(document_id) - .add(NamedKey::Quota::<&[u8]>(account_id), script_size as i64) + .add(DirectoryValue::UsedQuota(account_id), script_size as i64) .blob(blob_id.hash.clone(), BlobOp::Link, 0) .custom(builder); sieve_ids.insert(document_id); @@ -215,7 +218,7 @@ impl JMAP { std::cmp::Ordering::Equal => 0, }; if update_quota != 0 { - batch.add(NamedKey::Quota::<&[u8]>(account_id), update_quota); + batch.add(DirectoryValue::UsedQuota(account_id), update_quota); } // Update blobId @@ -393,7 +396,7 @@ impl JMAP { .value(Property::EmailIds, (), F_VALUE | F_CLEAR) .blob(blob_id.hash.clone(), BlobOp::Link, F_CLEAR) .add( - NamedKey::Quota::<&[u8]>(account_id), + DirectoryValue::UsedQuota(account_id), -(blob_id.section.as_ref().unwrap().size as i64), ) .custom(ObjectIndexBuilder::new(SCHEMA).with_current(obj)); diff --git a/crates/jmap/src/vacation/set.rs b/crates/jmap/src/vacation/set.rs index 20ed9f17..0ece164a 100644 --- a/crates/jmap/src/vacation/set.rs +++ b/crates/jmap/src/vacation/set.rs @@ -42,13 +42,16 @@ use jmap_proto::{ use mail_builder::MessageBuilder; use mail_parser::decoders::html::html_to_text; use store::{ - write::{assert::HashedValue, log::ChangeLogBuilder, BatchBuilder, BlobOp, F_CLEAR, F_VALUE}, + write::{ + assert::HashedValue, log::ChangeLogBuilder, BatchBuilder, BlobOp, DirectoryValue, F_CLEAR, + F_VALUE, + }, BlobClass, }; use crate::{ sieve::set::{ObjectBlobId, SCHEMA}, - NamedKey, JMAP, + JMAP, }; impl JMAP { @@ -286,10 +289,10 @@ impl JMAP { std::cmp::Ordering::Equal => 0, }; if quota != 0 { - batch.add(NamedKey::Quota::<&[u8]>(account_id), quota); + batch.add(DirectoryValue::UsedQuota(account_id), quota); } } else { - batch.add(NamedKey::Quota::<&[u8]>(account_id), script_size); + batch.add(DirectoryValue::UsedQuota(account_id), script_size); } }; diff --git a/crates/managesieve/src/op/putscript.rs b/crates/managesieve/src/op/putscript.rs index 87af5b4a..9ecac091 100644 --- a/crates/managesieve/src/op/putscript.rs +++ b/crates/managesieve/src/op/putscript.rs @@ -22,10 +22,7 @@ */ use imap_proto::receiver::Request; -use jmap::{ - sieve::set::{ObjectBlobId, SCHEMA}, - NamedKey, -}; +use jmap::sieve::set::{ObjectBlobId, SCHEMA}; use jmap_proto::{ object::{index::ObjectIndexBuilder, Object}, types::{blob::BlobId, collection::Collection, property::Property, value::Value}, @@ -33,7 +30,7 @@ use jmap_proto::{ use sieve::compiler::ErrorType; use store::{ query::Filter, - write::{assert::HashedValue, BatchBuilder, BlobOp, F_CLEAR}, + write::{assert::HashedValue, BatchBuilder, BlobOp, DirectoryValue, F_CLEAR}, BlobClass, }; use tokio::io::{AsyncRead, AsyncWrite}; @@ -142,7 +139,7 @@ impl Session { std::cmp::Ordering::Equal => 0, }; if update_quota != 0 { - batch.add(NamedKey::Quota::<&[u8]>(account_id), update_quota); + batch.add(DirectoryValue::UsedQuota(account_id), update_quota); } batch.custom( @@ -183,7 +180,7 @@ impl Session { .with_account_id(account_id) .with_collection(Collection::SieveScript) .create_document(document_id) - .add(NamedKey::Quota::<&[u8]>(account_id), script_size) + .add(DirectoryValue::UsedQuota(account_id), script_size) .blob(blob_id.hash.clone(), BlobOp::Link, 0) .custom( ObjectIndexBuilder::new(SCHEMA).with_changes( diff --git a/crates/smtp/src/config/queue.rs b/crates/smtp/src/config/queue.rs index 09b0470a..de950842 100644 --- a/crates/smtp/src/config/queue.rs +++ b/crates/smtp/src/config/queue.rs @@ -23,7 +23,7 @@ use std::time::Duration; -use directory::memory::MemoryDirectory; +use directory::backend::memory::MemoryDirectory; use mail_send::Credentials; use super::{ diff --git a/crates/smtp/src/core/management.rs b/crates/smtp/src/core/management.rs index 34c810e4..a9e0eb27 100644 --- a/crates/smtp/src/core/management.rs +++ b/crates/smtp/src/core/management.rs @@ -23,7 +23,7 @@ use std::{borrow::Cow, fmt::Display, net::IpAddr, sync::Arc, time::Instant}; -use directory::Type; +use directory::{QueryBy, Type}; use http_body_util::{combinators::BoxBody, BodyExt, Empty, Full}; use hyper::{ body::{self, Bytes}, @@ -255,7 +255,10 @@ impl SMTP { .queue .config .management_lookup - .authenticate(&Credentials::Plain { username, secret }) + .query(QueryBy::credentials(&Credentials::Plain { + username, + secret, + })) .await { Ok(Some(principal)) if principal.typ == Type::Superuser => { diff --git a/crates/smtp/src/inbound/auth.rs b/crates/smtp/src/inbound/auth.rs index b15db7fa..a3a6afee 100644 --- a/crates/smtp/src/inbound/auth.rs +++ b/crates/smtp/src/inbound/auth.rs @@ -21,6 +21,7 @@ * for more details. */ +use directory::QueryBy; use mail_parser::decoders::base64::base64_decode; use mail_send::Credentials; use smtp_proto::{IntoString, AUTH_LOGIN, AUTH_OAUTHBEARER, AUTH_PLAIN, AUTH_XOAUTH2}; @@ -180,8 +181,10 @@ impl Session { | Credentials::XOauth2 { username, .. } | Credentials::OAuthBearer { token: username } => username.to_string(), }; - if let Ok(is_authenticated) = - lookup.authenticate(&credentials).await.map(|r| r.is_some()) + if let Ok(is_authenticated) = lookup + .query(QueryBy::credentials(&credentials)) + .await + .map(|r| r.is_some()) { tracing::debug!( parent: &self.span, diff --git a/crates/store/src/backend/mysql/main.rs b/crates/store/src/backend/mysql/main.rs index aee2d503..e12ca6f6 100644 --- a/crates/store/src/backend/mysql/main.rs +++ b/crates/store/src/backend/mysql/main.rs @@ -26,7 +26,7 @@ use utils::config::utils::AsKey; use crate::{ SUBSPACE_BITMAPS, SUBSPACE_BLOBS, SUBSPACE_BLOB_DATA, SUBSPACE_COUNTERS, SUBSPACE_INDEXES, - SUBSPACE_INDEX_VALUES, SUBSPACE_LOGS, SUBSPACE_VALUES, + SUBSPACE_LOGS, SUBSPACE_VALUES, }; use super::MysqlStore; @@ -80,7 +80,7 @@ impl MysqlStore { pub(super) async fn create_tables(&self) -> crate::Result<()> { let mut conn = self.conn_pool.get_conn().await?; - for table in [SUBSPACE_VALUES, SUBSPACE_LOGS, SUBSPACE_INDEX_VALUES] { + for table in [SUBSPACE_VALUES, SUBSPACE_LOGS] { let table = char::from(table); conn.query_drop(&format!( "CREATE TABLE IF NOT EXISTS {table} ( diff --git a/crates/store/src/backend/postgres/main.rs b/crates/store/src/backend/postgres/main.rs index 0a4fb52a..91abcba9 100644 --- a/crates/store/src/backend/postgres/main.rs +++ b/crates/store/src/backend/postgres/main.rs @@ -23,8 +23,7 @@ use crate::{ backend::postgres::tls::MakeRustlsConnect, SUBSPACE_BITMAPS, SUBSPACE_BLOBS, - SUBSPACE_BLOB_DATA, SUBSPACE_COUNTERS, SUBSPACE_INDEXES, SUBSPACE_INDEX_VALUES, SUBSPACE_LOGS, - SUBSPACE_VALUES, + SUBSPACE_BLOB_DATA, SUBSPACE_COUNTERS, SUBSPACE_INDEXES, SUBSPACE_LOGS, SUBSPACE_VALUES, }; use super::PostgresStore; @@ -75,12 +74,7 @@ impl PostgresStore { pub(super) async fn create_tables(&self) -> crate::Result<()> { let conn = self.conn_pool.get().await?; - for table in [ - SUBSPACE_VALUES, - SUBSPACE_LOGS, - SUBSPACE_INDEX_VALUES, - SUBSPACE_BLOB_DATA, - ] { + for table in [SUBSPACE_VALUES, SUBSPACE_LOGS, SUBSPACE_BLOB_DATA] { let table = char::from(table); conn.execute( &format!( diff --git a/crates/store/src/backend/rocksdb/main.rs b/crates/store/src/backend/rocksdb/main.rs index 5eb34e10..75836006 100644 --- a/crates/store/src/backend/rocksdb/main.rs +++ b/crates/store/src/backend/rocksdb/main.rs @@ -38,8 +38,7 @@ use utils::{ use crate::{Deserialize, Error}; use super::{ - RocksDbStore, CF_BITMAPS, CF_BLOBS, CF_BLOB_DATA, CF_COUNTERS, CF_INDEXES, CF_INDEX_VALUES, - CF_LOGS, CF_VALUES, + RocksDbStore, CF_BITMAPS, CF_BLOBS, CF_BLOB_DATA, CF_COUNTERS, CF_INDEXES, CF_LOGS, CF_VALUES, }; impl RocksDbStore { @@ -80,7 +79,7 @@ impl RocksDbStore { cfs.push(ColumnFamilyDescriptor::new(CF_BLOB_DATA, cf_opts)); // Other cfs - for cf in [CF_BLOBS, CF_INDEXES, CF_INDEX_VALUES, CF_LOGS, CF_VALUES] { + for cf in [CF_BLOBS, CF_INDEXES, CF_LOGS, CF_VALUES] { let cf_opts = Options::default(); cfs.push(ColumnFamilyDescriptor::new(cf, cf_opts)); } diff --git a/crates/store/src/backend/rocksdb/mod.rs b/crates/store/src/backend/rocksdb/mod.rs index 5295f928..9dd3cc5e 100644 --- a/crates/store/src/backend/rocksdb/mod.rs +++ b/crates/store/src/backend/rocksdb/mod.rs @@ -27,7 +27,7 @@ use rocksdb::{MultiThreaded, OptimisticTransactionDB}; use crate::{ SUBSPACE_BITMAPS, SUBSPACE_BLOBS, SUBSPACE_BLOB_DATA, SUBSPACE_COUNTERS, SUBSPACE_INDEXES, - SUBSPACE_INDEX_VALUES, SUBSPACE_LOGS, SUBSPACE_VALUES, + SUBSPACE_LOGS, SUBSPACE_VALUES, }; pub mod bitmap; @@ -42,7 +42,6 @@ static CF_LOGS: &str = unsafe { std::str::from_utf8_unchecked(&[SUBSPACE_LOGS]) static CF_INDEXES: &str = unsafe { std::str::from_utf8_unchecked(&[SUBSPACE_INDEXES]) }; static CF_BLOBS: &str = unsafe { std::str::from_utf8_unchecked(&[SUBSPACE_BLOBS]) }; static CF_BLOB_DATA: &str = unsafe { std::str::from_utf8_unchecked(&[SUBSPACE_BLOB_DATA]) }; -static CF_INDEX_VALUES: &str = unsafe { std::str::from_utf8_unchecked(&[SUBSPACE_INDEX_VALUES]) }; static CF_COUNTERS: &str = unsafe { std::str::from_utf8_unchecked(&[SUBSPACE_COUNTERS]) }; impl From for crate::Error { diff --git a/crates/store/src/backend/rocksdb/write.rs b/crates/store/src/backend/rocksdb/write.rs index 62fb369a..fd003bf8 100644 --- a/crates/store/src/backend/rocksdb/write.rs +++ b/crates/store/src/backend/rocksdb/write.rs @@ -31,12 +31,11 @@ use rocksdb::{Direction, ErrorKind, IteratorMode}; use super::{ bitmap::{clear_bit, set_bit}, - RocksDbStore, CF_BITMAPS, CF_BLOBS, CF_COUNTERS, CF_INDEXES, CF_INDEX_VALUES, CF_LOGS, - CF_VALUES, + RocksDbStore, CF_BITMAPS, CF_BLOBS, CF_COUNTERS, CF_INDEXES, CF_LOGS, CF_VALUES, }; use crate::{ write::{Batch, Operation, ValueOp, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME}, - BitmapKey, BlobKey, IndexKey, Key, LogKey, ValueKey, SUBSPACE_INDEX_VALUES, SUBSPACE_VALUES, + BitmapKey, BlobKey, IndexKey, Key, LogKey, ValueKey, SUBSPACE_VALUES, }; impl RocksDbStore { @@ -52,7 +51,6 @@ impl RocksDbStore { let cf_indexes = db.cf_handle(CF_INDEXES).unwrap(); let cf_logs = db.cf_handle(CF_LOGS).unwrap(); let cf_blobs = db.cf_handle(CF_BLOBS).unwrap(); - let cf_index_values = db.cf_handle(CF_INDEX_VALUES).unwrap(); let cf_counters = db.cf_handle(CF_COUNTERS).unwrap(); loop { @@ -101,17 +99,12 @@ impl RocksDbStore { document_id, class, }; - let cf = match key.subspace() { - SUBSPACE_VALUES => &cf_values, - SUBSPACE_INDEX_VALUES => &cf_index_values, - _ => unreachable!(), - }; let key = key.serialize(false); if let ValueOp::Set(value) = op { - wb.put_cf(cf, &key, value); + wb.put_cf(&cf_values, &key, value); } else { - wb.delete_cf(cf, &key); + wb.delete_cf(&cf_values, &key); } } Operation::Index { field, key, set } => { @@ -187,14 +180,9 @@ impl RocksDbStore { document_id, class, }; - let cf = match key.subspace() { - SUBSPACE_VALUES => &cf_values, - SUBSPACE_INDEX_VALUES => &cf_index_values, - _ => unreachable!(), - }; let key = key.serialize(false); let matches = txn - .get_cf(cf, &key)? + .get_cf(&cf_values, &key)? .map(|value| assert_value.matches(&value)) .unwrap_or_else(|| assert_value.is_none()); if !matches { diff --git a/crates/store/src/backend/sqlite/main.rs b/crates/store/src/backend/sqlite/main.rs index e6829992..1a48cd4d 100644 --- a/crates/store/src/backend/sqlite/main.rs +++ b/crates/store/src/backend/sqlite/main.rs @@ -30,7 +30,7 @@ use utils::{ use crate::{ SUBSPACE_BITMAPS, SUBSPACE_BLOBS, SUBSPACE_BLOB_DATA, SUBSPACE_COUNTERS, SUBSPACE_INDEXES, - SUBSPACE_INDEX_VALUES, SUBSPACE_LOGS, SUBSPACE_VALUES, + SUBSPACE_LOGS, SUBSPACE_VALUES, }; use super::{pool::SqliteConnectionManager, SqliteStore}; @@ -75,12 +75,7 @@ impl SqliteStore { pub(super) fn create_tables(&self) -> crate::Result<()> { let conn = self.conn_pool.get()?; - for table in [ - SUBSPACE_VALUES, - SUBSPACE_LOGS, - SUBSPACE_INDEX_VALUES, - SUBSPACE_BLOB_DATA, - ] { + for table in [SUBSPACE_VALUES, SUBSPACE_LOGS, SUBSPACE_BLOB_DATA] { let table = char::from(table); conn.execute( &format!( diff --git a/crates/store/src/dispatch/lookup.rs b/crates/store/src/dispatch/lookup.rs index 696ce235..9587d9f2 100644 --- a/crates/store/src/dispatch/lookup.rs +++ b/crates/store/src/dispatch/lookup.rs @@ -65,7 +65,7 @@ impl LookupStore { LookupStore::Store(store) => { let (class, op) = match value { LookupValue::Value { value, expires } => ( - ValueClass::Key { key }, + ValueClass::Key(key), ValueOp::Set( KeySerializer::new(value.len() + U64_LEN) .write(if expires > 0 { @@ -77,7 +77,7 @@ impl LookupStore { .finalize(), ), ), - LookupValue::Counter { num } => (ValueClass::Key { key }, ValueOp::Add(num)), + LookupValue::Counter { num } => (ValueClass::Key(key), ValueOp::Add(num)), LookupValue::None => return Ok(()), }; @@ -102,7 +102,7 @@ impl LookupStore { account_id: 0, collection: 0, document_id: 0, - class: ValueClass::Key { key }, + class: ValueClass::Key(key), }) .await .map(|value| value.unwrap_or(LookupValue::None)), @@ -111,7 +111,7 @@ impl LookupStore { account_id: 0, collection: 0, document_id: 0, - class: ValueClass::Key { key }, + class: ValueClass::Key(key), }) .await .map(|num| LookupValue::Counter { num }), @@ -129,15 +129,13 @@ impl LookupStore { account_id: 0, collection: 0, document_id: 0, - class: ValueClass::Key { key: vec![0u8] }, + class: ValueClass::Key(vec![0u8]), }; let to_key = ValueKey { account_id: 0, collection: 0, document_id: 0, - class: ValueClass::Key { - key: vec![u8::MAX; 10], - }, + class: ValueClass::Key(vec![u8::MAX; 10]), }; let current_time = now(); @@ -154,7 +152,7 @@ impl LookupStore { let mut batch = BatchBuilder::new(); for key in expired_keys { batch.ops.push(Operation::Value { - class: ValueClass::Key { key }, + class: ValueClass::Key(key), op: ValueOp::Clear, }); if batch.ops.len() >= 1000 { @@ -168,7 +166,7 @@ impl LookupStore { } } #[cfg(feature = "redis")] - LookupStore::Redis(store) => {} + LookupStore::Redis(_) => {} LookupStore::Memory(_) => {} } diff --git a/crates/store/src/dispatch/store.rs b/crates/store/src/dispatch/store.rs index f7d8a890..64bb7f0e 100644 --- a/crates/store/src/dispatch/store.rs +++ b/crates/store/src/dispatch/store.rs @@ -28,7 +28,7 @@ use roaring::RoaringBitmap; use crate::{ write::{key::KeySerializer, AnyKey, Batch, BitmapClass, ValueClass}, BitmapKey, Deserialize, IterateParams, Key, Store, ValueKey, SUBSPACE_BITMAPS, - SUBSPACE_INDEXES, SUBSPACE_INDEX_VALUES, SUBSPACE_LOGS, SUBSPACE_VALUES, U32_LEN, + SUBSPACE_INDEXES, SUBSPACE_LOGS, SUBSPACE_VALUES, U32_LEN, }; impl Store { @@ -185,12 +185,7 @@ impl Store { } pub async fn purge_account(&self, account_id: u32) -> crate::Result<()> { - for subspace in [ - SUBSPACE_BITMAPS, - SUBSPACE_VALUES, - SUBSPACE_LOGS, - SUBSPACE_INDEXES, - ] { + for subspace in [SUBSPACE_BITMAPS, SUBSPACE_LOGS, SUBSPACE_INDEXES] { self.delete_range( AnyKey { subspace, @@ -204,37 +199,27 @@ impl Store { .await?; } - for (from_key, to_key) in [ - ( - ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::Acl(account_id), - }, - ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::Acl(account_id + 1), - }, - ), - ( + for (from_class, to_class) in [ + (ValueClass::Acl(account_id), ValueClass::Acl(account_id + 1)), + (ValueClass::ReservedId, ValueClass::ReservedId), + (ValueClass::Property(0), ValueClass::Property(u8::MAX)), + (ValueClass::TermIndex, ValueClass::TermIndex), + ] { + self.delete_range( ValueKey { account_id, collection: 0, document_id: 0, - class: ValueClass::ReservedId, + class: from_class, }, ValueKey { account_id: account_id + 1, collection: 0, document_id: 0, - class: ValueClass::ReservedId, + class: to_class, }, - ), - ] { - self.delete_range(from_key, to_key).await?; + ) + .await?; } Ok(()) @@ -295,7 +280,6 @@ impl Store { SUBSPACE_BITMAPS, SUBSPACE_INDEXES, SUBSPACE_BLOBS, - SUBSPACE_INDEX_VALUES, SUBSPACE_COUNTERS, SUBSPACE_BLOB_DATA, ] { @@ -389,7 +373,6 @@ impl Store { for (subspace, with_values) in [ (SUBSPACE_VALUES, true), - (SUBSPACE_INDEX_VALUES, true), (SUBSPACE_COUNTERS, false), (SUBSPACE_BLOB_DATA, true), (SUBSPACE_BITMAPS, false), @@ -434,12 +417,9 @@ impl Store { value ); } - SUBSPACE_INDEX_VALUES if key[0] >= 3 => { - // Ignore named keys - return Ok(true); - } SUBSPACE_VALUES - if key.get(0..4).unwrap_or_default() == u32::MAX.to_be_bytes() => + if key[0] >= 6 + || key.get(1..5).unwrap_or_default() == u32::MAX.to_be_bytes() => { // Ignore lastId counter and ID mappings return Ok(true); diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index e88ad5ae..adbf59ab 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -180,7 +180,6 @@ pub const SUBSPACE_LOGS: u8 = b'l'; pub const SUBSPACE_INDEXES: u8 = b'i'; pub const SUBSPACE_BLOBS: u8 = b'o'; pub const SUBSPACE_BLOB_DATA: u8 = b't'; -pub const SUBSPACE_INDEX_VALUES: u8 = b'a'; pub const SUBSPACE_COUNTERS: u8 = b'c'; pub struct IterateParams { @@ -601,6 +600,22 @@ impl From for Vec { } } +impl From for Vec { + fn from(value: Row) -> Self { + value + .values + .into_iter() + .filter_map(|v| { + if let Value::Integer(v) = v { + Some(v as u32) + } else { + None + } + }) + .collect() + } +} + impl From for Vec { fn from(value: Rows) -> Self { value @@ -610,3 +625,21 @@ impl From for Vec { .collect() } } + +impl From for Vec { + fn from(value: Rows) -> Self { + value + .rows + .into_iter() + .flat_map(|v| { + v.values.into_iter().filter_map(|v| { + if let Value::Integer(v) = v { + Some(v as u32) + } else { + None + } + }) + }) + .collect() + } +} diff --git a/crates/store/src/write/key.rs b/crates/store/src/write/key.rs index 50a32595..5cace1ad 100644 --- a/crates/store/src/write/key.rs +++ b/crates/store/src/write/key.rs @@ -26,11 +26,10 @@ use utils::codec::leb128::Leb128_; use crate::{ BitmapKey, BlobHash, BlobKey, IndexKey, IndexKeyPrefix, Key, LogKey, ValueKey, BLOB_HASH_LEN, - SUBSPACE_BITMAPS, SUBSPACE_INDEXES, SUBSPACE_INDEX_VALUES, SUBSPACE_LOGS, SUBSPACE_VALUES, - U32_LEN, U64_LEN, + SUBSPACE_BITMAPS, SUBSPACE_INDEXES, SUBSPACE_LOGS, SUBSPACE_VALUES, U32_LEN, U64_LEN, }; -use super::{AnyKey, BitmapClass, BlobOp, TagValue, ValueClass}; +use super::{AnyKey, BitmapClass, BlobOp, DirectoryValue, TagValue, ValueClass}; pub struct KeySerializer { pub buf: Vec, @@ -217,69 +216,51 @@ impl Key for LogKey { impl + Sync + Send> Key for ValueKey { fn subspace(&self) -> u8 { - if matches!( - self.class.as_ref(), - ValueClass::Property(_) | ValueClass::TermIndex - ) { - SUBSPACE_VALUES - } else { - SUBSPACE_INDEX_VALUES - } + SUBSPACE_VALUES } fn serialize(&self, include_subspace: bool) -> Vec { + let serializer = if include_subspace { + KeySerializer::new(self.class.as_ref().serialized_size() + 2).write(self.subspace()) + } else { + KeySerializer::new(self.class.as_ref().serialized_size() + 1) + }; + match self.class.as_ref() { - ValueClass::Property(field) => if include_subspace { - KeySerializer::new(U32_LEN * 2 + 3).write(crate::SUBSPACE_VALUES) - } else { - KeySerializer::new(U32_LEN * 2 + 2) - } - .write(self.account_id) - .write(self.collection) - .write(self.document_id) - .write(*field), - ValueClass::TermIndex => if include_subspace { - KeySerializer::new(U32_LEN * 2 + 3).write(crate::SUBSPACE_VALUES) - } else { - KeySerializer::new(U32_LEN * 2 + 2) - } - .write(self.account_id) - .write(self.collection) - .write(self.document_id) - .write(u8::MAX), - ValueClass::Acl(grant_account_id) => if include_subspace { - KeySerializer::new(U32_LEN * 3 + 3).write(crate::SUBSPACE_INDEX_VALUES) - } else { - KeySerializer::new(U32_LEN * 3 + 2) - } - .write(0u8) - .write(*grant_account_id) - .write(self.account_id) - .write(self.collection) - .write(self.document_id), - ValueClass::ReservedId => if include_subspace { - KeySerializer::new(U32_LEN * 2 + 2).write(crate::SUBSPACE_INDEX_VALUES) - } else { - KeySerializer::new(U32_LEN * 2 + 1) - } - .write(1u8) - .write(self.account_id) - .write(self.collection) - .write(self.document_id), - ValueClass::Key { key } => if include_subspace { - KeySerializer::new(key.len() + U64_LEN + 2).write(crate::SUBSPACE_INDEX_VALUES) - } else { - KeySerializer::new(key.len() + U64_LEN + 1) - } - .write(2u8) - .write(key.as_slice()), - ValueClass::Subspace { key, id } => if include_subspace { - KeySerializer::new(key.len() + 2).write(crate::SUBSPACE_INDEX_VALUES) - } else { - KeySerializer::new(key.len() + 1) - } - .write(3 + *id) - .write(key.as_slice()), + ValueClass::Property(field) => serializer + .write(0u8) + .write(self.account_id) + .write(self.collection) + .write_leb128(self.document_id) + .write(*field), + ValueClass::TermIndex => serializer + .write(1u8) + .write(self.account_id) + .write(self.collection) + .write_leb128(self.document_id), + ValueClass::Acl(grant_account_id) => serializer + .write(2u8) + .write(*grant_account_id) + .write(self.account_id) + .write(self.collection) + .write(self.document_id), + ValueClass::ReservedId => serializer + .write(3u8) + .write(self.account_id) + .write(self.collection) + .write(self.document_id), + ValueClass::Key(key) => serializer.write(4u8).write(key.as_slice()), + ValueClass::IndexEmail(seq) => serializer + .write(5u8) + .write(*seq) + .write(self.account_id) + .write(self.document_id), + ValueClass::Directory(directory) => match directory { + DirectoryValue::NameToId(name) => serializer.write(6u8).write(name.as_slice()), + DirectoryValue::EmailToId(email) => serializer.write(7u8).write(email.as_slice()), + DirectoryValue::Principal(uid) => serializer.write(8u8).write_leb128(*uid), + DirectoryValue::UsedQuota(uid) => serializer.write(9u8).write_leb128(*uid), + }, } .finalize() } @@ -434,3 +415,48 @@ impl + Sync + Send> Key for AnyKey { self.subspace } } + +impl ValueClass { + pub fn serialized_size(&self) -> usize { + match self { + ValueClass::Property(_) | ValueClass::TermIndex | ValueClass::ReservedId => { + U32_LEN * 2 + 3 + } + ValueClass::Acl(_) => U32_LEN * 3 + 2, + ValueClass::Key(v) => v.len(), + ValueClass::Directory(d) => match d { + DirectoryValue::NameToId(v) | DirectoryValue::EmailToId(v) => v.len(), + DirectoryValue::Principal(_) | DirectoryValue::UsedQuota(_) => U32_LEN, + }, + ValueClass::IndexEmail { .. } => U64_LEN * 2, + } + } +} + +impl From for ValueKey { + fn from(class: ValueClass) -> Self { + ValueKey { + account_id: 0, + collection: 0, + document_id: 0, + class, + } + } +} + +impl From for ValueKey { + fn from(value: DirectoryValue) -> Self { + ValueKey { + account_id: 0, + collection: 0, + document_id: 0, + class: ValueClass::Directory(value), + } + } +} + +impl From for ValueClass { + fn from(value: DirectoryValue) -> Self { + ValueClass::Directory(value) + } +} diff --git a/crates/store/src/write/mod.rs b/crates/store/src/write/mod.rs index b9ca6c1d..132596f1 100644 --- a/crates/store/src/write/mod.rs +++ b/crates/store/src/write/mod.rs @@ -138,10 +138,19 @@ pub enum TagValue { pub enum ValueClass { Property(u8), Acl(u32), - Subspace { key: Vec, id: u8 }, - Key { key: Vec }, + Key(Vec), TermIndex, ReservedId, + Directory(DirectoryValue), + IndexEmail(u64), +} + +#[derive(Debug, PartialEq, Clone, Eq, Hash)] +pub enum DirectoryValue { + NameToId(Vec), + EmailToId(Vec), + Principal(u32), + UsedQuota(u32), } #[derive(Debug, PartialEq, Eq, Hash, Default)] diff --git a/resources/config/directory/memory.toml b/resources/config/directory/memory.toml index 03b478c9..a2dbcf82 100644 --- a/resources/config/directory/memory.toml +++ b/resources/config/directory/memory.toml @@ -12,14 +12,14 @@ subaddressing = true #subaddressing = { map = "^([^.]+)\.([^.]+)@(.+)$", to = "${2}@${3}" } superuser-group = "superusers" -[[directory."default".users]] +[[directory."default".principals]] name = "admin" description = "Superuser" secret = "changeme" email = ["postmaster@%{DEFAULT_DOMAIN}%"] member-of = ["superusers"] -[[directory."default".users]] +[[directory."default".principals]] name = "john" description = "John Doe" secret = "12345" @@ -27,7 +27,7 @@ email = ["john@%{DEFAULT_DOMAIN}%", "jdoe@%{DEFAULT_DOMAIN}%", "john.doe@%{DEFAU email-list = ["info@%{DEFAULT_DOMAIN}%"] member-of = ["sales"] -[[directory."default".users]] +[[directory."default".principals]] name = "jane" description = "Jane Doe" secret = "abcde" @@ -35,7 +35,7 @@ email = ["jane@%{DEFAULT_DOMAIN}%", "jane.doe@%{DEFAULT_DOMAIN}%"] email-list = ["info@%{DEFAULT_DOMAIN}%"] member-of = ["sales", "support"] -[[directory."default".users]] +[[directory."default".principals]] name = "bill" description = "Bill Foobar" secret = "$2y$05$bvIG6Nmid91Mu9RcmmWZfO5HJIMCT8riNW0hEp8f6/FuA2/mHZFpe" @@ -43,11 +43,11 @@ quota = 50000000 email = ["bill@%{DEFAULT_DOMAIN}%", "bill.foobar@%{DEFAULT_DOMAIN}%"] email-list = ["info@%{DEFAULT_DOMAIN}%"] -[[directory."default".groups]] +[[directory."default".principals]] name = "sales" description = "Sales Team" -[[directory."default".groups]] +[[directory."default".principals]] name = "support" description = "Support Team" diff --git a/tests/src/directory/imap.rs b/tests/src/directory/imap.rs index f0651b27..3367b184 100644 --- a/tests/src/directory/imap.rs +++ b/tests/src/directory/imap.rs @@ -23,6 +23,7 @@ use std::sync::Arc; +use directory::QueryBy; use mail_parser::decoders::base64::base64_decode; use mail_send::Credentials; use tokio::{ @@ -78,7 +79,7 @@ async fn imap_directory() { assert_eq!( &LookupResult::from( handle - .authenticate(item.as_credentials()) + .query(QueryBy::credentials(item.as_credentials())) .await .unwrap() .is_some() @@ -98,7 +99,7 @@ async fn imap_directory() { tokio::spawn(async move { LookupResult::from( handle - .authenticate(item.as_credentials()) + .query(QueryBy::credentials(item.as_credentials())) .await .unwrap() .is_some(), diff --git a/tests/src/directory/ldap.rs b/tests/src/directory/ldap.rs index 04aa86f2..1b70b863 100644 --- a/tests/src/directory/ldap.rs +++ b/tests/src/directory/ldap.rs @@ -23,10 +23,10 @@ use std::fmt::Debug; -use directory::{Principal, Type}; +use directory::{backend::internal::manage::ManageDirectory, Principal, QueryBy, Type}; use mail_send::Credentials; -use crate::directory::parse_config; +use crate::directory::{map_account_ids, parse_config, IntoSortedPrincipal}; #[tokio::test] async fn ldap_directory() { @@ -41,36 +41,52 @@ async fn ldap_directory() { // Obtain directory handle let mut config = parse_config().await; let handle = config.directories.directories.remove("ldap").unwrap(); + let base_store = config.stores.stores.get("sqlite").unwrap(); // Test authentication assert_eq!( handle - .authenticate(&Credentials::Plain { - username: "john".to_string(), - secret: "12345".to_string() - }) + .query( + QueryBy::credentials(&Credentials::Plain { + username: "john".to_string(), + secret: "12345".to_string() + }) + .with_store(base_store) + ) .await .unwrap() - .unwrap(), + .unwrap() + .into_sorted(), Principal { + id: base_store.get_account_id("john").await.unwrap().unwrap(), name: "john".to_string(), description: "John Doe".to_string().into(), secrets: vec!["12345".to_string()], typ: Type::Individual, - member_of: vec!["sales".to_string()], + member_of: map_account_ids(base_store, vec!["sales"]).await, + emails: vec![ + "john@example.org".to_string(), + "john.doe@example.org".to_string() + ], ..Default::default() } + .into_sorted() ); assert_eq!( handle - .authenticate(&Credentials::Plain { - username: "bill".to_string(), - secret: "password".to_string() - }) + .query( + QueryBy::credentials(&Credentials::Plain { + username: "bill".to_string(), + secret: "password".to_string() + }) + .with_store(base_store) + ) .await .unwrap() - .unwrap(), + .unwrap() + .into_sorted(), Principal { + id: base_store.get_account_id("bill").await.unwrap().unwrap(), name: "bill".to_string(), description: "Bill Foobar".to_string().into(), secrets: vec![ @@ -78,37 +94,53 @@ async fn ldap_directory() { ], typ: Type::Individual, quota: 500000, + emails: vec!["bill@example.org".to_string(),], ..Default::default() } + .into_sorted() ); assert!(handle - .authenticate(&Credentials::Plain { - username: "bill".to_string(), - secret: "invalid".to_string() - }) + .query( + QueryBy::credentials(&Credentials::Plain { + username: "bill".to_string(), + secret: "invalid".to_string() + }) + .with_store(base_store) + ) .await .unwrap() .is_none()); // Get user by name - let mut principal = handle.principal("jane").await.unwrap().unwrap(); - principal.member_of.sort_unstable(); assert_eq!( - principal, + handle + .query(QueryBy::name("jane").with_store(base_store)) + .await + .unwrap() + .unwrap() + .into_sorted(), Principal { + id: base_store.get_account_id("jane").await.unwrap().unwrap(), name: "jane".to_string(), description: "Jane Doe".to_string().into(), typ: Type::Individual, secrets: vec!["abcde".to_string()], - member_of: vec!["sales".to_string(), "support".to_string()], + member_of: map_account_ids(base_store, vec!["sales", "support"]).await, + emails: vec!["jane@example.org".to_string(),], ..Default::default() } + .into_sorted() ); // Get group by name assert_eq!( - handle.principal("sales").await.unwrap().unwrap(), + handle + .query(QueryBy::name("sales").with_store(base_store)) + .await + .unwrap() + .unwrap(), Principal { + id: base_store.get_account_id("sales").await.unwrap().unwrap(), name: "sales".to_string(), description: "sales".to_string().into(), typ: Type::Group, @@ -116,52 +148,48 @@ async fn ldap_directory() { } ); - // Emails by id - compare_sorted( - handle.emails_by_name("john").await.unwrap(), - vec![ - "john@example.org".to_string(), - "john.doe@example.org".to_string(), - ], - ); - compare_sorted( - handle.emails_by_name("bill").await.unwrap(), - vec!["bill@example.org".to_string()], - ); - // Ids by email compare_sorted( - handle.names_by_email("jane@example.org").await.unwrap(), - vec!["jane".to_string()], + handle + .email_to_ids("jane@example.org", base_store) + .await + .unwrap(), + map_account_ids(base_store, vec!["jane"]).await, ); compare_sorted( handle - .names_by_email("jane+alias@example.org") + .email_to_ids("jane+alias@example.org", base_store) .await .unwrap(), - vec!["jane".to_string()], - ); - compare_sorted( - handle.names_by_email("info@example.org").await.unwrap(), - vec!["john".to_string(), "jane".to_string(), "bill".to_string()], + map_account_ids(base_store, vec!["jane"]).await, ); compare_sorted( handle - .names_by_email("info+alias@example.org") + .email_to_ids("info@example.org", base_store) .await .unwrap(), - vec!["john".to_string(), "jane".to_string(), "bill".to_string()], + map_account_ids(base_store, vec!["bill", "jane", "john"]).await, ); compare_sorted( - handle.names_by_email("unknown@example.org").await.unwrap(), - Vec::::new(), + handle + .email_to_ids("info+alias@example.org", base_store) + .await + .unwrap(), + map_account_ids(base_store, vec!["bill", "jane", "john"]).await, + ); + compare_sorted( + handle + .email_to_ids("unknown@example.org", base_store) + .await + .unwrap(), + Vec::::new(), ); assert_eq!( handle - .names_by_email("anything@catchall.org") + .email_to_ids("anything@catchall.org", base_store) .await .unwrap(), - vec!["robert".to_string()] + map_account_ids(base_store, vec!["robert"]).await ); // Domain validation diff --git a/tests/src/directory/mod.rs b/tests/src/directory/mod.rs index 5a0731f2..c9355760 100644 --- a/tests/src/directory/mod.rs +++ b/tests/src/directory/mod.rs @@ -27,13 +27,16 @@ pub mod smtp; pub mod sql; use ::smtp::core::Lookup; -use directory::{config::ConfigDirectory, AddressMapping, Directories}; +use directory::{ + backend::internal::manage::ManageDirectory, config::ConfigDirectory, AddressMapping, + Directories, Principal, +}; use mail_send::Credentials; use rustls::ServerConfig; use rustls_pemfile::{certs, pkcs8_private_keys}; use rustls_pki_types::PrivateKeyDer; use std::{borrow::Cow, io::BufReader, path::PathBuf, sync::Arc}; -use store::{config::ConfigStore, LookupStore, Stores}; +use store::{config::ConfigStore, LookupStore, Store, Stores}; use tokio_rustls::TlsAcceptor; use crate::store::TempDir; @@ -164,10 +167,6 @@ verify = "(&(|(objectClass=posixAccount)(objectClass=posixGroup))(|(mail=*?*)(gi expand = "(&(|(objectClass=posixAccount)(objectClass=posixGroup))(sn=?))" domains = "(&(|(objectClass=posixAccount)(objectClass=posixGroup))(|(mail=*@?)(givenName=*@?)(sn=*@?)))" -[directory."ldap".object-classes] -user = "posixAccount" -group = "posixGroup" - # Glauth does not support searchable custom attributes so # 'sn' and 'givenName' are used to search for aliases/lists. @@ -179,6 +178,7 @@ groups = ["memberOf", "otherGroups"] email = "mail" email-alias = "givenName" quota = "diskQuota" +type = "objectClass" ############################################################################## @@ -225,36 +225,41 @@ type = "memory" catch-all = true subaddressing = true -[[directory."local".users]] +[[directory."local".principals]] name = "john" +type = "individual" description = "John Doe" secret = "12345" email = ["john@example.org", "jdoe@example.org", "john.doe@example.org"] email-list = ["info@example.org"] member-of = ["sales"] -[[directory."local".users]] +[[directory."local".principals]] name = "jane" +type = "individual" description = "Jane Doe" secret = "abcde" email = "jane@example.org" email-list = ["info@example.org"] member-of = ["sales", "support"] -[[directory."local".users]] +[[directory."local".principals]] name = "bill" +type = "individual" description = "Bill Foobar" secret = "$2y$05$bvIG6Nmid91Mu9RcmmWZfO5HJIMCT8riNW0hEp8f6/FuA2/mHZFpe" quota = 500000 email = "bill@example.org" email-list = ["info@example.org"] -[[directory."local".groups]] +[[directory."local".principals]] name = "sales" +type = "group" description = "Sales Team" -[[directory."local".groups]] +[[directory."local".principals]] name = "support" +type = "group" description = "Support Team" "#; @@ -607,3 +612,23 @@ fn address_mappings() { ); } } + +async fn map_account_ids(store: &Store, names: Vec>) -> Vec { + let mut ids = Vec::with_capacity(names.len()); + for name in names { + ids.push(store.get_account_id(name.as_ref()).await.unwrap().unwrap()); + } + ids +} + +trait IntoSortedPrincipal: Sized { + fn into_sorted(self) -> Self; +} + +impl IntoSortedPrincipal for Principal { + fn into_sorted(mut self) -> Self { + self.member_of.sort_unstable(); + self.emails.sort_unstable(); + self + } +} diff --git a/tests/src/directory/smtp.rs b/tests/src/directory/smtp.rs index b5595d9f..e9dc1953 100644 --- a/tests/src/directory/smtp.rs +++ b/tests/src/directory/smtp.rs @@ -23,7 +23,7 @@ use std::sync::Arc; -use directory::DirectoryError; +use directory::{DirectoryError, QueryBy}; use mail_parser::decoders::base64::base64_decode; use mail_send::Credentials; use tokio::{ @@ -96,7 +96,12 @@ async fn smtp_directory() { for (item, expected) in &tests { let result: LookupResult = match item { Item::IsAccount(v) => handle.rcpt(v).await.unwrap().into(), - Item::Authenticate(v) => handle.authenticate(v).await.unwrap().is_some().into(), + Item::Authenticate(v) => handle + .query(QueryBy::credentials(v)) + .await + .unwrap() + .is_some() + .into(), Item::Verify(v) => match handle.vrfy(v).await { Ok(v) => v.into(), Err(DirectoryError::Unsupported) => LookupResult::False, @@ -123,7 +128,12 @@ async fn smtp_directory() { tokio::spawn(async move { let result: LookupResult = match &item { Item::IsAccount(v) => handle.rcpt(v).await.unwrap().into(), - Item::Authenticate(v) => handle.authenticate(v).await.unwrap().is_some().into(), + Item::Authenticate(v) => handle + .query(QueryBy::credentials(v)) + .await + .unwrap() + .is_some() + .into(), Item::Verify(v) => match handle.vrfy(v).await { Ok(v) => v.into(), Err(DirectoryError::Unsupported) => LookupResult::False, diff --git a/tests/src/directory/sql.rs b/tests/src/directory/sql.rs index 631bc193..a414a417 100644 --- a/tests/src/directory/sql.rs +++ b/tests/src/directory/sql.rs @@ -22,12 +22,12 @@ */ use ahash::AHashMap; -use directory::{Principal, Type}; +use directory::{backend::internal::manage::ManageDirectory, Principal, QueryBy, Type}; use mail_send::Credentials; use smtp::core::Lookup; use store::{LookupStore, Store}; -use crate::directory::parse_config; +use crate::directory::{map_account_ids, parse_config}; use super::DirectoryStore; @@ -57,6 +57,7 @@ async fn sql_directory() { let store = DirectoryStore { store: config.stores.lookup_stores.remove(directory_id).unwrap(), }; + let base_store = config.stores.stores.get(directory_id).unwrap(); // Create tables store.create_test_directory().await; @@ -132,32 +133,45 @@ async fn sql_directory() { // Test authentication assert_eq!( handle - .authenticate(&Credentials::Plain { - username: "john".to_string(), - secret: "12345".to_string() - }) + .query( + QueryBy::credentials(&Credentials::Plain { + username: "john".to_string(), + secret: "12345".to_string() + }) + .with_store(base_store) + ) .await .unwrap() .unwrap(), Principal { + id: base_store.get_account_id("john").await.unwrap().unwrap(), name: "john".to_string(), description: "John Doe".to_string().into(), secrets: vec!["12345".to_string()], typ: Type::Individual, - member_of: vec!["sales".to_string()], + member_of: map_account_ids(base_store, vec!["sales"]).await, + emails: vec![ + "john@example.org".to_string(), + "jdoe@example.org".to_string(), + "john.doe@example.org".to_string() + ], ..Default::default() } ); assert_eq!( handle - .authenticate(&Credentials::Plain { - username: "bill".to_string(), - secret: "password".to_string() - }) + .query( + QueryBy::credentials(&Credentials::Plain { + username: "bill".to_string(), + secret: "password".to_string() + }) + .with_store(base_store) + ) .await .unwrap() .unwrap(), Principal { + id: base_store.get_account_id("bill").await.unwrap().unwrap(), name: "bill".to_string(), description: "Bill Foobar".to_string().into(), secrets: vec![ @@ -165,35 +179,50 @@ async fn sql_directory() { ], typ: Type::Individual, quota: 500000, + emails: vec!["bill@example.org".to_string(),], ..Default::default() } ); assert!(handle - .authenticate(&Credentials::Plain { - username: "bill".to_string(), - secret: "invalid".to_string() - }) + .query( + QueryBy::credentials(&Credentials::Plain { + username: "bill".to_string(), + secret: "invalid".to_string() + }) + .with_store(base_store) + ) .await .unwrap() .is_none()); // Get user by name assert_eq!( - handle.principal("jane").await.unwrap().unwrap(), + handle + .query(QueryBy::name("jane").with_store(base_store)) + .await + .unwrap() + .unwrap(), Principal { + id: base_store.get_account_id("jane").await.unwrap().unwrap(), name: "jane".to_string(), description: "Jane Doe".to_string().into(), typ: Type::Individual, secrets: vec!["abcde".to_string()], - member_of: vec!["sales".to_string(), "support".to_string()], + member_of: map_account_ids(base_store, vec!["sales", "support"]).await, + emails: vec!["jane@example.org".to_string(),], ..Default::default() } ); // Get group by name assert_eq!( - handle.principal("sales").await.unwrap().unwrap(), + handle + .query(QueryBy::name("sales").with_store(base_store)) + .await + .unwrap() + .unwrap(), Principal { + id: base_store.get_account_id("sales").await.unwrap().unwrap(), name: "sales".to_string(), description: "Sales Team".to_string().into(), typ: Type::Group, @@ -201,53 +230,48 @@ async fn sql_directory() { } ); - // Emails by id - assert_eq!( - handle.emails_by_name("john").await.unwrap(), - vec![ - "john@example.org".to_string(), - "jdoe@example.org".to_string(), - "john.doe@example.org".to_string(), - ] - ); - assert_eq!( - handle.emails_by_name("bill").await.unwrap(), - vec!["bill@example.org".to_string(),] - ); - // Ids by email assert_eq!( - handle.names_by_email("jane@example.org").await.unwrap(), - vec!["jane".to_string()] - ); - assert_eq!( - handle.names_by_email("info@example.org").await.unwrap(), - vec!["bill".to_string(), "jane".to_string(), "john".to_string()] + handle + .email_to_ids("jane@example.org", base_store) + .await + .unwrap(), + map_account_ids(base_store, vec!["jane"]).await ); assert_eq!( handle - .names_by_email("jane+alias@example.org") + .email_to_ids("info@example.org", base_store) .await .unwrap(), - vec!["jane".to_string()] + map_account_ids(base_store, vec!["bill", "jane", "john"]).await ); assert_eq!( handle - .names_by_email("info+alias@example.org") + .email_to_ids("jane+alias@example.org", base_store) .await .unwrap(), - vec!["bill".to_string(), "jane".to_string(), "john".to_string()] - ); - assert_eq!( - handle.names_by_email("unknown@example.org").await.unwrap(), - Vec::::new() + map_account_ids(base_store, vec!["jane"]).await ); assert_eq!( handle - .names_by_email("anything@catchall.org") + .email_to_ids("info+alias@example.org", base_store) .await .unwrap(), - vec!["robert".to_string()] + map_account_ids(base_store, vec!["bill", "jane", "john"]).await + ); + assert_eq!( + handle + .email_to_ids("unknown@example.org", base_store) + .await + .unwrap(), + Vec::::new() + ); + assert_eq!( + handle + .email_to_ids("anything@catchall.org", base_store) + .await + .unwrap(), + map_account_ids(base_store, vec!["robert"]).await ); // Domain validation @@ -317,7 +341,7 @@ impl DirectoryStore { "CREATE TABLE emails (name TEXT NOT NULL, address TEXT NOT", " NULL, type TEXT, PRIMARY KEY (name, address))" ), - "INSERT INTO accounts (name, secret, type) VALUES ('admin', 'secret', 'individual')", + "INSERT INTO accounts (name, secret, type) VALUES ('admin', 'secret', 'admin')", ] { let query = if matches!(self.store, LookupStore::Store(Store::MySQL(_))) { query.replace("TEXT", "VARCHAR(255)") @@ -333,25 +357,35 @@ impl DirectoryStore { } pub async fn create_test_user(&self, login: &str, secret: &str, name: &str) { + let account_type = if login == "admin" { + "admin" + } else { + "individual" + }; self.store .query::( if matches!(self.store, LookupStore::Store(Store::PostgreSQL(_))) { concat!( "INSERT INTO accounts (name, secret, description, ", - "type, active) VALUES ($1, $2, $3, 'individual', true) ON CONFLICT (name) DO NOTHING" + "type, active) VALUES ($1, $2, $3, $4, true) ON CONFLICT (name) DO NOTHING" ) } else if matches!(self.store, LookupStore::Store(Store::MySQL(_))) { concat!( "INSERT IGNORE INTO accounts (name, secret, description, ", - "type, active) VALUES (?, ?, ?, 'individual', true)" + "type, active) VALUES (?, ?, ?, ?, true)" ) } else { concat!( "INSERT OR IGNORE INTO accounts (name, secret, description, ", - "type, active) VALUES (?, ?, ?, 'individual', true)" + "type, active) VALUES (?, ?, ?, ?, true)" ) }, - vec![login.into(), secret.into(), name.into()], + vec![ + login.into(), + secret.into(), + name.into(), + account_type.into(), + ], ) .await .unwrap(); diff --git a/tests/src/imap/mod.rs b/tests/src/imap/mod.rs index a3240a04..5d43654e 100644 --- a/tests/src/imap/mod.rs +++ b/tests/src/imap/mod.rs @@ -40,7 +40,7 @@ use std::{path::PathBuf, sync::Arc, time::Duration}; use ::managesieve::core::ManageSieveSessionManager; use ::store::config::ConfigStore; use ahash::AHashSet; -use directory::config::ConfigDirectory; +use directory::{backend::internal::manage::ManageDirectory, config::ConfigDirectory}; use imap::core::{ImapSessionManager, IMAP}; use imap_proto::ResponseType; use jmap::{api::JmapSessionManager, services::IPC_CHANNEL_BUFFER, JMAP}; @@ -337,7 +337,7 @@ async fn init_imap_tests(store_id: &str, delete_if_exists: bool) -> IMAPTest { } // Assign Id 0 to admin (required for some tests) - jmap.get_account_id("admin").await.unwrap(); + jmap.store.get_or_create_account_id("admin").await.unwrap(); IMAPTest { jmap, diff --git a/tests/src/jmap/auth_acl.rs b/tests/src/jmap/auth_acl.rs index 216ba95d..ad2690fd 100644 --- a/tests/src/jmap/auth_acl.rs +++ b/tests/src/jmap/auth_acl.rs @@ -21,6 +21,7 @@ * for more details. */ +use directory::backend::internal::manage::ManageDirectory; use jmap::mailbox::{INBOX_ID, TRASH_ID}; use jmap_client::{ core::{ @@ -64,22 +65,26 @@ pub async fn test(params: &mut JMAPTest) { .create_test_group_with_email("sales@example.com", "Sales Group") .await; let john_id: Id = server - .get_account_id("jdoe@example.com") + .store + .get_or_create_account_id("jdoe@example.com") .await .unwrap() .into(); let jane_id: Id = server - .get_account_id("jane.smith@example.com") + .store + .get_or_create_account_id("jane.smith@example.com") .await .unwrap() .into(); let bill_id: Id = server - .get_account_id("bill@example.com") + .store + .get_or_create_account_id("bill@example.com") .await .unwrap() .into(); let sales_id: Id = server - .get_account_id("sales@example.com") + .store + .get_or_create_account_id("sales@example.com") .await .unwrap() .into(); @@ -784,7 +789,7 @@ pub async fn test(params: &mut JMAPTest) { // Destroy test account data for id in [john_id, bill_id, jane_id, sales_id] { params.client.set_default_account_id(&id.to_string()); - destroy_all_mailboxes(¶ms.client).await; + destroy_all_mailboxes(params).await; } assert_is_empty(server).await; } diff --git a/tests/src/jmap/auth_limits.rs b/tests/src/jmap/auth_limits.rs index 1677a4b4..fe809fee 100644 --- a/tests/src/jmap/auth_limits.rs +++ b/tests/src/jmap/auth_limits.rs @@ -23,6 +23,7 @@ use std::{sync::Arc, time::Duration}; +use directory::backend::internal::manage::ManageDirectory; use jmap_client::{ client::{Client, Credentials}, core::set::{SetError, SetErrorType}, @@ -43,7 +44,14 @@ pub async fn test(params: &mut JMAPTest) { .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; - let account_id = Id::from(server.get_account_id("jdoe@example.com").await.unwrap()).to_string(); + let account_id = Id::from( + server + .store + .get_or_create_account_id("jdoe@example.com") + .await + .unwrap(), + ) + .to_string(); params .directory .link_test_address("jdoe@example.com", "john.doe@example.com", "alias") @@ -199,6 +207,6 @@ pub async fn test(params: &mut JMAPTest) { // Destroy test accounts params.client.set_default_account_id(&account_id); - destroy_all_mailboxes(¶ms.client).await; + destroy_all_mailboxes(params).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/auth_oauth.rs b/tests/src/jmap/auth_oauth.rs index beed2597..c1038003 100644 --- a/tests/src/jmap/auth_oauth.rs +++ b/tests/src/jmap/auth_oauth.rs @@ -24,6 +24,7 @@ use std::time::{Duration, Instant}; use bytes::Bytes; +use directory::backend::internal::manage::ManageDirectory; use jmap::auth::oauth::{DeviceAuthResponse, ErrorType, OAuthMetadata, TokenResponse}; use jmap_client::{ client::{Client, Credentials}, @@ -47,7 +48,14 @@ pub async fn test(params: &mut JMAPTest) { .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; - let john_id = Id::from(server.get_account_id("jdoe@example.com").await.unwrap()).to_string(); + let john_id = Id::from( + server + .store + .get_or_create_account_id("jdoe@example.com") + .await + .unwrap(), + ) + .to_string(); // Obtain OAuth metadata let metadata: OAuthMetadata = @@ -305,7 +313,7 @@ pub async fn test(params: &mut JMAPTest) { // Destroy test accounts params.client.set_default_account_id(john_id); - destroy_all_mailboxes(¶ms.client).await; + destroy_all_mailboxes(params).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/blob.rs b/tests/src/jmap/blob.rs index 0fa691b7..490826ca 100644 --- a/tests/src/jmap/blob.rs +++ b/tests/src/jmap/blob.rs @@ -21,6 +21,7 @@ * for more details. */ +use directory::backend::internal::manage::ManageDirectory; use jmap::mailbox::INBOX_ID; use jmap_proto::types::id::Id; use serde_json::Value; @@ -36,7 +37,13 @@ pub async fn test(params: &mut JMAPTest) { .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; - let account_id = Id::from(server.get_account_id("jdoe@example.com").await.unwrap()); + let account_id = Id::from( + server + .store + .get_or_create_account_id("jdoe@example.com") + .await + .unwrap(), + ); server.store.blob_hash_expire_all().await; @@ -488,6 +495,6 @@ pub async fn test(params: &mut JMAPTest) { // Remove test data params.client.set_default_account_id(account_id.to_string()); - destroy_all_mailboxes(¶ms.client).await; + destroy_all_mailboxes(params).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/crypto.rs b/tests/src/jmap/crypto.rs index 2c29f844..09e7a0a1 100644 --- a/tests/src/jmap/crypto.rs +++ b/tests/src/jmap/crypto.rs @@ -24,6 +24,7 @@ use std::{path::PathBuf, time::Duration}; use ahash::AHashMap; +use directory::backend::internal::manage::ManageDirectory; use jmap::email::crypto::{ try_parse_certs, Algorithm, EncryptMessage, EncryptionMethod, EncryptionParams, }; @@ -44,7 +45,14 @@ pub async fn test(params: &mut JMAPTest) { .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; - let account_id = Id::from(server.get_account_id("jdoe@example.com").await.unwrap()).to_string(); + let account_id = Id::from( + server + .store + .get_or_create_account_id("jdoe@example.com") + .await + .unwrap(), + ) + .to_string(); // Update let mut params = AHashMap::from_iter([ diff --git a/tests/src/jmap/delivery.rs b/tests/src/jmap/delivery.rs index 6445c302..b8fcee7e 100644 --- a/tests/src/jmap/delivery.rs +++ b/tests/src/jmap/delivery.rs @@ -23,6 +23,7 @@ use std::time::Duration; +use directory::backend::internal::manage::ManageDirectory; use jmap_proto::types::{collection::Collection, id::Id}; use tokio::{ @@ -51,12 +52,30 @@ pub async fn test(params: &mut JMAPTest) { .directory .create_test_user_with_email("bill@example.com", "098765", "Bill Foobar") .await; - let account_id_1 = - Id::from(server.get_account_id("jdoe@example.com").await.unwrap()).to_string(); - let account_id_2 = - Id::from(server.get_account_id("jane@example.com").await.unwrap()).to_string(); - let account_id_3 = - Id::from(server.get_account_id("bill@example.com").await.unwrap()).to_string(); + let account_id_1 = Id::from( + server + .store + .get_or_create_account_id("jdoe@example.com") + .await + .unwrap(), + ) + .to_string(); + let account_id_2 = Id::from( + server + .store + .get_or_create_account_id("jane@example.com") + .await + .unwrap(), + ) + .to_string(); + let account_id_3 = Id::from( + server + .store + .get_or_create_account_id("bill@example.com") + .await + .unwrap(), + ) + .to_string(); params .directory .link_test_address("jdoe@example.com", "john.doe@example.com", "alias") @@ -261,7 +280,7 @@ pub async fn test(params: &mut JMAPTest) { // Remove test data for account_id in [&account_id_1, &account_id_2, &account_id_3] { params.client.set_default_account_id(account_id); - destroy_all_mailboxes(¶ms.client).await; + destroy_all_mailboxes(params).await; } assert_is_empty(server).await; } diff --git a/tests/src/jmap/email_copy.rs b/tests/src/jmap/email_copy.rs index e8c6e5ff..84d3a99e 100644 --- a/tests/src/jmap/email_copy.rs +++ b/tests/src/jmap/email_copy.rs @@ -118,8 +118,8 @@ pub async fn test(params: &mut JMAPTest) { .is_none()); // Empty store - destroy_all_mailboxes(¶ms.client).await; + destroy_all_mailboxes(params).await; params.client.set_default_account_id(Id::new(2).to_string()); - destroy_all_mailboxes(¶ms.client).await; + destroy_all_mailboxes(params).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/email_get.rs b/tests/src/jmap/email_get.rs index cb9aaf38..c10653b8 100644 --- a/tests/src/jmap/email_get.rs +++ b/tests/src/jmap/email_get.rs @@ -188,7 +188,7 @@ pub async fn test(params: &mut JMAPTest) { } } - destroy_all_mailboxes(¶ms.client).await; + destroy_all_mailboxes(params).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/email_parse.rs b/tests/src/jmap/email_parse.rs index 93b4d789..ecd829ad 100644 --- a/tests/src/jmap/email_parse.rs +++ b/tests/src/jmap/email_parse.rs @@ -252,6 +252,6 @@ pub async fn test(params: &mut JMAPTest) { panic!("Test failed, output saved to {}", test_file.display()); } - destroy_all_mailboxes(¶ms.client).await; + destroy_all_mailboxes(params).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/email_query.rs b/tests/src/jmap/email_query.rs index 6240ca7b..bbd0a3a0 100644 --- a/tests/src/jmap/email_query.rs +++ b/tests/src/jmap/email_query.rs @@ -120,7 +120,7 @@ pub async fn test(params: &mut JMAPTest, insert: bool) { .unwrap_set_email() .unwrap(); - destroy_all_mailboxes(¶ms.client).await; + destroy_all_mailboxes(params).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/email_query_changes.rs b/tests/src/jmap/email_query_changes.rs index 4786d65b..9935fc8f 100644 --- a/tests/src/jmap/email_query_changes.rs +++ b/tests/src/jmap/email_query_changes.rs @@ -276,7 +276,7 @@ pub async fn test(params: &mut JMAPTest) { states.push(new_state); } - destroy_all_mailboxes(¶ms.client).await; + destroy_all_mailboxes(params).await; // Delete virtual threads let mut batch = BatchBuilder::new(); diff --git a/tests/src/jmap/email_search_snippet.rs b/tests/src/jmap/email_search_snippet.rs index 42c09edf..f3b88e4d 100644 --- a/tests/src/jmap/email_search_snippet.rs +++ b/tests/src/jmap/email_search_snippet.rs @@ -183,6 +183,6 @@ pub async fn test(params: &mut JMAPTest) { } // Destroy test data - destroy_all_mailboxes(¶ms.client).await; + destroy_all_mailboxes(params).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/email_set.rs b/tests/src/jmap/email_set.rs index fde4f2c8..024d1413 100644 --- a/tests/src/jmap/email_set.rs +++ b/tests/src/jmap/email_set.rs @@ -46,7 +46,7 @@ pub async fn test(params: &mut JMAPTest) { create(&mut params.client, &mailbox_id).await; update(&mut params.client, &mailbox_id).await; - destroy_all_mailboxes(¶ms.client).await; + destroy_all_mailboxes(params).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/email_submission.rs b/tests/src/jmap/email_submission.rs index be96c292..f2ce9f49 100644 --- a/tests/src/jmap/email_submission.rs +++ b/tests/src/jmap/email_submission.rs @@ -22,6 +22,7 @@ */ use ahash::AHashMap; +use directory::backend::internal::manage::ManageDirectory; use jmap_client::{ core::set::{SetError, SetErrorType, SetObject}, email_submission::{query::Filter, Address, Delivered, DeliveryStatus, Displayed, UndoStatus}, @@ -96,7 +97,14 @@ pub async fn test(params: &mut JMAPTest) { .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; - let account_id = Id::from(server.get_account_id("jdoe@example.com").await.unwrap()).to_string(); + let account_id = Id::from( + server + .store + .get_or_create_account_id("jdoe@example.com") + .await + .unwrap(), + ) + .to_string(); // Create an identity without using a valid address should fail match client @@ -474,7 +482,7 @@ pub async fn test(params: &mut JMAPTest) { { client.email_submission_destroy(&id).await.unwrap(); } - destroy_all_mailboxes(¶ms.client).await; + destroy_all_mailboxes(params).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/event_source.rs b/tests/src/jmap/event_source.rs index c31f7858..7173a705 100644 --- a/tests/src/jmap/event_source.rs +++ b/tests/src/jmap/event_source.rs @@ -26,6 +26,7 @@ use std::time::Duration; use crate::jmap::{ assert_is_empty, delivery::SmtpConnection, mailbox::destroy_all_mailboxes, test_account_login, }; +use directory::backend::internal::manage::ManageDirectory; use futures::StreamExt; use jmap::mailbox::INBOX_ID; use jmap_client::{event_source::Changes, mailbox::Role, TypeState}; @@ -45,7 +46,14 @@ pub async fn test(params: &mut JMAPTest) { .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; - let account_id = Id::from(server.get_account_id("jdoe@example.com").await.unwrap()).to_string(); + let account_id = Id::from( + server + .store + .get_or_create_account_id("jdoe@example.com") + .await + .unwrap(), + ) + .to_string(); let client = test_account_login("jdoe@example.com", "12345").await; let mut changes = client @@ -136,7 +144,7 @@ pub async fn test(params: &mut JMAPTest) { assert_ping(&mut event_rx).await; assert_ping(&mut event_rx).await; - destroy_all_mailboxes(¶ms.client).await; + destroy_all_mailboxes(params).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/mailbox.rs b/tests/src/jmap/mailbox.rs index b7c60cdd..b92606c0 100644 --- a/tests/src/jmap/mailbox.rs +++ b/tests/src/jmap/mailbox.rs @@ -36,7 +36,7 @@ use store::ahash::AHashMap; use crate::jmap::assert_is_empty; -use super::JMAPTest; +use super::{wait_for_index, JMAPTest}; pub async fn test(params: &mut JMAPTest) { println!("Running Mailbox tests..."); @@ -607,8 +607,8 @@ pub async fn test(params: &mut JMAPTest) { ["inbox", "sent", "spam"] ); - destroy_all_mailboxes(client).await; - client.set_default_account_id(Id::from(1u64)); + destroy_all_mailboxes(params).await; + params.client.set_default_account_id(Id::from(1u64)); assert_is_empty(server).await; } @@ -657,7 +657,12 @@ fn build_create_query( } } -pub async fn destroy_all_mailboxes(client: &Client) { +pub async fn destroy_all_mailboxes(test: &JMAPTest) { + wait_for_index(&test.server).await; + destroy_all_mailboxes_no_wait(&test.client).await; +} + +pub async fn destroy_all_mailboxes_no_wait(client: &Client) { let mut request = client.build(); request.query_mailbox().arguments().sort_as_tree(true); let mut ids = request.send_query_mailbox().await.unwrap().take_ids(); diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 45c86047..23b6380c 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -282,7 +282,7 @@ pub async fn jmap_tests() { .await; let coco = 1; //email_query::test(&mut params, delete).await; - /*email_get::test(&mut params).await; + //email_get::test(&mut params).await; email_set::test(&mut params).await; email_parse::test(&mut params).await; email_search_snippet::test(&mut params).await; @@ -291,7 +291,7 @@ pub async fn jmap_tests() { email_copy::test(&mut params).await; thread_get::test(&mut params).await; thread_merge::test(&mut params).await; - mailbox::test(&mut params).await;*/ + mailbox::test(&mut params).await; delivery::test(&mut params).await; auth_acl::test(&mut params).await; auth_limits::test(&mut params).await; @@ -404,7 +404,6 @@ async fn init_jmap_tests(store_id: &str, delete_if_exists: bool) -> JMAPTest { directory .create_test_user("admin", "secret", "Superuser") .await; - directory.add_to_group("admin", "superusers").await; if delete_if_exists { jmap.store.destroy().await; diff --git a/tests/src/jmap/push_subscription.rs b/tests/src/jmap/push_subscription.rs index 38a1c47e..2c0c1529 100644 --- a/tests/src/jmap/push_subscription.rs +++ b/tests/src/jmap/push_subscription.rs @@ -30,6 +30,7 @@ use std::{ }; use base64::{engine::general_purpose, Engine}; +use directory::backend::internal::manage::ManageDirectory; use ece::EcKeyComponents; use hyper::{body, header::CONTENT_ENCODING, server::conn::http1, service::service_fn, StatusCode}; use hyper_util::rt::TokioIo; @@ -86,7 +87,13 @@ pub async fn test(params: &mut JMAPTest) { .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; - let account_id = Id::from(server.get_account_id("jdoe@example.com").await.unwrap()); + let account_id = Id::from( + server + .store + .get_or_create_account_id("jdoe@example.com") + .await + .unwrap(), + ); params.client.set_default_account_id(account_id); let client = test_account_login("jdoe@example.com", "12345").await; @@ -219,7 +226,7 @@ pub async fn test(params: &mut JMAPTest) { client.mailbox_destroy(&mailbox_id, true).await.unwrap(); expect_nothing(&mut event_rx).await; - destroy_all_mailboxes(¶ms.client).await; + destroy_all_mailboxes(params).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/quota.rs b/tests/src/jmap/quota.rs index b270507a..98a29ead 100644 --- a/tests/src/jmap/quota.rs +++ b/tests/src/jmap/quota.rs @@ -25,6 +25,7 @@ use crate::jmap::{ assert_is_empty, delivery::SmtpConnection, jmap_raw_request, mailbox::destroy_all_mailboxes, test_account_login, }; +use directory::backend::internal::manage::ManageDirectory; use jmap::{blob::upload::DISABLE_UPLOAD_QUOTA, mailbox::INBOX_ID}; use jmap_client::{ core::set::{SetErrorType, SetObject}, @@ -45,8 +46,20 @@ pub async fn test(params: &mut JMAPTest) { .directory .create_test_user_with_email("robert@example.com", "aabbcc", "Robert Foobar") .await; - let other_account_id = Id::from(server.get_account_id("jdoe@example.com").await.unwrap()); - let account_id = Id::from(server.get_account_id("robert@example.com").await.unwrap()); + let other_account_id = Id::from( + server + .store + .get_or_create_account_id("jdoe@example.com") + .await + .unwrap(), + ); + let account_id = Id::from( + server + .store + .get_or_create_account_id("robert@example.com") + .await + .unwrap(), + ); params .directory .set_test_quota("robert@example.com", 1024) @@ -326,7 +339,7 @@ pub async fn test(params: &mut JMAPTest) { // Remove test data for account_id in [&account_id, &other_account_id] { params.client.set_default_account_id(account_id.to_string()); - destroy_all_mailboxes(¶ms.client).await; + destroy_all_mailboxes(params).await; } assert_is_empty(server).await; } diff --git a/tests/src/jmap/sieve_script.rs b/tests/src/jmap/sieve_script.rs index 56ad8348..adedeb61 100644 --- a/tests/src/jmap/sieve_script.rs +++ b/tests/src/jmap/sieve_script.rs @@ -21,6 +21,7 @@ * for more details. */ +use directory::backend::internal::manage::ManageDirectory; use jmap_client::{ core::set::{SetError, SetErrorType}, email, mailbox, @@ -53,7 +54,14 @@ pub async fn test(params: &mut JMAPTest) { .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; - let account_id = Id::from(server.get_account_id("jdoe@example.com").await.unwrap()).to_string(); + let account_id = Id::from( + server + .store + .get_or_create_account_id("jdoe@example.com") + .await + .unwrap(), + ) + .to_string(); client.set_default_account_id(&account_id); // Validate scripts @@ -486,7 +494,7 @@ pub async fn test(params: &mut JMAPTest) { for id in request.send_query_sieve_script().await.unwrap().take_ids() { client.sieve_script_destroy(&id).await.unwrap(); } - destroy_all_mailboxes(¶ms.client).await; + destroy_all_mailboxes(params).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/stress_test.rs b/tests/src/jmap/stress_test.rs index 24b7c7c3..9a13a4fa 100644 --- a/tests/src/jmap/stress_test.rs +++ b/tests/src/jmap/stress_test.rs @@ -23,7 +23,7 @@ use std::{sync::Arc, time::Duration}; -use crate::jmap::mailbox::destroy_all_mailboxes; +use crate::jmap::{mailbox::destroy_all_mailboxes_no_wait, wait_for_index}; use futures::future::join_all; use jmap::{mailbox::UidMailbox, JMAP}; use jmap_client::{ @@ -274,7 +274,8 @@ async fn email_tests(server: Arc, client: Arc) { } } - destroy_all_mailboxes(&client).await; + wait_for_index(&server).await; + destroy_all_mailboxes_no_wait(&client).await; assert_is_empty(server.clone()).await; } } @@ -353,7 +354,8 @@ async fn mailbox_tests(server: Arc, client: Arc) { join_all(futures).await; - destroy_all_mailboxes(&client).await; + wait_for_index(&server).await; + destroy_all_mailboxes_no_wait(&client).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/thread_get.rs b/tests/src/jmap/thread_get.rs index 2e8e42cc..6a89ffc5 100644 --- a/tests/src/jmap/thread_get.rs +++ b/tests/src/jmap/thread_get.rs @@ -68,6 +68,6 @@ pub async fn test(params: &mut JMAPTest) { expected_result ); - destroy_all_mailboxes(¶ms.client).await; + destroy_all_mailboxes(params).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/thread_merge.rs b/tests/src/jmap/thread_merge.rs index 34100a01..c719ba53 100644 --- a/tests/src/jmap/thread_merge.rs +++ b/tests/src/jmap/thread_merge.rs @@ -199,8 +199,10 @@ pub async fn test(params: &mut JMAPTest) { // Delete all messages and make sure no keys are left in the store. for (base_test_num, mailbox_ids) in all_mailboxes { for (test_num, _) in mailbox_ids.into_iter().enumerate() { - client.set_default_account_id(Id::new((base_test_num + test_num) as u64).to_string()); - destroy_all_mailboxes(client).await; + params + .client + .set_default_account_id(Id::new((base_test_num + test_num) as u64).to_string()); + destroy_all_mailboxes(params).await; } } diff --git a/tests/src/jmap/vacation_response.rs b/tests/src/jmap/vacation_response.rs index 7144b251..942e41b5 100644 --- a/tests/src/jmap/vacation_response.rs +++ b/tests/src/jmap/vacation_response.rs @@ -23,6 +23,7 @@ use chrono::{Duration, Utc}; +use directory::backend::internal::manage::ManageDirectory; use jmap_proto::types::id::Id; use std::time::Instant; @@ -47,7 +48,14 @@ pub async fn test(params: &mut JMAPTest) { .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; - let account_id = Id::from(server.get_account_id("jdoe@example.com").await.unwrap()).to_string(); + let account_id = Id::from( + server + .store + .get_or_create_account_id("jdoe@example.com") + .await + .unwrap(), + ) + .to_string(); client.set_default_account_id(&account_id); // Start mock SMTP server @@ -175,6 +183,6 @@ pub async fn test(params: &mut JMAPTest) { // Remove test data client.vacation_response_destroy().await.unwrap(); - destroy_all_mailboxes(¶ms.client).await; + destroy_all_mailboxes(params).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/websocket.rs b/tests/src/jmap/websocket.rs index 5fb12906..80ac2e39 100644 --- a/tests/src/jmap/websocket.rs +++ b/tests/src/jmap/websocket.rs @@ -22,6 +22,7 @@ */ use ahash::AHashSet; +use directory::backend::internal::manage::ManageDirectory; use futures::StreamExt; use jmap_client::{ client_ws::WebSocketMessage, @@ -49,7 +50,14 @@ pub async fn test(params: &mut JMAPTest) { .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; - let account_id = Id::from(server.get_account_id("jdoe@example.com").await.unwrap()).to_string(); + let account_id = Id::from( + server + .store + .get_or_create_account_id("jdoe@example.com") + .await + .unwrap(), + ) + .to_string(); let client = test_account_login("jdoe@example.com", "12345").await; let mut ws_stream = client.connect_ws().await.unwrap(); @@ -124,7 +132,7 @@ pub async fn test(params: &mut JMAPTest) { expect_nothing(&mut stream_rx).await; params.client.set_default_account_id(account_id); - destroy_all_mailboxes(¶ms.client).await; + destroy_all_mailboxes(params).await; assert_is_empty(server).await; } diff --git a/tests/src/smtp/inbound/auth.rs b/tests/src/smtp/inbound/auth.rs index a8fe7552..9510c7f5 100644 --- a/tests/src/smtp/inbound/auth.rs +++ b/tests/src/smtp/inbound/auth.rs @@ -39,7 +39,7 @@ const DIRECTORY: &str = r#" [directory."local"] type = "memory" -[[directory."local".users]] +[[directory."local".principals]] name = "john" description = "John Doe" secret = "secret" @@ -47,7 +47,7 @@ email = ["john@example.org", "jdoe@example.org", "john.doe@example.org"] email-list = ["info@example.org"] member-of = ["sales"] -[[directory."local".users]] +[[directory."local".principals]] name = "jane" description = "Jane Doe" secret = "p4ssw0rd" diff --git a/tests/src/smtp/inbound/data.rs b/tests/src/smtp/inbound/data.rs index 5d5e726e..91e9d3ee 100644 --- a/tests/src/smtp/inbound/data.rs +++ b/tests/src/smtp/inbound/data.rs @@ -39,25 +39,25 @@ const DIRECTORY: &str = r#" [directory."local"] type = "memory" -[[directory."local".users]] +[[directory."local".principals]] name = "john" description = "John Doe" secret = "secret" email = ["john@foobar.org", "jdoe@example.org", "john.doe@example.org"] -[[directory."local".users]] +[[directory."local".principals]] name = "jane" description = "Jane Doe" secret = "p4ssw0rd" email = "jane@domain.net" -[[directory."local".users]] +[[directory."local".principals]] name = "bill" description = "Bill Foobar" secret = "p4ssw0rd" email = "bill@foobar.org" -[[directory."local".users]] +[[directory."local".principals]] name = "mike" description = "Mike Foobar" secret = "p4ssw0rd" diff --git a/tests/src/smtp/inbound/dmarc.rs b/tests/src/smtp/inbound/dmarc.rs index f175ef24..2a8cafa5 100644 --- a/tests/src/smtp/inbound/dmarc.rs +++ b/tests/src/smtp/inbound/dmarc.rs @@ -53,7 +53,7 @@ const DIRECTORY: &str = r#" [directory."local"] type = "memory" -[[directory."local".users]] +[[directory."local".principals]] name = "john" description = "John Doe" secret = "secret" diff --git a/tests/src/smtp/inbound/rcpt.rs b/tests/src/smtp/inbound/rcpt.rs index 1805e27c..590e2adc 100644 --- a/tests/src/smtp/inbound/rcpt.rs +++ b/tests/src/smtp/inbound/rcpt.rs @@ -41,25 +41,25 @@ const DIRECTORY: &str = r#" [directory."local"] type = "memory" -[[directory."local".users]] +[[directory."local".principals]] name = "john" description = "John Doe" secret = "secret" email = "john@foobar.org" -[[directory."local".users]] +[[directory."local".principals]] name = "jane" description = "Jane Doe" secret = "p4ssw0rd" email = "jane@foobar.org" -[[directory."local".users]] +[[directory."local".principals]] name = "bill" description = "Bill Foobar" secret = "p4ssw0rd" email = "bill@foobar.org" -[[directory."local".users]] +[[directory."local".principals]] name = "mike" description = "Mike Foobar" secret = "p4ssw0rd" diff --git a/tests/src/smtp/inbound/sign.rs b/tests/src/smtp/inbound/sign.rs index 47135c18..158a3dcb 100644 --- a/tests/src/smtp/inbound/sign.rs +++ b/tests/src/smtp/inbound/sign.rs @@ -99,7 +99,7 @@ const DIRECTORY: &str = r#" [directory."local"] type = "memory" -[[directory."local".users]] +[[directory."local".principals]] name = "john" description = "John Doe" secret = "secret" diff --git a/tests/src/smtp/inbound/vrfy.rs b/tests/src/smtp/inbound/vrfy.rs index 12e6d5f9..610ff96a 100644 --- a/tests/src/smtp/inbound/vrfy.rs +++ b/tests/src/smtp/inbound/vrfy.rs @@ -38,21 +38,21 @@ const DIRECTORY: &str = r#" [directory."local"] type = "memory" -[[directory."local".users]] +[[directory."local".principals]] name = "john" description = "John Doe" secret = "secret" email = ["john@foobar.org"] email-list = ["sales@foobar.org"] -[[directory."local".users]] +[[directory."local".principals]] name = "jane" description = "Jane Doe" secret = "p4ssw0rd" email = "jane@foobar.org" email-list = ["sales@foobar.org"] -[[directory."local".users]] +[[directory."local".principals]] name = "bill" description = "Bill Foobar" secret = "p4ssw0rd" diff --git a/tests/src/smtp/management/queue.rs b/tests/src/smtp/management/queue.rs index eb820029..07150a94 100644 --- a/tests/src/smtp/management/queue.rs +++ b/tests/src/smtp/management/queue.rs @@ -54,7 +54,7 @@ type = "memory" [directory."local".options] superuser-group = "superusers" -[[directory."local".users]] +[[directory."local".principals]] name = "admin" description = "Superuser" secret = "secret" diff --git a/tests/src/smtp/management/report.rs b/tests/src/smtp/management/report.rs index 357059e8..b190a6bc 100644 --- a/tests/src/smtp/management/report.rs +++ b/tests/src/smtp/management/report.rs @@ -57,7 +57,7 @@ type = "memory" [directory."local".options] superuser-group = "superusers" -[[directory."local".users]] +[[directory."local".principals]] name = "admin" description = "Superuser" secret = "secret" diff --git a/tests/src/smtp/mod.rs b/tests/src/smtp/mod.rs index c0ca8b93..cf4bb95e 100644 --- a/tests/src/smtp/mod.rs +++ b/tests/src/smtp/mod.rs @@ -25,7 +25,7 @@ use std::{path::PathBuf, sync::Arc, time::Duration}; use ahash::AHashMap; use dashmap::DashMap; -use directory::memory::MemoryDirectory; +use directory::backend::memory::MemoryDirectory; use mail_auth::{ common::lru::{DnsCache, LruCache}, hickory_resolver::config::{ResolverConfig, ResolverOpts},