diff --git a/Cargo.lock b/Cargo.lock index 74debb06..09c9f2f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1910,6 +1910,7 @@ dependencies = [ "mail-parser", "mail-send", "md5", + "nlp", "password-hash", "pbkdf2", "proc_macros", diff --git a/crates/directory/Cargo.toml b/crates/directory/Cargo.toml index 5dc5c282..81aa38d3 100644 --- a/crates/directory/Cargo.toml +++ b/crates/directory/Cargo.toml @@ -9,6 +9,7 @@ utils = { path = "../utils" } proc_macros = { path = "../utils/proc-macros" } store = { path = "../store" } trc = { path = "../trc" } +nlp = { path = "../nlp" } jmap_proto = { path = "../jmap-proto" } smtp-proto = { version = "0.1" } mail-parser = { version = "0.10", features = ["full_encoding", "serde_support"] } diff --git a/crates/directory/src/backend/internal/manage.rs b/crates/directory/src/backend/internal/manage.rs index 2ad14fee..04ccb837 100644 --- a/crates/directory/src/backend/internal/manage.rs +++ b/crates/directory/src/backend/internal/manage.rs @@ -11,13 +11,16 @@ use super::{ use crate::{ MemberOf, Permission, PermissionGrant, Permissions, Principal, PrincipalData, PrincipalQuota, QueryBy, ROLE_ADMIN, ROLE_TENANT_ADMIN, ROLE_USER, Type, backend::RcptType, + core::principal::build_search_index, }; use ahash::{AHashMap, AHashSet}; - use compact_str::CompactString; use jmap_proto::types::collection::Collection; +use nlp::tokenizers::word::WordTokenizer; use store::{ Deserialize, IterateParams, Serialize, SerializeInfallible, Store, U32_LEN, ValueKey, + backend::MAX_TOKEN_LENGTH, + roaring::RoaringBitmap, write::{ AlignedBytes, Archive, Archiver, BatchBuilder, DirectoryClass, ValueClass, key::DeserializeBigEndian, @@ -148,7 +151,6 @@ impl ManageDirectory for Store { self.get_principal_info(name).await.map(|v| v.map(|v| v.id)) } async fn get_principal_info(&self, name: &str) -> trc::Result> { - let todo = "cache"; self.get_value::(ValueKey::from(ValueClass::Directory( DirectoryClass::NameToId(name.as_bytes().to_vec()), ))) @@ -195,7 +197,9 @@ impl ManageDirectory for Store { .with_account_id(u32::MAX) .with_collection(Collection::Principal) .assert_value(name_key.clone(), ()) - .create_document(principal_id) + .create_document(principal_id); + build_search_index(&mut batch, principal_id, None, Some(&principal)); + batch .set( name_key, PrincipalInfo::new(principal_id, typ, None).serialize(), @@ -545,7 +549,9 @@ impl ManageDirectory for Store { principal_create.name().as_bytes().to_vec(), )), (), - ) + ); + build_search_index(&mut batch, principal_id, None, Some(&principal_create)); + batch .set( ValueClass::Directory(DirectoryClass::Principal(principal_id)), principal_bytes, @@ -618,23 +624,33 @@ impl ManageDirectory for Store { QueryBy::Id(principal_id) => principal_id, QueryBy::Credentials(_) => unreachable!(), }; - let principal = self - .get_principal(principal_id) + + let principal_ = self + .get_value::>(ValueKey::from(ValueClass::Directory( + DirectoryClass::Principal(principal_id), + ))) .await .caused_by(trc::location!())? .ok_or_else(|| not_found(principal_id.to_string()))?; + let principal = principal_ + .unarchive::() + .caused_by(trc::location!())?; + let typ = Type::from(&principal.typ); + let mut batch = BatchBuilder::new(); + batch.with_account_id(u32::MAX); // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd // SPDX-License-Identifier: LicenseRef-SEL // Make sure tenant has no data + let tenant = principal.tenant.as_ref().map(|t| t.to_native()); #[cfg(feature = "enterprise")] - match principal.typ { + match typ { Type::Individual | Type::Group => { // Update tenant quota - if let Some(tenant_id) = principal.tenant() { + if let Some(tenant_id) = tenant { let quota = self .get_counter(DirectoryClass::UsedQuota(principal_id)) .await @@ -648,7 +664,7 @@ impl ManageDirectory for Store { let tenant_members = self .list_principals( None, - principal.id().into(), + principal_id.into(), &[ Type::Individual, Type::Group, @@ -688,8 +704,8 @@ impl ManageDirectory for Store { } } Type::Domain => { - if let Some(tenant_id) = principal.tenant() { - let name = principal.name(); + if let Some(tenant_id) = tenant { + let name = principal.name.as_str(); let tenant_members = self .list_principals( None, @@ -773,15 +789,17 @@ impl ManageDirectory for Store { // Delete principal batch - .with_account_id(principal_id) + .delete_document(principal_id) .clear(DirectoryClass::NameToId(principal.name.as_bytes().to_vec())) .clear(DirectoryClass::Principal(principal_id)) .clear(DirectoryClass::UsedQuota(principal_id)); - for email in principal.emails { + for email in principal.emails.iter() { batch.clear(DirectoryClass::EmailToId(email.as_bytes().to_vec())); } + build_search_index(&mut batch, principal_id, Some(principal), None); + for member in self .get_member_of(principal_id) .await @@ -790,7 +808,7 @@ impl ManageDirectory for Store { // Update changed principals changed_principals.add_member_change( principal_id, - principal.typ, + typ, member.principal_id, member.typ, ); @@ -817,12 +835,7 @@ impl ManageDirectory for Store { .await .caused_by(trc::location!())? { - changed_principals.add_member_change( - member_id, - member_info.typ, - principal_id, - principal.typ, - ); + changed_principals.add_member_change(member_id, member_info.typ, principal_id, typ); } // Remove members @@ -840,7 +853,7 @@ impl ManageDirectory for Store { .await .caused_by(trc::location!())?; - changed_principals.add_deletion(principal_id, principal.typ); + changed_principals.add_deletion(principal_id, typ); Ok(changed_principals) } @@ -908,13 +921,6 @@ impl ManageDirectory for Store { ) }); - if update_principal { - batch.assert_value( - ValueClass::Directory(DirectoryClass::Principal(principal_id)), - prev_principal, - ); - } - let mut used_quota: Option = None; // SPDX-SnippetBegin @@ -1860,12 +1866,24 @@ impl ManageDirectory for Store { } if update_principal { - batch.set( - ValueClass::Directory(DirectoryClass::Principal(principal_id)), - Archiver::new(principal) - .serialize() - .caused_by(trc::location!())?, + build_search_index( + &mut batch, + principal_id, + Some(prev_principal.inner), + Some(&principal), ); + + batch + .assert_value( + ValueClass::Directory(DirectoryClass::Principal(principal_id)), + prev_principal, + ) + .set( + ValueClass::Directory(DirectoryClass::Principal(principal_id)), + Archiver::new(principal) + .serialize() + .caused_by(trc::location!())?, + ); } self.write(batch.build_all()) @@ -1884,24 +1902,91 @@ impl ManageDirectory for Store { page: usize, limit: usize, ) -> trc::Result> { + let filter = if let Some(filter) = filter.filter(|f| !f.trim().is_empty()) { + let mut matches = RoaringBitmap::new(); + + for token in WordTokenizer::new(filter, MAX_TOKEN_LENGTH) { + let word_bytes = token.word.as_bytes(); + let from_key = ValueKey::from(ValueClass::Directory(DirectoryClass::Index { + word: word_bytes.to_vec(), + principal_id: 0, + })); + let to_key = ValueKey::from(ValueClass::Directory(DirectoryClass::Index { + word: word_bytes.to_vec(), + principal_id: u32::MAX, + })); + + let mut word_matches = RoaringBitmap::new(); + self.iterate( + IterateParams::new(from_key, to_key).no_values(), + |key, _| { + let id_pos = key.len() - U32_LEN; + if key.get(1..id_pos).is_some_and(|v| v == word_bytes) { + word_matches.insert(key.deserialize_be_u32(id_pos)?); + Ok(true) + } else { + Ok(false) + } + }, + ) + .await + .caused_by(trc::location!())?; + + if matches.is_empty() { + matches = word_matches; + } else { + matches &= word_matches; + if matches.is_empty() { + break; + } + } + } + + if !matches.is_empty() { + Some(matches) + } else { + return Ok(PrincipalList { + total: 0, + items: vec![], + }); + } + } else { + None + }; + let from_key = ValueKey::from(ValueClass::Directory(DirectoryClass::NameToId(vec![]))); let to_key = ValueKey::from(ValueClass::Directory(DirectoryClass::NameToId(vec![ u8::MAX; 10 ]))); - let mut results = Vec::new(); + let max_items = if limit > 0 { limit } else { usize::MAX }; + let mut offset = page.saturating_sub(1) * limit; + let mut result = PrincipalList { + items: Vec::new(), + total: 0, + }; self.iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { let pt = PrincipalInfo::deserialize(value).caused_by(trc::location!())?; - if (types.is_empty() || types.contains(&pt.typ)) && pt.has_tenant_access(tenant_id) + if (types.is_empty() || types.contains(&pt.typ)) + && pt.has_tenant_access(tenant_id) + && filter.as_ref().is_none_or(|filter| filter.contains(pt.id)) { - let mut principal = Principal::new(pt.id, pt.typ); - principal.name = - String::from_utf8_lossy(key.get(1..).unwrap_or_default()).into_owned(); - results.push(principal); + result.total += 1; + if offset == 0 { + if result.items.len() < max_items { + let mut principal = Principal::new(pt.id, pt.typ); + principal.name = + String::from_utf8_lossy(key.get(1..).unwrap_or_default()) + .into_owned(); + result.items.push(principal); + } + } else { + offset -= 1; + } } Ok(true) @@ -1910,69 +1995,23 @@ impl ManageDirectory for Store { .await .caused_by(trc::location!())?; - if filter.is_none() && !fetch { - return Ok(PrincipalList { - total: results.len() as u64, - items: results - .into_iter() - .skip(page.saturating_sub(1) * limit) - .take(if limit > 0 { limit } else { usize::MAX }) - .collect(), - }); + if fetch && !result.items.is_empty() { + let mut items = Vec::with_capacity(result.items.len()); + + for principal in result.items { + items.push( + self.query(QueryBy::Id(principal.id), fetch) + .await + .caused_by(trc::location!())? + .ok_or_else(|| not_found(principal.name().to_string()))?, + ); + } + result.items = items; + + Ok(result) + } else { + Ok(result) } - - let mut result = PrincipalList { - items: vec![], - total: 0, - }; - let filters = filter.and_then(|filter| { - let filters = filter - .split_whitespace() - .map(|r| r.to_lowercase()) - .collect::>(); - if !filters.is_empty() { - Some(filters) - } else { - None - } - }); - - let todo = "fix search"; - let mut offset = limit * page.saturating_sub(1); - let mut is_done = false; - - for mut principal in results { - if !is_done || filters.is_some() { - principal = self - .query(QueryBy::Id(principal.id), fetch) - .await - .caused_by(trc::location!())? - .ok_or_else(|| not_found(principal.name().to_string()))?; - } - - if filters.as_ref().is_none_or(|filters| { - filters.iter().all(|f| { - principal.name.contains(f) - || principal - .description - .as_ref() - .is_some_and(|n| n.contains(f)) - }) - }) { - result.total += 1; - - if offset == 0 { - if !is_done { - result.items.push(principal); - is_done = limit != 0 && result.items.len() >= limit; - } - } else { - offset -= 1; - } - } - } - - Ok(result) } async fn count_principals( @@ -2236,7 +2275,7 @@ impl ManageDirectory for Store { if let Some(tenant_id) = principal.tenant { if fields.is_empty() || fields.contains(&PrincipalField::Tenant) { if let Some(name) = self - .get_principal_name(tenant_id as u32) + .get_principal_name(tenant_id) .await .caused_by(trc::location!())? { diff --git a/crates/directory/src/core/principal.rs b/crates/directory/src/core/principal.rs index 28645e1d..87db3fdb 100644 --- a/crates/directory/src/core/principal.rs +++ b/crates/directory/src/core/principal.rs @@ -6,15 +6,21 @@ use std::{collections::hash_map::Entry, fmt, str::FromStr}; +use ahash::AHashSet; +use nlp::tokenizers::word::WordTokenizer; use serde::{ Deserializer, Serializer, de::{self, IgnoredAny, Visitor}, ser::SerializeMap, }; -use store::U64_LEN; +use store::{ + U64_LEN, + backend::MAX_TOKEN_LENGTH, + write::{BatchBuilder, DirectoryClass}, +}; use crate::{ - Permission, PermissionGrant, Principal, PrincipalData, ROLE_ADMIN, Type, + ArchivedPrincipal, Permission, PermissionGrant, Principal, PrincipalData, ROLE_ADMIN, Type, backend::internal::{PrincipalField, PrincipalSet, PrincipalUpdate, PrincipalValue}, }; @@ -801,6 +807,53 @@ impl From> for PrincipalValue { } } +pub(crate) fn build_search_index( + batch: &mut BatchBuilder, + principal_id: u32, + current: Option<&ArchivedPrincipal>, + new: Option<&Principal>, +) { + let mut current_words = AHashSet::new(); + let mut new_words = AHashSet::new(); + + if let Some(current) = current { + for word in [Some(current.name.as_str()), current.description.as_deref()] + .into_iter() + .chain(current.emails.iter().map(|s| Some(s.as_str()))) + .flatten() + { + current_words.extend(WordTokenizer::new(word, MAX_TOKEN_LENGTH).map(|t| t.word)); + } + } + + if let Some(new) = new { + for word in [Some(new.name.as_str()), new.description.as_deref()] + .into_iter() + .chain(new.emails.iter().map(|s| Some(s.as_str()))) + .flatten() + { + new_words.extend(WordTokenizer::new(word, MAX_TOKEN_LENGTH).map(|t| t.word)); + } + } + + for word in new_words.difference(¤t_words) { + batch.set( + DirectoryClass::Index { + word: word.as_bytes().to_vec(), + principal_id, + }, + vec![], + ); + } + + for word in current_words.difference(&new_words) { + batch.clear(DirectoryClass::Index { + word: word.as_bytes().to_vec(), + principal_id, + }); + } +} + impl Type { pub fn to_jmap(&self) -> &'static str { match self { diff --git a/crates/directory/src/lib.rs b/crates/directory/src/lib.rs index 35a3cfcc..a3c74e49 100644 --- a/crates/directory/src/lib.rs +++ b/crates/directory/src/lib.rs @@ -439,3 +439,21 @@ impl IntoError for LdapError { } } } + +impl From<&ArchivedType> for Type { + fn from(archived: &ArchivedType) -> Self { + match archived { + ArchivedType::Individual => Type::Individual, + ArchivedType::Group => Type::Group, + ArchivedType::Resource => Type::Resource, + ArchivedType::Location => Type::Location, + ArchivedType::List => Type::List, + ArchivedType::Other => Type::Other, + ArchivedType::Domain => Type::Domain, + ArchivedType::Tenant => Type::Tenant, + ArchivedType::Role => Type::Role, + ArchivedType::ApiKey => Type::ApiKey, + ArchivedType::OauthClient => Type::OauthClient, + } + } +} diff --git a/crates/store/src/write/key.rs b/crates/store/src/write/key.rs index 4899c24f..1421e09b 100644 --- a/crates/store/src/write/key.rs +++ b/crates/store/src/write/key.rs @@ -322,6 +322,10 @@ impl ValueClass { .write(6u8) .write(*principal_id) .write(*has_member), + DirectoryClass::Index { word, principal_id } => serializer + .write(7u8) + .write(word.as_slice()) + .write(*principal_id), }, ValueClass::Queue(queue) => match queue { QueueClass::Message(queue_id) => serializer.write(*queue_id), @@ -534,6 +538,7 @@ impl ValueClass { DirectoryClass::NameToId(v) | DirectoryClass::EmailToId(v) => v.len(), DirectoryClass::Principal(_) | DirectoryClass::UsedQuota(_) => U32_LEN, DirectoryClass::Members { .. } | DirectoryClass::MemberOf { .. } => U32_LEN * 2, + DirectoryClass::Index { word, .. } => word.len() + U32_LEN, }, ValueClass::Blob(op) => match op { BlobOp::Reserve { .. } => BLOB_HASH_LEN + U64_LEN + U32_LEN + 1, diff --git a/crates/store/src/write/mod.rs b/crates/store/src/write/mod.rs index aadfe2c1..3971c01b 100644 --- a/crates/store/src/write/mod.rs +++ b/crates/store/src/write/mod.rs @@ -203,6 +203,7 @@ pub enum InMemoryClass { pub enum DirectoryClass { NameToId(Vec), EmailToId(Vec), + Index { word: Vec, principal_id: u32 }, MemberOf { principal_id: u32, member_of: u32 }, Members { principal_id: u32, has_member: u32 }, Principal(u32), diff --git a/tests/src/directory/internal.rs b/tests/src/directory/internal.rs index cb835aa6..1559d5ad 100644 --- a/tests/src/directory/internal.rs +++ b/tests/src/directory/internal.rs @@ -618,6 +618,28 @@ async fn internal_directory() { .collect::>(), vec!["list"] ); + assert_eq!( + store + .list_principals("example.org".into(), None, &[], true, 0, 0) + .await + .unwrap() + .items + .into_iter() + .map(|p| p.name) + .collect::>(), + vec!["example.org", "jane", "john.doe", "list"] + ); + assert_eq!( + store + .list_principals("johnny doe".into(), None, &[], true, 0, 0) + .await + .unwrap() + .items + .into_iter() + .map(|p| p.name) + .collect::>(), + vec!["john.doe"] + ); // Write records on John's and Jane's accounts let mut document_id = u32::MAX; diff --git a/tests/src/directory/mod.rs b/tests/src/directory/mod.rs index dedbea32..c7aca6de 100644 --- a/tests/src/directory/mod.rs +++ b/tests/src/directory/mod.rs @@ -362,6 +362,7 @@ impl DirectoryTest { true, ) .await; + config.assert_no_errors(); // Enable catch-all and subaddressing