From b7c0f8447bb44da0cc68f372146aeb7056d3dca2 Mon Sep 17 00:00:00 2001 From: mdecimus Date: Sun, 23 Feb 2025 12:22:11 +0100 Subject: [PATCH] Improved object serialization --- Cargo.lock | 28 + crates/common/Cargo.toml | 1 + crates/common/src/config/jmap/settings.rs | 40 +- crates/common/src/ipc.rs | 2 +- crates/email/Cargo.toml | 1 + crates/email/src/identity/mod.rs | 23 + crates/email/src/identity/serialize.rs | 23 + crates/email/src/lib.rs | 11 +- crates/email/src/mailbox/index.rs | 50 + .../src/{mailbox.rs => mailbox/manage.rs} | 222 +---- crates/email/src/mailbox/mod.rs | 112 +++ crates/email/src/mailbox/serialize.rs | 67 ++ .../src/email => email/src/message}/bayes.rs | 5 +- crates/email/src/{ => message}/crypto.rs | 0 .../src/email => email/src/message}/delete.rs | 14 +- crates/email/src/{ => message}/delivery.rs | 35 +- crates/email/src/{ => message}/index.rs | 0 crates/email/src/{ => message}/ingest.rs | 46 +- crates/email/src/{ => message}/metadata.rs | 0 crates/email/src/message/mod.rs | 13 + crates/email/src/push/mod.rs | 27 + crates/email/src/push/serialize.rs | 23 + crates/email/src/sieve/index.rs | 33 + .../email/src/{sieve.rs => sieve/ingest.rs} | 208 +---- crates/email/src/sieve/mod.rs | 118 +++ crates/email/src/sieve/serialize.rs | 95 ++ crates/email/src/submission/index.rs | 42 + crates/email/src/submission/mod.rs | 96 ++ crates/email/src/submission/serialize.rs | 23 + crates/email/src/{ => thread}/cache.rs | 0 crates/email/src/thread/mod.rs | 7 + crates/imap-proto/src/parser/create.rs | 18 +- crates/imap-proto/src/protocol/create.rs | 4 +- crates/imap-proto/src/protocol/list.rs | 5 +- crates/imap/src/core/mailbox.rs | 96 +- crates/imap/src/core/message.rs | 15 +- crates/imap/src/op/acl.rs | 200 ++-- crates/imap/src/op/append.rs | 6 +- crates/imap/src/op/copy_move.rs | 4 +- crates/imap/src/op/create.rs | 48 +- crates/imap/src/op/expunge.rs | 10 +- crates/imap/src/op/fetch.rs | 8 +- crates/imap/src/op/rename.rs | 44 +- crates/imap/src/op/status.rs | 48 +- crates/imap/src/op/store.rs | 18 +- crates/imap/src/op/subscribe.rs | 31 +- crates/imap/src/op/thread.rs | 6 +- crates/jmap-proto/src/method/copy.rs | 9 +- crates/jmap-proto/src/method/get.rs | 17 +- crates/jmap-proto/src/method/import.rs | 7 +- crates/jmap-proto/src/method/parse.rs | 10 +- crates/jmap-proto/src/method/set.rs | 18 +- .../jmap-proto/src/object/email_submission.rs | 11 +- crates/jmap-proto/src/object/index.rs | 867 ++++++++---------- crates/jmap-proto/src/object/mod.rs | 328 ++----- crates/jmap-proto/src/response/references.rs | 23 +- crates/jmap-proto/src/types/date.rs | 8 +- crates/jmap-proto/src/types/property.rs | 129 ++- crates/jmap-proto/src/types/value.rs | 110 ++- crates/jmap/src/api/form.rs | 13 +- .../src/api/management/enterprise/undelete.rs | 10 +- crates/jmap/src/api/management/stores.rs | 29 +- crates/jmap/src/auth/acl.rs | 154 ++-- crates/jmap/src/blob/get.rs | 8 +- crates/jmap/src/email/body.rs | 9 +- crates/jmap/src/email/copy.rs | 31 +- crates/jmap/src/email/crypto.rs | 15 +- crates/jmap/src/email/get.rs | 13 +- crates/jmap/src/email/headers.rs | 35 +- crates/jmap/src/email/import.rs | 6 +- crates/jmap/src/email/mod.rs | 2 - crates/jmap/src/email/parse.rs | 12 +- crates/jmap/src/email/query.rs | 14 +- crates/jmap/src/email/set.rs | 77 +- crates/jmap/src/email/snippet.rs | 8 +- crates/jmap/src/identity/get.rs | 77 +- crates/jmap/src/identity/set.rs | 144 +-- crates/jmap/src/mailbox/get.rs | 98 +- crates/jmap/src/mailbox/query.rs | 25 +- crates/jmap/src/mailbox/set.rs | 430 ++++----- crates/jmap/src/principal/get.rs | 10 +- crates/jmap/src/push/get.rs | 149 +-- crates/jmap/src/push/set.rs | 182 ++-- crates/jmap/src/quota/get.rs | 11 +- crates/jmap/src/services/housekeeper.rs | 210 +++-- crates/jmap/src/services/index.rs | 71 +- crates/jmap/src/sieve/get.rs | 42 +- crates/jmap/src/sieve/set.rs | 238 ++--- crates/jmap/src/submission/get.rs | 153 ++-- crates/jmap/src/submission/query.rs | 14 +- crates/jmap/src/submission/set.rs | 269 +++--- crates/jmap/src/thread/get.rs | 7 +- crates/jmap/src/vacation/get.rs | 62 +- crates/jmap/src/vacation/set.rs | 201 ++-- crates/managesieve/Cargo.toml | 1 + crates/managesieve/src/op/getscript.rs | 19 +- crates/managesieve/src/op/listscripts.rs | 21 +- crates/managesieve/src/op/putscript.rs | 109 +-- crates/managesieve/src/op/renamescript.rs | 18 +- crates/pop3/src/mailbox.rs | 16 +- crates/pop3/src/op/delete.rs | 4 +- crates/pop3/src/op/fetch.rs | 4 +- crates/smtp/src/outbound/local.rs | 6 +- crates/utils/Cargo.toml | 3 + crates/utils/src/json/mod.rs | 26 + crates/utils/src/json/parser/base32.rs | 65 ++ crates/utils/src/json/parser/impls.rs | 308 +++++++ crates/utils/src/json/parser/json.rs | 386 ++++++++ crates/utils/src/json/parser/mod.rs | 158 ++++ crates/utils/src/json/parser/pointer.rs | 222 +++++ crates/utils/src/json/pointer.rs | 106 +++ crates/utils/src/lib.rs | 8 +- crates/utils/src/map/vec_map.rs | 8 +- tests/src/jmap/crypto.rs | 28 +- tests/src/jmap/mod.rs | 3 +- tests/src/jmap/permissions.rs | 6 +- tests/src/jmap/purge.rs | 10 +- tests/src/jmap/thread_merge.rs | 4 +- 118 files changed, 4666 insertions(+), 3165 deletions(-) create mode 100644 crates/email/src/identity/mod.rs create mode 100644 crates/email/src/identity/serialize.rs create mode 100644 crates/email/src/mailbox/index.rs rename crates/email/src/{mailbox.rs => mailbox/manage.rs} (61%) create mode 100644 crates/email/src/mailbox/mod.rs create mode 100644 crates/email/src/mailbox/serialize.rs rename crates/{jmap/src/email => email/src/message}/bayes.rs (94%) rename crates/email/src/{ => message}/crypto.rs (100%) rename crates/{jmap/src/email => email/src/message}/delete.rs (98%) rename crates/email/src/{ => message}/delivery.rs (93%) rename crates/email/src/{ => message}/index.rs (100%) rename crates/email/src/{ => message}/ingest.rs (96%) rename crates/email/src/{ => message}/metadata.rs (100%) create mode 100644 crates/email/src/message/mod.rs create mode 100644 crates/email/src/push/mod.rs create mode 100644 crates/email/src/push/serialize.rs create mode 100644 crates/email/src/sieve/index.rs rename crates/email/src/{sieve.rs => sieve/ingest.rs} (84%) create mode 100644 crates/email/src/sieve/mod.rs create mode 100644 crates/email/src/sieve/serialize.rs create mode 100644 crates/email/src/submission/index.rs create mode 100644 crates/email/src/submission/mod.rs create mode 100644 crates/email/src/submission/serialize.rs rename crates/email/src/{ => thread}/cache.rs (100%) create mode 100644 crates/email/src/thread/mod.rs create mode 100644 crates/utils/src/json/mod.rs create mode 100644 crates/utils/src/json/parser/base32.rs create mode 100644 crates/utils/src/json/parser/impls.rs create mode 100644 crates/utils/src/json/parser/json.rs create mode 100644 crates/utils/src/json/parser/mod.rs create mode 100644 crates/utils/src/json/parser/pointer.rs create mode 100644 crates/utils/src/json/pointer.rs diff --git a/Cargo.lock b/Cargo.lock index 21c7f726..54e73c77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1217,6 +1217,7 @@ dependencies = [ "directory", "dns-update", "futures", + "hashify", "hostname 0.4.0", "hyper 1.6.0", "idna", @@ -1896,6 +1897,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "downcast-rs" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea8a8b81cacc08888170eef4d13b775126db426d0b348bee9d18c2c1eaf123cf" + [[package]] name = "dsa" version = "0.6.3" @@ -2055,6 +2062,7 @@ dependencies = [ "cbc", "common", "directory", + "hashify", "jmap_proto", "mail-builder", "mail-parser", @@ -2124,6 +2132,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "erased-serde" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24e2389d65ab4fab27dc2a5de7b191e1f6617d1f1c8855c0dc569c94a4cbb18d" +dependencies = [ + "serde", + "typeid", +] + [[package]] name = "errno" version = "0.3.10" @@ -4014,6 +4032,7 @@ dependencies = [ "bincode", "common", "directory", + "email", "imap", "imap_proto", "jmap", @@ -7435,6 +7454,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "typeid" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e13db2e0ccd5e14a544e8a246ba2312cd25223f616442d7f2cb0e3db614236e" + [[package]] name = "typenum" version = "1.17.0" @@ -7595,6 +7620,9 @@ dependencies = [ "base64 0.22.1", "blake3", "chrono", + "downcast-rs", + "erased-serde", + "fast-float", "form_urlencoded", "futures", "http-body-util", diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 7c53a1db..d3890197 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -66,6 +66,7 @@ rsa = "0.9.2" p256 = { version = "0.13", features = ["ecdh"] } p384 = { version = "0.13", features = ["ecdh"] } num_cpus = "1.13.1" +hashify = "0.2" [target.'cfg(unix)'.dependencies] privdrop = "0.5.3" diff --git a/crates/common/src/config/jmap/settings.rs b/crates/common/src/config/jmap/settings.rs index 2a16ab6c..294f7467 100644 --- a/crates/common/src/config/jmap/settings.rs +++ b/crates/common/src/config/jmap/settings.rs @@ -8,7 +8,7 @@ use std::{str::FromStr, time::Duration}; use jmap_proto::request::capability::BaseCapabilities; use nlp::language::Language; -use utils::config::{cron::SimpleCron, utils::ParseValue, Config, Rate}; +use utils::config::{Config, Rate, cron::SimpleCron, utils::ParseValue}; #[derive(Default, Clone)] pub struct JmapConfig { @@ -93,6 +93,7 @@ pub enum SpecialUse { Archive, Sent, Shared, + Important, None, } @@ -372,16 +373,33 @@ impl JmapConfig { impl ParseValue for SpecialUse { fn parse_value(value: &str) -> Result { - match value { - "inbox" => Ok(SpecialUse::Inbox), - "trash" => Ok(SpecialUse::Trash), - "junk" => Ok(SpecialUse::Junk), - "drafts" => Ok(SpecialUse::Drafts), - "archive" => Ok(SpecialUse::Archive), - "sent" => Ok(SpecialUse::Sent), - "shared" => Ok(SpecialUse::Shared), - //"none" => Ok(SpecialUse::None), - other => Err(format!("Unknown folder role {other:?}")), + hashify::tiny_map_ignore_case!(value.as_bytes(), + b"inbox" => SpecialUse::Inbox, + b"trash" => SpecialUse::Trash, + b"junk" => SpecialUse::Junk, + b"drafts" => SpecialUse::Drafts, + b"archive" => SpecialUse::Archive, + b"sent" => SpecialUse::Sent, + b"shared" => SpecialUse::Shared, + b"important" => SpecialUse::Important, + + ) + .ok_or_else(|| format!("Unknown folder role {:?}", value)) + } +} + +impl SpecialUse { + pub fn as_str(&self) -> Option<&'static str> { + match self { + SpecialUse::Inbox => Some("inbox"), + SpecialUse::Trash => Some("trash"), + SpecialUse::Junk => Some("junk"), + SpecialUse::Drafts => Some("drafts"), + SpecialUse::Archive => Some("archive"), + SpecialUse::Sent => Some("sent"), + SpecialUse::Shared => Some("shared"), + SpecialUse::Important => Some("important"), + SpecialUse::None => None, } } } diff --git a/crates/common/src/ipc.rs b/crates/common/src/ipc.rs index 06d95faf..7853596e 100644 --- a/crates/common/src/ipc.rs +++ b/crates/common/src/ipc.rs @@ -11,7 +11,7 @@ use jmap_proto::types::{state::StateChange, type_state::DataType}; use mail_auth::{ dmarc::Dmarc, mta_sts::TlsRpt, - report::{tlsrpt::FailureDetails, Record}, + report::{Record, tlsrpt::FailureDetails}, }; use store::{BlobStore, InMemoryStore, Store}; use tokio::sync::mpsc; diff --git a/crates/email/Cargo.toml b/crates/email/Cargo.toml index 39340f4c..e44eef71 100644 --- a/crates/email/Cargo.toml +++ b/crates/email/Cargo.toml @@ -31,6 +31,7 @@ rasn-pkix = "0.10" rsa = "0.9.2" rand = "0.8" sequoia-openpgp = { version = "1.16", default-features = false, features = ["crypto-rust", "allow-experimental-crypto", "allow-variable-time-crypto"] } +hashify = "0.2" [features] test_mode = [] diff --git a/crates/email/src/identity/mod.rs b/crates/email/src/identity/mod.rs new file mode 100644 index 00000000..5bd0e735 --- /dev/null +++ b/crates/email/src/identity/mod.rs @@ -0,0 +1,23 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod serialize; + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct Identity { + pub name: String, + pub email: String, + pub reply_to: Option>, + pub bcc: Option>, + pub text_signature: String, + pub html_signature: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EmailAddress { + pub name: Option, + pub email: String, +} diff --git a/crates/email/src/identity/serialize.rs b/crates/email/src/identity/serialize.rs new file mode 100644 index 00000000..369d44ab --- /dev/null +++ b/crates/email/src/identity/serialize.rs @@ -0,0 +1,23 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use store::{Deserialize, Serialize}; + +use super::Identity; + +impl Serialize for Identity { + fn serialize(self) -> Vec { + let todo = 1; + todo!() + } +} + +impl Deserialize for Identity { + fn deserialize(bytes: &[u8]) -> trc::Result { + let todo = 1; + todo!() + } +} diff --git a/crates/email/src/lib.rs b/crates/email/src/lib.rs index bf9ff2fa..4d22e499 100644 --- a/crates/email/src/lib.rs +++ b/crates/email/src/lib.rs @@ -4,11 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -pub mod cache; -pub mod crypto; -pub mod delivery; -pub mod index; -pub mod ingest; +pub mod identity; pub mod mailbox; -pub mod metadata; +pub mod message; +pub mod push; pub mod sieve; +pub mod submission; +pub mod thread; diff --git a/crates/email/src/mailbox/index.rs b/crates/email/src/mailbox/index.rs new file mode 100644 index 00000000..a86610cd --- /dev/null +++ b/crates/email/src/mailbox/index.rs @@ -0,0 +1,50 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use common::config::jmap::settings::SpecialUse; +use jmap_proto::{ + object::index::{IndexValue, IndexableObject}, + types::property::Property, +}; + +use super::Mailbox; + +impl IndexableObject for Mailbox { + fn index_values(&self) -> impl Iterator> { + [ + IndexValue::Text { + field: Property::Name.into(), + value: self.name.as_str(), + tokenize: true, + index: true, + }, + IndexValue::Text { + field: Property::Role.into(), + value: self.role.as_str().unwrap_or_default(), + tokenize: false, + index: true, + }, + IndexValue::Tag { + field: Property::Role.into(), + is_set: !matches!(self.role, SpecialUse::None), + }, + IndexValue::U32 { + field: Property::ParentId.into(), + value: self.parent_id.into(), + }, + IndexValue::U32 { + field: Property::SortOrder.into(), + value: self.sort_order, + }, + IndexValue::U32List { + field: Property::IsSubscribed.into(), + value: &self.subscribers, + }, + IndexValue::Acl { value: &self.acls }, + ] + .into_iter() + } +} diff --git a/crates/email/src/mailbox.rs b/crates/email/src/mailbox/manage.rs similarity index 61% rename from crates/email/src/mailbox.rs rename to crates/email/src/mailbox/manage.rs index 20d27156..f4a90b52 100644 --- a/crates/email/src/mailbox.rs +++ b/crates/email/src/mailbox/manage.rs @@ -4,69 +4,19 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{future::Future, slice::Iter}; +use std::future::Future; -use common::{config::jmap::settings::SpecialUse, Server}; +use common::{Server, config::jmap::settings::SpecialUse}; use jmap_proto::{ - object::{ - index::{IndexAs, IndexProperty, ObjectIndexBuilder}, - Object, - }, - types::{collection::Collection, id::Id, keyword::Keyword, property::Property, value::Value}, -}; -use store::{ - ahash::AHashSet, - query::Filter, - rand, - roaring::RoaringBitmap, - write::{ - BatchBuilder, BitmapClass, DeserializeFrom, MaybeDynamicId, Operation, SerializeInto, - TagValue, ToBitmaps, - }, - Serialize, U32_LEN, + object::index::ObjectIndexBuilder, + types::{collection::Collection, keyword::Keyword, property::Property}, }; +use store::{ahash::AHashSet, query::Filter, roaring::RoaringBitmap, write::BatchBuilder}; use trc::AddContext; -use utils::codec::leb128::{Leb128Iterator, Leb128Vec}; -use crate::cache::ThreadCache; +use crate::thread::cache::ThreadCache; -pub const INBOX_ID: u32 = 0; -pub const TRASH_ID: u32 = 1; -pub const JUNK_ID: u32 = 2; -pub const DRAFTS_ID: u32 = 3; -pub const SENT_ID: u32 = 4; -pub const ARCHIVE_ID: u32 = 5; -pub const TOMBSTONE_ID: u32 = u32::MAX - 1; - -#[derive(Debug)] -pub struct ExpandPath<'x> { - pub path: Vec<&'x str>, - pub found_names: Vec<(String, u32, u32)>, -} - -pub static SCHEMA: &[IndexProperty] = &[ - IndexProperty::new(Property::Name) - .index_as(IndexAs::Text { - tokenize: true, - index: true, - }) - .required(), - IndexProperty::new(Property::Role).index_as(IndexAs::Text { - tokenize: false, - index: true, - }), - IndexProperty::new(Property::Role).index_as(IndexAs::HasProperty), - IndexProperty::new(Property::ParentId).index_as(IndexAs::Integer), - IndexProperty::new(Property::SortOrder).index_as(IndexAs::Integer), - IndexProperty::new(Property::IsSubscribed).index_as(IndexAs::IntegerList), - IndexProperty::new(Property::Acl).index_as(IndexAs::Acl), -]; - -#[derive(Debug, Clone, Copy)] -pub struct UidMailbox { - pub mailbox_id: u32, - pub uid: u32, -} +use super::*; pub trait MailboxFnc: Sync + Send { fn mailbox_get_or_create( @@ -109,7 +59,7 @@ pub trait MailboxFnc: Sync + Send { fn mailbox_get_by_role( &self, account_id: u32, - role: &str, + role: SpecialUse, ) -> impl Future>> + Send; } @@ -136,39 +86,27 @@ impl MailboxFnc for Server { // Create mailboxes let mut last_document_id = ARCHIVE_ID; for folder in &self.core.jmap.default_folders { - let (role, document_id) = match folder.special_use { - SpecialUse::Inbox => ("inbox", INBOX_ID), - SpecialUse::Trash => ("trash", TRASH_ID), - SpecialUse::Junk => ("junk", JUNK_ID), - SpecialUse::Drafts => ("drafts", DRAFTS_ID), - SpecialUse::Sent => ("sent", SENT_ID), - SpecialUse::Archive => ("archive", ARCHIVE_ID), - SpecialUse::None => { + let document_id = match folder.special_use { + SpecialUse::Inbox => INBOX_ID, + SpecialUse::Trash => TRASH_ID, + SpecialUse::Junk => JUNK_ID, + SpecialUse::Drafts => DRAFTS_ID, + SpecialUse::Sent => SENT_ID, + SpecialUse::Archive => ARCHIVE_ID, + SpecialUse::None | SpecialUse::Important => { last_document_id += 1; - ("", last_document_id) + last_document_id } SpecialUse::Shared => unreachable!(), }; - let mut object = Object::with_capacity(4) - .with_property(Property::Name, folder.name.clone()) - .with_property(Property::ParentId, Value::Id(0u64.into())) - .with_property( - Property::Cid, - Value::UnsignedInt(rand::random::() as u64), - ); - if !role.is_empty() { - object.set(Property::Role, role); - } + let mut object = Mailbox::new(folder.name.clone()).with_role(folder.special_use); if folder.subscribe { - object.set( - Property::IsSubscribed, - Value::List(vec![Value::Id(account_id.into())]), - ); + object.add_subscriber(account_id); } batch .create_document_with_id(document_id) - .custom(ObjectIndexBuilder::new(SCHEMA).with_changes(object)); + .custom(ObjectIndexBuilder::new().with_changes(object)); mailbox_ids.insert(document_id); } @@ -224,18 +162,8 @@ impl MailboxFnc for Server { .with_collection(Collection::Mailbox) .create_document() .custom( - ObjectIndexBuilder::new(SCHEMA).with_changes( - Object::with_capacity(3) - .with_property(Property::Name, name) - .with_property( - Property::ParentId, - Value::Id(Id::from(next_parent_id)), - ) - .with_property( - Property::Cid, - Value::UnsignedInt(rand::random::() as u64), - ), - ), + ObjectIndexBuilder::new() + .with_changes(Mailbox::new(name).with_parent_id(next_parent_id)), ); let document_id = self .store() @@ -333,11 +261,7 @@ impl MailboxFnc for Server { .split('/') .filter_map(|p| { let p = p.trim(); - if !p.is_empty() { - p.into() - } else { - None - } + if !p.is_empty() { p.into() } else { None } }) .collect::>(); if path.is_empty() || path.len() > self.core.jmap.mailbox_max_depth { @@ -374,8 +298,8 @@ impl MailboxFnc for Server { let mut found_names = Vec::new(); for document_id in document_ids { - if let Some(mut obj) = self - .get_property::>( + if let Some(obj) = self + .get_property::( account_id, Collection::Mailbox, document_id, @@ -383,19 +307,7 @@ impl MailboxFnc for Server { ) .await? { - if let Some(Value::Text(value)) = obj.properties.remove(&Property::Name) { - found_names.push(( - value, - if let Some(Value::Id(value)) = obj.properties.remove(&Property::ParentId) { - value.document_id() - } else { - 0 - }, - document_id + 1, - )); - } else { - return Ok(None); - } + found_names.push((obj.name, obj.parent_id, document_id + 1)); } else { return Ok(None); } @@ -427,69 +339,23 @@ impl MailboxFnc for Server { })) } - async fn mailbox_get_by_role(&self, account_id: u32, role: &str) -> trc::Result> { - self.store() - .filter( - account_id, - Collection::Mailbox, - vec![Filter::eq(Property::Role, role)], - ) - .await - .caused_by(trc::location!()) - .map(|r| r.results.min()) - } -} - -impl PartialEq for UidMailbox { - fn eq(&self, other: &Self) -> bool { - self.mailbox_id == other.mailbox_id - } -} - -impl Eq for UidMailbox {} - -impl ToBitmaps for UidMailbox { - fn to_bitmaps(&self, ops: &mut Vec, field: u8, set: bool) { - ops.push(Operation::Bitmap { - class: BitmapClass::Tag { - field, - value: TagValue::Id(MaybeDynamicId::Static(self.mailbox_id)), - }, - set, - }); - } -} - -impl SerializeInto for UidMailbox { - fn serialize_into(&self, buf: &mut Vec) { - buf.push_leb128(self.mailbox_id); - buf.push_leb128(self.uid); - } -} - -impl DeserializeFrom for UidMailbox { - fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option { - Some(UidMailbox { - mailbox_id: bytes.next_leb128()?, - uid: bytes.next_leb128()?, - }) - } -} - -impl Serialize for UidMailbox { - fn serialize(self) -> Vec { - let mut buf = Vec::with_capacity(U32_LEN * 2); - self.serialize_into(&mut buf); - buf - } -} - -impl UidMailbox { - pub fn new(mailbox_id: u32, uid: u32) -> Self { - UidMailbox { mailbox_id, uid } - } - - pub fn new_unassigned(mailbox_id: u32) -> Self { - UidMailbox { mailbox_id, uid: 0 } + async fn mailbox_get_by_role( + &self, + account_id: u32, + role: SpecialUse, + ) -> trc::Result> { + if let Some(role) = role.as_str() { + self.store() + .filter( + account_id, + Collection::Mailbox, + vec![Filter::eq(Property::Role, role.to_string())], + ) + .await + .caused_by(trc::location!()) + .map(|r| r.results.min()) + } else { + Ok(None) + } } } diff --git a/crates/email/src/mailbox/mod.rs b/crates/email/src/mailbox/mod.rs new file mode 100644 index 00000000..4aced5b0 --- /dev/null +++ b/crates/email/src/mailbox/mod.rs @@ -0,0 +1,112 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use common::config::jmap::settings::SpecialUse; +use jmap_proto::types::value::AclGrant; + +pub mod index; +pub mod manage; +pub mod serialize; + +pub const INBOX_ID: u32 = 0; +pub const TRASH_ID: u32 = 1; +pub const JUNK_ID: u32 = 2; +pub const DRAFTS_ID: u32 = 3; +pub const SENT_ID: u32 = 4; +pub const ARCHIVE_ID: u32 = 5; +pub const TOMBSTONE_ID: u32 = u32::MAX - 1; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Mailbox { + pub name: String, + pub role: SpecialUse, + pub parent_id: u32, + pub sort_order: Option, + pub uid_validity: u32, + pub subscribers: Vec, + pub acls: Vec, +} + +#[derive(Debug, Clone, Copy)] +pub struct UidMailbox { + pub mailbox_id: u32, + pub uid: u32, +} + +#[derive(Debug)] +pub struct ExpandPath<'x> { + pub path: Vec<&'x str>, + pub found_names: Vec<(String, u32, u32)>, +} + +impl Mailbox { + pub fn new(name: impl Into) -> Self { + Mailbox { + name: name.into(), + role: SpecialUse::None, + parent_id: 0, + sort_order: None, + uid_validity: rand::random::(), + subscribers: vec![], + acls: vec![], + } + } + + pub fn with_role(mut self, role: SpecialUse) -> Self { + self.role = role; + self + } + + pub fn with_parent_id(mut self, parent_id: u32) -> Self { + self.parent_id = parent_id; + self + } + + pub fn with_sort_order(mut self, sort_order: u32) -> Self { + self.sort_order = Some(sort_order); + self + } + + pub fn with_subscriber(mut self, subscriber: u32) -> Self { + self.subscribers.push(subscriber); + self + } + + pub fn add_subscriber(&mut self, subscriber: u32) -> bool { + if !self.subscribers.contains(&subscriber) { + self.subscribers.push(subscriber); + true + } else { + false + } + } + + pub fn remove_subscriber(&mut self, subscriber: u32) { + self.subscribers.retain(|&x| x != subscriber); + } + + pub fn is_subscribed(&self, subscriber: u32) -> bool { + self.subscribers.contains(&subscriber) + } +} + +impl PartialEq for UidMailbox { + fn eq(&self, other: &Self) -> bool { + self.mailbox_id == other.mailbox_id + } +} + +impl Eq for UidMailbox {} + +impl UidMailbox { + pub fn new(mailbox_id: u32, uid: u32) -> Self { + UidMailbox { mailbox_id, uid } + } + + pub fn new_unassigned(mailbox_id: u32) -> Self { + UidMailbox { mailbox_id, uid: 0 } + } +} diff --git a/crates/email/src/mailbox/serialize.rs b/crates/email/src/mailbox/serialize.rs new file mode 100644 index 00000000..01bfe64c --- /dev/null +++ b/crates/email/src/mailbox/serialize.rs @@ -0,0 +1,67 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + + use std::slice::Iter; + +use store::{ + Deserialize, Serialize, U32_LEN, + write::{ + BitmapClass, DeserializeFrom, MaybeDynamicId, Operation, SerializeInto, TagValue, ToBitmaps, + }, +}; +use utils::codec::leb128::{Leb128Iterator, Leb128Vec}; + +use super::{Mailbox, UidMailbox}; + +impl Serialize for Mailbox { + fn serialize(self) -> Vec { + let todo = 1; + todo!() + } +} + +impl Deserialize for Mailbox { + fn deserialize(bytes: &[u8]) -> trc::Result { + let todo = 1; + todo!() + } +} + +impl ToBitmaps for UidMailbox { + fn to_bitmaps(&self, ops: &mut Vec, field: u8, set: bool) { + ops.push(Operation::Bitmap { + class: BitmapClass::Tag { + field, + value: TagValue::Id(MaybeDynamicId::Static(self.mailbox_id)), + }, + set, + }); + } +} + +impl SerializeInto for UidMailbox { + fn serialize_into(&self, buf: &mut Vec) { + buf.push_leb128(self.mailbox_id); + buf.push_leb128(self.uid); + } +} + +impl DeserializeFrom for UidMailbox { + fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option { + Some(UidMailbox { + mailbox_id: bytes.next_leb128()?, + uid: bytes.next_leb128()?, + }) + } +} + +impl Serialize for UidMailbox { + fn serialize(self) -> Vec { + let mut buf = Vec::with_capacity(U32_LEN * 2); + self.serialize_into(&mut buf); + buf + } +} diff --git a/crates/jmap/src/email/bayes.rs b/crates/email/src/message/bayes.rs similarity index 94% rename from crates/jmap/src/email/bayes.rs rename to crates/email/src/message/bayes.rs index 9d8dd4c8..0c9ad8cf 100644 --- a/crates/jmap/src/email/bayes.rs +++ b/crates/email/src/message/bayes.rs @@ -7,15 +7,16 @@ use std::future::Future; use common::Server; -use email::metadata::MessageMetadata; use jmap_proto::types::{collection::Collection, property::Property}; use mail_parser::Message; use spam_filter::{ - analysis::init::SpamFilterInit, modules::bayes::BayesClassifier, SpamFilterInput, + SpamFilterInput, analysis::init::SpamFilterInit, modules::bayes::BayesClassifier, }; use store::write::{Bincode, TaskQueueClass}; use trc::StoreEvent; +use super::metadata::MessageMetadata; + pub trait EmailBayesTrain: Sync + Send { fn email_bayes_train( &self, diff --git a/crates/email/src/crypto.rs b/crates/email/src/message/crypto.rs similarity index 100% rename from crates/email/src/crypto.rs rename to crates/email/src/message/crypto.rs diff --git a/crates/jmap/src/email/delete.rs b/crates/email/src/message/delete.rs similarity index 98% rename from crates/jmap/src/email/delete.rs rename to crates/email/src/message/delete.rs index d421be88..c981f3ea 100644 --- a/crates/jmap/src/email/delete.rs +++ b/crates/email/src/message/delete.rs @@ -7,11 +7,6 @@ use std::time::Duration; use common::{KV_LOCK_PURGE_ACCOUNT, Server}; -use email::{ - index::EmailIndexBuilder, - mailbox::{JUNK_ID, TOMBSTONE_ID, TRASH_ID, UidMailbox}, - metadata::MessageMetadata, -}; use jmap_proto::types::{ collection::Collection, id::Id, keyword::Keyword, property::Property, state::StateChange, type_state::DataType, @@ -28,8 +23,13 @@ use store::{ use trc::{AddContext, StoreEvent}; use utils::codec::leb128::Leb128Reader; -use rand::prelude::SliceRandom; use std::future::Future; +use store::rand::prelude::SliceRandom; + +use crate::{ + mailbox::*, + message::{index::EmailIndexBuilder, metadata::MessageMetadata}, +}; pub trait EmailDeletion: Sync + Send { fn emails_tombstone( @@ -238,7 +238,7 @@ impl EmailDeletion for Server { let mut account_ids: Vec = account_ids.into_iter().collect(); // Shuffle account ids - account_ids.shuffle(&mut rand::rng()); + account_ids.shuffle(&mut store::rand::rng()); for account_id in account_ids { self.purge_account(account_id).await; diff --git a/crates/email/src/delivery.rs b/crates/email/src/message/delivery.rs similarity index 93% rename from crates/email/src/delivery.rs rename to crates/email/src/message/delivery.rs index ea829a9c..28aec0ad 100644 --- a/crates/email/src/delivery.rs +++ b/crates/email/src/message/delivery.rs @@ -12,11 +12,9 @@ use std::{borrow::Cow, future::Future}; use store::ahash::AHashMap; use utils::BlobHash; -use crate::{ - ingest::{EmailIngest, IngestEmail, IngestSource}, - mailbox::INBOX_ID, - sieve::SieveScriptIngest, -}; +use crate::{mailbox::INBOX_ID, sieve::ingest::SieveScriptIngest}; + +use super::ingest::{EmailIngest, IngestEmail, IngestSource}; #[derive(Debug)] pub struct IngestMessage { @@ -153,10 +151,11 @@ impl MailDelivery for Server { }; } Err(err) => { - trc::error!(err - .details("Failed to fetch message blob.") - .span_id(message.session_id) - .caused_by(trc::location!())); + trc::error!( + err.details("Failed to fetch message blob.") + .span_id(message.session_id) + .caused_by(trc::location!()) + ); return LocalDeliveryResult { status: (0..message.recipients.len()) @@ -191,11 +190,12 @@ impl MailDelivery for Server { continue; } Err(err) => { - trc::error!(err - .details("Failed to lookup recipient.") - .ctx(trc::Key::To, rcpt) - .span_id(message.session_id) - .caused_by(trc::location!())); + trc::error!( + err.details("Failed to lookup recipient.") + .ctx(trc::Key::To, rcpt) + .span_id(message.session_id) + .caused_by(trc::location!()) + ); result.status.push(LocalDeliveryStatus::TemporaryFailure { reason: "Address lookup failed.".into(), }); @@ -307,9 +307,10 @@ impl MailDelivery for Server { }, }; - trc::error!(err - .ctx(trc::Key::To, rcpt.to_string()) - .span_id(message.session_id)); + trc::error!( + err.ctx(trc::Key::To, rcpt.to_string()) + .span_id(message.session_id) + ); status } diff --git a/crates/email/src/index.rs b/crates/email/src/message/index.rs similarity index 100% rename from crates/email/src/index.rs rename to crates/email/src/message/index.rs diff --git a/crates/email/src/ingest.rs b/crates/email/src/message/ingest.rs similarity index 96% rename from crates/email/src/ingest.rs rename to crates/email/src/message/ingest.rs index 37e8ff98..41ef73f5 100644 --- a/crates/email/src/ingest.rs +++ b/crates/email/src/message/ingest.rs @@ -11,49 +11,51 @@ use std::{ }; use common::{ - auth::{AccessToken, ResourceToken}, Server, + auth::{AccessToken, ResourceToken}, }; use directory::Permission; -use jmap_proto::{ - object::Object, - types::{ - blob::BlobId, collection::Collection, id::Id, keyword::Keyword, property::Property, - value::Value, - }, +use jmap_proto::types::{ + blob::BlobId, + collection::Collection, + id::Id, + keyword::Keyword, + property::Property, + value::{Object, Value}, }; use mail_parser::{ - parsers::fields::thread::thread_name, Header, HeaderName, HeaderValue, Message, MessageParser, - PartType, + Header, HeaderName, HeaderValue, Message, MessageParser, PartType, + parsers::fields::thread::thread_name, }; use spam_filter::{ - analysis::init::SpamFilterInit, modules::bayes::BayesClassifier, SpamFilterInput, + SpamFilterInput, analysis::init::SpamFilterInit, modules::bayes::BayesClassifier, }; use std::future::Future; use store::rand::Rng; use store::{ + BitmapKey, BlobClass, Serialize, ahash::AHashSet, query::Filter, write::{ + AssignedIds, BatchBuilder, BitmapClass, F_BITMAP, F_CLEAR, F_VALUE, MaybeDynamicId, + MaybeDynamicValue, SerializeWithId, TagValue, TaskQueueClass, ValueClass, log::{ChangeLogBuilder, Changes, LogInsert}, - now, AssignedIds, BatchBuilder, BitmapClass, MaybeDynamicId, MaybeDynamicValue, - SerializeWithId, TagValue, TaskQueueClass, ValueClass, F_BITMAP, F_CLEAR, F_VALUE, + now, }, - BitmapKey, BlobClass, Serialize, }; use trc::{AddContext, MessageIngestEvent}; use utils::map::vec_map::VecMap; use crate::{ - index::{IndexMessage, VisitValues, MAX_ID_LENGTH}, - mailbox::{UidMailbox, INBOX_ID, JUNK_ID}, + mailbox::{INBOX_ID, JUNK_ID, UidMailbox}, + message::index::{IndexMessage, MAX_ID_LENGTH, VisitValues}, + thread::cache::ThreadCache, }; use super::{ - cache::ThreadCache, crypto::{EncryptMessage, EncryptMessageError, EncryptionParams}, - index::{TrimTextValue, MAX_SORT_FIELD_LENGTH}, + index::{MAX_SORT_FIELD_LENGTH, TrimTextValue}, }; #[derive(Default)] @@ -425,10 +427,12 @@ impl EmailIngest for Server { } } Err(EncryptMessageError::Error(err)) => { - trc::bail!(trc::StoreEvent::CryptoError - .into_err() - .caused_by(trc::location!()) - .reason(err)); + trc::bail!( + trc::StoreEvent::CryptoError + .into_err() + .caused_by(trc::location!()) + .reason(err) + ); } _ => unreachable!(), } diff --git a/crates/email/src/metadata.rs b/crates/email/src/message/metadata.rs similarity index 100% rename from crates/email/src/metadata.rs rename to crates/email/src/message/metadata.rs diff --git a/crates/email/src/message/mod.rs b/crates/email/src/message/mod.rs new file mode 100644 index 00000000..8024560e --- /dev/null +++ b/crates/email/src/message/mod.rs @@ -0,0 +1,13 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod bayes; +pub mod crypto; +pub mod delete; +pub mod delivery; +pub mod index; +pub mod ingest; +pub mod metadata; diff --git a/crates/email/src/push/mod.rs b/crates/email/src/push/mod.rs new file mode 100644 index 00000000..ad9efb9b --- /dev/null +++ b/crates/email/src/push/mod.rs @@ -0,0 +1,27 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod serialize; + +use jmap_proto::types::type_state::DataType; +use utils::map::bitmap::Bitmap; + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct PushSubscription { + pub url: String, + pub device_client_id: String, + pub expires: u64, + pub verification_code: String, + pub verified: bool, + pub types: Bitmap, + pub keys: Option, +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct Keys { + pub p256dh: Vec, + pub auth: Vec, +} diff --git a/crates/email/src/push/serialize.rs b/crates/email/src/push/serialize.rs new file mode 100644 index 00000000..23cefb6c --- /dev/null +++ b/crates/email/src/push/serialize.rs @@ -0,0 +1,23 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use store::{Deserialize, Serialize}; + +use super::PushSubscription; + +impl Serialize for PushSubscription { + fn serialize(self) -> Vec { + let todo = 1; + todo!() + } +} + +impl Deserialize for PushSubscription { + fn deserialize(bytes: &[u8]) -> trc::Result { + let todo = 1; + todo!() + } +} diff --git a/crates/email/src/sieve/index.rs b/crates/email/src/sieve/index.rs new file mode 100644 index 00000000..49215942 --- /dev/null +++ b/crates/email/src/sieve/index.rs @@ -0,0 +1,33 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use jmap_proto::{ + object::index::{IndexValue, IndexableObject}, + types::property::Property, +}; + +use super::SieveScript; + +impl IndexableObject for SieveScript { + fn index_values(&self) -> impl Iterator> { + [ + IndexValue::Text { + field: Property::Name.into(), + value: self.name.as_str(), + tokenize: true, + index: true, + }, + IndexValue::U32 { + field: Property::IsActive.into(), + value: Some(self.is_active as u32), + }, + IndexValue::Quota { + used: self.blob_id.section.as_ref().map_or(0, |b| b.size as u32), + }, + ] + .into_iter() + } +} diff --git a/crates/email/src/sieve.rs b/crates/email/src/sieve/ingest.rs similarity index 84% rename from crates/email/src/sieve.rs rename to crates/email/src/sieve/ingest.rs index d81babf8..af4d9429 100644 --- a/crates/email/src/sieve.rs +++ b/crates/email/src/sieve/ingest.rs @@ -7,55 +7,38 @@ use std::{borrow::Cow, sync::Arc}; use crate::{ - delivery::AutogeneratedMessage, - ingest::{EmailIngest, IngestEmail, IngestSource, IngestedEmail}, - mailbox::{MailboxFnc, INBOX_ID, TRASH_ID}, + mailbox::{INBOX_ID, TRASH_ID, manage::MailboxFnc}, + message::{ + delivery::AutogeneratedMessage, + ingest::{EmailIngest, IngestEmail, IngestSource, IngestedEmail}, + }, }; -use common::{auth::AccessToken, scripts::plugins::PluginContext, Server}; -use directory::{backend::internal::PrincipalField, Permission, QueryBy}; -use jmap_proto::{ - object::Object, - types::{collection::Collection, id::Id, keyword::Keyword, property::Property, value::Value}, +use common::{ + Server, auth::AccessToken, config::jmap::settings::SpecialUse, scripts::plugins::PluginContext, }; +use directory::{Permission, QueryBy, backend::internal::PrincipalField}; +use jmap_proto::types::{collection::Collection, id::Id, keyword::Keyword, property::Property}; use mail_parser::MessageParser; -use serde::ser::SerializeSeq; use sieve::{Envelope, Event, Input, Mailbox, Recipient, Sieve}; use store::{ - ahash::AHashSet, - blake3, - query::Filter, - write::{assert::HashedValue, now, BatchBuilder, Bincode, BlobOp, F_VALUE}, Deserialize, Serialize, + ahash::AHashSet, + query::Filter, + write::{BatchBuilder, Bincode, BlobOp, F_VALUE, assert::HashedValue, now}, }; use trc::{AddContext, SieveEvent}; +use utils::config::utils::ParseValue; use std::future::Future; +use super::{ActiveScript, SeenIdHash, SeenIds, SieveScript}; + struct SieveMessage<'x> { pub raw_message: Cow<'x, [u8]>, pub file_into: Vec, pub flags: Vec, } -pub struct ActiveScript { - pub document_id: u32, - pub script_name: String, - pub script: Arc, - pub seen_ids: SeenIds, -} - -#[derive(Debug, Clone)] -pub struct SeenIdHash { - hash: [u8; 32], - expiry: u64, -} - -#[derive(Debug, Clone, Default)] -pub struct SeenIds { - pub ids: AHashSet, - pub has_changes: bool, -} - pub trait SieveScriptIngest: Sync + Send { #[allow(clippy::too_many_arguments)] fn sieve_script_ingest( @@ -84,7 +67,7 @@ pub trait SieveScriptIngest: Sync + Send { &self, account_id: u32, document_id: u32, - ) -> impl Future)>> + Send; + ) -> impl Future> + Send; } impl SieveScriptIngest for Server { @@ -201,14 +184,14 @@ impl SieveScriptIngest for Server { TRASH_ID } else { let mut mailbox_id = u32::MAX; - let role = role.to_ascii_lowercase(); - if is_valid_role(&role) { + if let Ok(role) = SpecialUse::parse_value(&role) { if let Ok(Some(mailbox_id_)) = - self.mailbox_get_by_role(account_id, &role).await + self.mailbox_get_by_role(account_id, role).await { mailbox_id = mailbox_id_; } } + mailbox_id }); } @@ -246,10 +229,11 @@ impl SieveScriptIngest for Server { if !role.eq_ignore_ascii_case("inbox") && !role.eq_ignore_ascii_case("trash") { - let role = role.to_ascii_lowercase(); - if !is_valid_role(&role) + let role = SpecialUse::parse_value(&role); + if role.is_err() || !matches!( - self.mailbox_get_by_role(account_id, &role).await, + self.mailbox_get_by_role(account_id, role.unwrap()) + .await, Ok(Some(_)) ) { @@ -325,14 +309,11 @@ impl SieveScriptIngest for Server { target_id = INBOX_ID; } else if special_use.eq_ignore_ascii_case("trash") { target_id = TRASH_ID; - } else { - let role = special_use.to_ascii_lowercase(); - if is_valid_role(&role) { - if let Ok(Some(mailbox_id_)) = - self.mailbox_get_by_role(account_id, &role).await - { - target_id = mailbox_id_; - } + } else if let Ok(role) = SpecialUse::parse_value(&special_use) { + if let Ok(Some(mailbox_id_)) = + self.mailbox_get_by_role(account_id, role).await + { + target_id = mailbox_id_; } } } @@ -587,16 +568,12 @@ impl SieveScriptIngest for Server { .results .min() { - let (script, mut script_object) = + let (script, script_object) = self.sieve_script_compile(account_id, document_id).await?; Ok(Some(ActiveScript { document_id, script: Arc::new(script), - script_name: script_object - .properties - .remove(&Property::Name) - .and_then(|name| name.try_unwrap_string()) - .unwrap_or_else(|| account_id.to_string()), + script_name: script_object.name, seen_ids: self .get_property::>( account_id, @@ -644,10 +621,10 @@ impl SieveScriptIngest for Server { &self, account_id: u32, document_id: u32, - ) -> trc::Result<(Sieve, Object)> { + ) -> trc::Result<(Sieve, SieveScript)> { // Obtain script object let script_object = self - .get_property::>>( + .get_property::>( account_id, Collection::SieveScript, document_id, @@ -662,18 +639,17 @@ impl SieveScriptIngest for Server { })?; // Obtain the sieve script length - let (script_offset, blob_id) = script_object - .inner - .properties - .get(&Property::BlobId) - .and_then(|v| v.as_blob_id()) - .and_then(|v| (v.section.as_ref()?.size, v).into()) + let blob_id = &script_object.inner.blob_id; + let script_offset = blob_id + .section + .as_ref() .ok_or_else(|| { trc::StoreEvent::NotFound .into_err() .caused_by(trc::location!()) .document_id(document_id) - })?; + })? + .size; // Obtain the sieve script blob let script_bytes = self @@ -722,7 +698,7 @@ impl SieveScriptIngest for Server { .await? .hash; let mut new_script_object = script_object.inner.clone(); - new_script_object.set(Property::BlobId, new_blob_id.clone()); + new_script_object.blob_id = new_blob_id.clone(); // Update script object let mut batch = BatchBuilder::new(); @@ -757,6 +733,7 @@ impl SieveScriptIngest for Server { } } +/* #[inline(always)] pub fn is_valid_role(role: &str) -> bool { [ @@ -771,107 +748,4 @@ pub fn is_valid_role(role: &str) -> bool { ] .contains(&role) } - -impl SeenIdHash { - pub fn new(id: &str, expiry: u64) -> Self { - let mut hasher = blake3::Hasher::new(); - hasher.update(id.as_bytes()); - SeenIdHash { - hash: hasher.finalize().into(), - expiry, - } - } -} - -impl PartialOrd for SeenIdHash { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for SeenIdHash { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.expiry.cmp(&other.expiry) - } -} - -impl std::hash::Hash for SeenIdHash { - fn hash(&self, state: &mut H) { - self.hash.hash(state); - } -} - -impl PartialEq for SeenIdHash { - fn eq(&self, other: &Self) -> bool { - self.hash == other.hash - } -} - -impl Eq for SeenIdHash {} - -// SeenIds serializer -impl serde::Serialize for SeenIds { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - let mut seq = serializer.serialize_seq((self.ids.len() * 2).into())?; - for id in &self.ids { - seq.serialize_element(&id.expiry)?; - seq.serialize_element(&id.hash)?; - } - - seq.end() - } -} - -impl<'de> serde::Deserialize<'de> for SeenIds { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - deserializer.deserialize_seq(SeenIdsVisitor) - } -} - -struct SeenIdsVisitor; - -impl<'de> serde::de::Visitor<'de> for SeenIdsVisitor { - type Value = SeenIds; - - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { - formatter.write_str("invalid SeenIds") - } - - fn visit_seq(self, mut seq: A) -> Result - where - A: serde::de::SeqAccess<'de>, - { - let num_entries = seq.size_hint().unwrap_or(0) / 2; - let mut seen_ids = SeenIds { - ids: AHashSet::with_capacity(num_entries), - has_changes: false, - }; - let now = now(); - - for _ in 0..num_entries { - let expiry = seq - .next_element::()? - .ok_or_else(|| serde::de::Error::custom("Expected expiry."))?; - if expiry > now { - seen_ids.ids.insert(SeenIdHash { - hash: seq - .next_element()? - .ok_or_else(|| serde::de::Error::custom("Expected hash."))?, - expiry, - }); - } else { - seq.next_element::<[u8; 32]>()? - .ok_or_else(|| serde::de::Error::custom("Expected hash."))?; - seen_ids.has_changes = true; - } - } - - Ok(seen_ids) - } -} +*/ diff --git a/crates/email/src/sieve/mod.rs b/crates/email/src/sieve/mod.rs new file mode 100644 index 00000000..1b046020 --- /dev/null +++ b/crates/email/src/sieve/mod.rs @@ -0,0 +1,118 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::sync::Arc; + +use jmap_proto::types::blob::BlobId; +use sieve::Sieve; +use store::{ahash::AHashSet, blake3}; + +pub mod index; +pub mod ingest; +pub mod serialize; + +pub struct ActiveScript { + pub document_id: u32, + pub script_name: String, + pub script: Arc, + pub seen_ids: SeenIds, +} + +#[derive(Debug, Clone)] +pub struct SeenIdHash { + hash: [u8; 32], + expiry: u64, +} + +#[derive(Debug, Clone, Default)] +pub struct SeenIds { + pub ids: AHashSet, + pub has_changes: bool, +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct SieveScript { + pub name: String, + pub is_active: bool, + pub blob_id: BlobId, + pub vacation_response: Option, +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct VacationResponse { + pub from_date: Option, + pub to_date: Option, + pub subject: Option, + pub text_body: Option, + pub html_body: Option, +} + +impl SieveScript { + pub fn new(name: impl Into, blob_id: BlobId) -> Self { + SieveScript { + name: name.into(), + is_active: false, + blob_id, + vacation_response: None, + } + } + + pub fn with_name(mut self, name: impl Into) -> Self { + self.name = name.into(); + self + } + + pub fn with_blob_id(mut self, blob_id: BlobId) -> Self { + self.blob_id = blob_id; + self + } + + pub fn with_is_active(mut self, is_active: bool) -> Self { + self.is_active = is_active; + self + } + + pub fn set_is_active(&mut self, is_active: bool) { + self.is_active = is_active; + } +} + +impl SeenIdHash { + pub fn new(id: &str, expiry: u64) -> Self { + let mut hasher = blake3::Hasher::new(); + hasher.update(id.as_bytes()); + SeenIdHash { + hash: hasher.finalize().into(), + expiry, + } + } +} + +impl PartialOrd for SeenIdHash { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for SeenIdHash { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.expiry.cmp(&other.expiry) + } +} + +impl std::hash::Hash for SeenIdHash { + fn hash(&self, state: &mut H) { + self.hash.hash(state); + } +} + +impl PartialEq for SeenIdHash { + fn eq(&self, other: &Self) -> bool { + self.hash == other.hash + } +} + +impl Eq for SeenIdHash {} diff --git a/crates/email/src/sieve/serialize.rs b/crates/email/src/sieve/serialize.rs new file mode 100644 index 00000000..044b2161 --- /dev/null +++ b/crates/email/src/sieve/serialize.rs @@ -0,0 +1,95 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use serde::ser::SerializeSeq; +use store::{Deserialize, Serialize, ahash::AHashSet, write::now}; + +use super::{SeenIdHash, SeenIds, SieveScript}; + +// SeenIds serializer +impl serde::Serialize for SeenIds { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let mut seq = serializer.serialize_seq((self.ids.len() * 2).into())?; + for id in &self.ids { + seq.serialize_element(&id.expiry)?; + seq.serialize_element(&id.hash)?; + } + + seq.end() + } +} + +impl<'de> serde::Deserialize<'de> for SeenIds { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_seq(SeenIdsVisitor) + } +} + +struct SeenIdsVisitor; + +impl<'de> serde::de::Visitor<'de> for SeenIdsVisitor { + type Value = SeenIds; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("invalid SeenIds") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let num_entries = seq.size_hint().unwrap_or(0) / 2; + let mut seen_ids = SeenIds { + ids: AHashSet::with_capacity(num_entries), + has_changes: false, + }; + let now = now(); + + for _ in 0..num_entries { + let expiry = seq + .next_element::()? + .ok_or_else(|| serde::de::Error::custom("Expected expiry."))?; + if expiry > now { + seen_ids.ids.insert(SeenIdHash { + hash: seq + .next_element()? + .ok_or_else(|| serde::de::Error::custom("Expected hash."))?, + expiry, + }); + } else { + seq.next_element::<[u8; 32]>()? + .ok_or_else(|| serde::de::Error::custom("Expected hash."))?; + seen_ids.has_changes = true; + } + } + + Ok(seen_ids) + } +} + +impl Serialize for SieveScript { + fn serialize(self) -> Vec { + todo!() + } +} + +impl Serialize for &SieveScript { + fn serialize(self) -> Vec { + todo!() + } +} + +impl Deserialize for SieveScript { + fn deserialize(bytes: &[u8]) -> trc::Result { + todo!() + } +} diff --git a/crates/email/src/submission/index.rs b/crates/email/src/submission/index.rs new file mode 100644 index 00000000..c8c98417 --- /dev/null +++ b/crates/email/src/submission/index.rs @@ -0,0 +1,42 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + + use jmap_proto::{ + object::index::{IndexValue, IndexableObject}, + types::property::Property, +}; + +use super::EmailSubmission; + +impl IndexableObject for EmailSubmission { + fn index_values(&self) -> impl Iterator> { + [ + IndexValue::Text { + field: Property::UndoStatus.into(), + value: self.undo_status.as_index(), + tokenize: false, + index: true, + }, + IndexValue::U32 { + field: Property::EmailId.into(), + value: Some(self.email_id), + }, + IndexValue::U32 { + field: Property::ThreadId.into(), + value: Some(self.thread_id), + }, + IndexValue::U32 { + field: Property::IdentityId.into(), + value: Some(self.identity_id), + }, + IndexValue::U64 { + field: Property::SendAt.into(), + value: Some(self.send_at), + }, + ] + .into_iter() + } +} diff --git a/crates/email/src/submission/mod.rs b/crates/email/src/submission/mod.rs new file mode 100644 index 00000000..278537e4 --- /dev/null +++ b/crates/email/src/submission/mod.rs @@ -0,0 +1,96 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use utils::map::vec_map::VecMap; + +pub mod index; +pub mod serialize; + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct EmailSubmission { + pub email_id: u32, + pub thread_id: u32, + pub identity_id: u32, + pub send_at: u64, + pub queue_id: Option, + pub undo_status: UndoStatus, + pub envelope: Envelope, + pub delivery_status: VecMap, +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct Envelope { + pub mail_from: Address, + pub rcpt_to: Vec
, +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct Address { + pub email: String, + pub parameters: Option>>, +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct DeliveryStatus { + pub smtp_reply: String, + pub delivered: Delivered, + pub displayed: bool, +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub enum Delivered { + Queued, + Yes, + No, + #[default] + Unknown, +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub enum UndoStatus { + #[default] + Pending, + Final, + Canceled, +} + +impl UndoStatus { + pub fn parse(s: &str) -> Option { + hashify::tiny_map!(s.as_bytes(), + "pending" => UndoStatus::Pending, + "final" => UndoStatus::Final, + "canceled" => UndoStatus::Canceled, + "cancelled" => UndoStatus::Canceled, + ) + } + + pub fn as_str(&self) -> &'static str { + match self { + UndoStatus::Pending => "pending", + UndoStatus::Final => "final", + UndoStatus::Canceled => "canceled", + } + } + + pub fn as_index(&self) -> &'static str { + match self { + UndoStatus::Pending => "p", + UndoStatus::Final => "f", + UndoStatus::Canceled => "c", + } + } +} + +impl Delivered { + pub fn as_str(&self) -> &'static str { + match self { + Delivered::Queued => "queued", + Delivered::Yes => "yes", + Delivered::No => "no", + Delivered::Unknown => "unknown", + } + } +} diff --git a/crates/email/src/submission/serialize.rs b/crates/email/src/submission/serialize.rs new file mode 100644 index 00000000..bf3114c2 --- /dev/null +++ b/crates/email/src/submission/serialize.rs @@ -0,0 +1,23 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use store::{Deserialize, Serialize}; + +use super::EmailSubmission; + +impl Serialize for EmailSubmission { + fn serialize(self) -> Vec { + let todo = 1; + todo!() + } +} + +impl Deserialize for EmailSubmission { + fn deserialize(bytes: &[u8]) -> trc::Result { + let todo = 1; + todo!() + } +} diff --git a/crates/email/src/cache.rs b/crates/email/src/thread/cache.rs similarity index 100% rename from crates/email/src/cache.rs rename to crates/email/src/thread/cache.rs diff --git a/crates/email/src/thread/mod.rs b/crates/email/src/thread/mod.rs new file mode 100644 index 00000000..e785c447 --- /dev/null +++ b/crates/email/src/thread/mod.rs @@ -0,0 +1,7 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod cache; diff --git a/crates/imap-proto/src/parser/create.rs b/crates/imap-proto/src/parser/create.rs index 120b31c7..e72a1d0d 100644 --- a/crates/imap-proto/src/parser/create.rs +++ b/crates/imap-proto/src/parser/create.rs @@ -6,7 +6,7 @@ use crate::{ Command, - protocol::{ProtocolVersion, create}, + protocol::{ProtocolVersion, create, list::Attribute}, receiver::{Request, Token, bad}, utf7::utf7_maybe_decode, }; @@ -39,12 +39,12 @@ impl Request { match tokens.next() { Some(Token::Argument(value)) => { let r = hashify::tiny_map_ignore_case!(value.as_slice(), - "\\Archive" => Some("archive"), - "\\Drafts" => Some("drafts"), - "\\Junk" => Some("junk"), - "\\Sent" => Some("sent"), - "\\Trash" => Some("trash"), - "\\Important" => Some("important"), + "\\Archive" => Some(Attribute::Archive), + "\\Drafts" => Some(Attribute::Drafts), + "\\Junk" => Some(Attribute::Junk), + "\\Sent" => Some(Attribute::Sent), + "\\Trash" => Some(Attribute::Trash), + "\\Important" => Some(Attribute::Important), "\\All" => None, ); @@ -90,7 +90,7 @@ impl Request { mod tests { use crate::{ - protocol::{ProtocolVersion, create}, + protocol::{ProtocolVersion, create, list::Attribute}, receiver::Receiver, }; @@ -120,7 +120,7 @@ mod tests { create::Arguments { tag: "t1".to_string(), mailbox_name: "Important Messages".to_string(), - mailbox_role: Some("important"), + mailbox_role: Some(Attribute::Important), }, ), ( diff --git a/crates/imap-proto/src/protocol/create.rs b/crates/imap-proto/src/protocol/create.rs index dad0ce97..32962541 100644 --- a/crates/imap-proto/src/protocol/create.rs +++ b/crates/imap-proto/src/protocol/create.rs @@ -4,9 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::list::Attribute; + #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { pub tag: String, pub mailbox_name: String, - pub mailbox_role: Option<&'static str>, + pub mailbox_role: Option, } diff --git a/crates/imap-proto/src/protocol/list.rs b/crates/imap-proto/src/protocol/list.rs index 8d0c2c42..9dede9f7 100644 --- a/crates/imap-proto/src/protocol/list.rs +++ b/crates/imap-proto/src/protocol/list.rs @@ -7,9 +7,8 @@ use crate::utf7::utf7_encode; use super::{ - quoted_string, + ImapResponse, quoted_string, status::{Status, StatusItem}, - ImapResponse, }; #[derive(Debug, Clone, PartialEq, Eq)] @@ -277,8 +276,8 @@ impl ImapResponse for Response { #[cfg(test)] mod tests { use crate::protocol::{ - status::{Status, StatusItem, StatusItemType}, ImapResponse, + status::{Status, StatusItem, StatusItemType}, }; use super::{Attribute, ChildInfo, ListItem, Tag}; diff --git a/crates/imap/src/core/mailbox.rs b/crates/imap/src/core/mailbox.rs index ac54f58d..b02b4b5e 100644 --- a/crates/imap/src/core/mailbox.rs +++ b/crates/imap/src/core/mailbox.rs @@ -1,26 +1,23 @@ use std::{ collections::BTreeMap, - sync::{atomic::Ordering, Arc}, + sync::{Arc, atomic::Ordering}, }; use ahash::AHashMap; use common::{ + AccountId, Mailbox, auth::AccessToken, config::jmap::settings::SpecialUse, - listener::{limiter::InFlight, SessionStream}, - AccountId, Mailbox, + listener::{SessionStream, limiter::InFlight}, }; -use directory::{backend::internal::PrincipalField, QueryBy}; -use email::mailbox::{MailboxFnc, INBOX_ID}; +use directory::{QueryBy, backend::internal::PrincipalField}; +use email::mailbox::{INBOX_ID, manage::MailboxFnc}; use imap_proto::protocol::list::Attribute; use jmap::{ auth::acl::{AclMethods, EffectiveAcl}, changes::get::ChangesLookup, }; -use jmap_proto::{ - object::Object, - types::{acl::Acl, collection::Collection, id::Id, property::Property, value::Value}, -}; +use jmap_proto::types::{acl::Acl, collection::Collection, id::Id, property::Property}; use parking_lot::Mutex; use store::query::log::{Change, Query}; use trc::AddContext; @@ -46,10 +43,12 @@ impl SessionData { let access_token = session.access_token.clone(); // Fetch mailboxes for the main account - let mut mailboxes = vec![session - .fetch_account_mailboxes(session.account_id, None, &access_token) - .await - .caused_by(trc::location!())?]; + let mut mailboxes = vec![ + session + .fetch_account_mailboxes(session.account_id, None, &access_token) + .await + .caused_by(trc::location!())?, + ]; // Fetch shared mailboxes for &account_id in access_token.shared_accounts(Collection::Mailbox) { @@ -146,9 +145,9 @@ impl SessionData { // Fetch mailboxes let mut mailboxes = Vec::with_capacity(10); let mut special_uses = AHashMap::new(); - for (mailbox_id, values) in self + for (mailbox_id, mailbox) in self .server - .get_properties::, _, _>( + .get_properties::( account_id, Collection::Mailbox, &mailbox_ids, @@ -158,34 +157,12 @@ impl SessionData { .caused_by(trc::location!())? { // Map special uses - if let Some(Value::Text(role)) = values.properties.get(&Property::Role) { - let special_use = match role.as_str() { - "archive" => SpecialUse::Archive, - "drafts" => SpecialUse::Drafts, - "junk" => SpecialUse::Junk, - "sent" => SpecialUse::Sent, - "trash" => SpecialUse::Trash, - "inbox" => SpecialUse::Inbox, - _ => SpecialUse::None, - }; - if special_use != SpecialUse::None { - special_uses.insert(special_use, mailbox_id); - } + if mailbox.role != SpecialUse::None { + special_uses.insert(mailbox.role, mailbox_id); } // Add mailbox id - mailboxes.push(( - mailbox_id, - values - .properties - .get(&Property::ParentId) - .map(|parent_id| match parent_id { - Value::Id(value) => value.document_id(), - _ => 0, - }) - .unwrap_or(0), - values, - )); + mailboxes.push((mailbox_id, mailbox.parent_id, mailbox)); } // Build tree @@ -228,13 +205,7 @@ impl SessionData { if *mailbox_parent_id == parent_id { let mut mailbox_path = path.clone(); if *mailbox_id != INBOX_ID || account.prefix.is_some() { - mailbox_path.push( - mailbox - .get(&Property::Name) - .as_string() - .unwrap_or_default() - .to_string(), - ); + mailbox_path.push(mailbox.name.clone()); } else { mailbox_path.push("INBOX".to_string()); } @@ -246,21 +217,16 @@ impl SessionData { *mailbox_id, Mailbox { has_children, - is_subscribed: mailbox - .properties - .get(&Property::IsSubscribed) - .map(|parent_id| match parent_id { - Value::List(values) => values - .contains(&Value::Id(access_token.primary_id().into())), - _ => false, - }) - .unwrap_or(false), - special_use: mailbox.properties.get(&Property::Role).and_then( - |parent_id| match parent_id { - Value::Text(role) => Attribute::try_from(role.as_str()).ok(), - _ => None, - }, - ), + is_subscribed: mailbox.is_subscribed(access_token.primary_id()), + special_use: match mailbox.role { + SpecialUse::Trash => Some(Attribute::Trash), + SpecialUse::Junk => Some(Attribute::Junk), + SpecialUse::Drafts => Some(Attribute::Drafts), + SpecialUse::Archive => Some(Attribute::Archive), + SpecialUse::Sent => Some(Attribute::Sent), + SpecialUse::Important => Some(Attribute::Important), + _ => None, + }, total_messages: self .server .get_tag( @@ -606,7 +572,7 @@ impl SessionData { if account .prefix .as_ref() - .is_none_or( |p| mailbox_name.starts_with(p)) + .is_none_or(|p| mailbox_name.starts_with(p)) { for (mailbox_name_, mailbox_id_) in account.mailbox_names.iter() { if (!is_inbox && mailbox_name_ == mailbox_name) @@ -634,14 +600,14 @@ impl SessionData { Ok(access_token.is_member(account_id) || self .server - .get_property::>( + .get_property::( account_id, Collection::Mailbox, document_id, Property::Value, ) .await? - .map(|mailbox| mailbox.effective_acl(&access_token).contains(item)) + .map(|mailbox| mailbox.acls.effective_acl(&access_token).contains(item)) .ok_or_else(|| { trc::ImapEvent::Error .caused_by(trc::location!()) diff --git a/crates/imap/src/core/message.rs b/crates/imap/src/core/message.rs index 6f37a2a2..0bea7b3e 100644 --- a/crates/imap/src/core/message.rs +++ b/crates/imap/src/core/message.rs @@ -10,14 +10,8 @@ use ahash::AHashMap; use common::{NextMailboxState, listener::SessionStream}; use email::mailbox::UidMailbox; use imap_proto::protocol::{Sequence, expunge, select::Exists}; -use jmap_proto::{ - object::Object, - types::{collection::Collection, property::Property, value::Value}, -}; -use store::{ - ValueKey, - write::{ValueClass, assert::HashedValue}, -}; +use jmap_proto::types::{collection::Collection, property::Property}; +use store::write::assert::HashedValue; use trc::AddContext; use crate::core::ImapId; @@ -231,14 +225,13 @@ impl SessionData { pub async fn get_uid_validity(&self, mailbox: &MailboxId) -> trc::Result { self.server - .get_property::>( + .get_property::( mailbox.account_id, Collection::Mailbox, mailbox.mailbox_id, &Property::Value, ) .await? - .and_then(|obj| obj.get(&Property::Cid).as_uint()) .ok_or_else(|| { trc::ImapEvent::Error .caused_by(trc::location!()) @@ -247,7 +240,7 @@ impl SessionData { .collection(Collection::Mailbox) .document_id(mailbox.mailbox_id) }) - .map(|v| v as u32) + .map(|m| m.uid_validity) } pub async fn get_uid_next(&self, mailbox: &MailboxId) -> trc::Result { diff --git a/crates/imap/src/op/acl.rs b/crates/imap/src/op/acl.rs index 9f9e3595..f758ffcf 100644 --- a/crates/imap/src/op/acl.rs +++ b/crates/imap/src/op/acl.rs @@ -6,33 +6,28 @@ use std::{sync::Arc, time::Instant}; -use common::{auth::AccessToken, listener::SessionStream, MailboxId}; +use common::{MailboxId, auth::AccessToken, listener::SessionStream}; use directory::{ - backend::internal::{manage::ChangedPrincipals, PrincipalField}, Permission, QueryBy, Type, + backend::internal::{PrincipalField, manage::ChangedPrincipals}, }; -use email::mailbox::SCHEMA; use imap_proto::{ + Command, ResponseCode, StatusResponse, protocol::acl::{ Arguments, GetAclResponse, ListRightsResponse, ModRightsOp, MyRightsResponse, Rights, }, receiver::Request, - Command, ResponseCode, StatusResponse, }; use jmap::auth::acl::EffectiveAcl; use jmap_proto::{ - object::{index::ObjectIndexBuilder, Object}, + object::index::ObjectIndexBuilder, types::{ - acl::Acl, - collection::Collection, - property::Property, - state::StateChange, - type_state::DataType, - value::{AclGrant, Value}, + acl::Acl, collection::Collection, property::Property, state::StateChange, + type_state::DataType, value::AclGrant, }, }; -use store::write::{assert::HashedValue, log::ChangeLogBuilder, BatchBuilder}; +use store::write::{BatchBuilder, assert::HashedValue, log::ChangeLogBuilder}; use trc::AddContext; use utils::map::bitmap::Bitmap; @@ -53,70 +48,64 @@ impl Session { let data = self.state.session_data(); spawn_op!(data, { - let (mailbox, values, _) = data + let (mailbox_id, mailbox, _) = data .get_acl_mailbox(&arguments, true) .await .imap_ctx(&arguments.tag, trc::location!())?; let mut permissions = Vec::new(); - if let Some(acls) = values - .inner - .properties - .get(&Property::Acl) - .and_then(|v| v.as_acl()) - { - for item in acls { - if let Some(account_name) = data - .server - .core - .storage - .directory - .query(QueryBy::Id(item.account_id), false) - .await - .imap_ctx(&arguments.tag, trc::location!())? - .and_then(|mut p| p.take_str(PrincipalField::Name)) - { - let mut rights = Vec::new(); - for acl in item.grants { - match acl { - Acl::Read => { - rights.push(Rights::Lookup); - } - Acl::Modify => { - rights.push(Rights::CreateMailbox); - } - Acl::Delete => { - rights.push(Rights::DeleteMailbox); - } - Acl::ReadItems => { - rights.push(Rights::Read); - } - Acl::AddItems => { - rights.push(Rights::Insert); - } - Acl::ModifyItems => { - rights.push(Rights::Write); - rights.push(Rights::Seen); - } - Acl::RemoveItems => { - rights.push(Rights::DeleteMessages); - rights.push(Rights::Expunge); - } - Acl::CreateChild => { - rights.push(Rights::CreateMailbox); - } - Acl::Administer => { - rights.push(Rights::Administer); - } - Acl::Submit => { - rights.push(Rights::Post); - } - Acl::None => (), + for item in mailbox.inner.acls { + if let Some(account_name) = data + .server + .core + .storage + .directory + .query(QueryBy::Id(item.account_id), false) + .await + .imap_ctx(&arguments.tag, trc::location!())? + .and_then(|mut p| p.take_str(PrincipalField::Name)) + { + let mut rights = Vec::new(); + + for acl in item.grants { + match acl { + Acl::Read => { + rights.push(Rights::Lookup); } + Acl::Modify => { + rights.push(Rights::CreateMailbox); + } + Acl::Delete => { + rights.push(Rights::DeleteMailbox); + } + Acl::ReadItems => { + rights.push(Rights::Read); + } + Acl::AddItems => { + rights.push(Rights::Insert); + } + Acl::ModifyItems => { + rights.push(Rights::Write); + rights.push(Rights::Seen); + } + Acl::RemoveItems => { + rights.push(Rights::DeleteMessages); + rights.push(Rights::Expunge); + } + Acl::CreateChild => { + rights.push(Rights::CreateMailbox); + } + Acl::Administer => { + rights.push(Rights::Administer); + } + Acl::Submit => { + rights.push(Rights::Post); + } + Acl::None => (), } - - permissions.push((account_name, rights)); } + + permissions.push((account_name, rights)); } } @@ -124,8 +113,8 @@ impl Session { Imap(trc::ImapEvent::GetAcl), SpanId = data.session_id, MailboxName = arguments.mailbox_name.clone(), - AccountId = mailbox.account_id, - MailboxId = mailbox.mailbox_id, + AccountId = mailbox_id.account_id, + MailboxId = mailbox_id.mailbox_id, Total = permissions.len(), Elapsed = op_start.elapsed() ); @@ -160,7 +149,7 @@ impl Session { .await .imap_ctx(&arguments.tag, trc::location!())?; let rights = if access_token.is_shared(mailbox.account_id) { - let acl = values.inner.effective_acl(&access_token); + let acl = values.inner.acls.effective_acl(&access_token); let mut rights = Vec::with_capacity(5); if acl.contains(Acl::ReadItems) { rights.push(Rights::Read); @@ -241,7 +230,7 @@ impl Session { spawn_op!(data, { // Validate mailbox - let (mailbox, values, _) = data + let (mailbox_id, current_mailbox, _) = data .get_acl_mailbox(&arguments, false) .await .imap_ctx(&arguments.tag, trc::location!())?; @@ -265,7 +254,7 @@ impl Session { .id(); // Prepare changes - let mut changes = Object::with_capacity(1); + let mut mailbox = current_mailbox.inner.clone(); let (op, rights) = arguments .mod_rights .map(|mr| { @@ -275,27 +264,9 @@ impl Session { ) }) .unwrap_or_else(|| (ModRightsOp::Replace, Bitmap::new())); - let acl = if let Value::Acl(acl) = - changes - .properties - .get_mut_or_insert_with(Property::Acl, || { - values - .inner - .properties - .get(&Property::Acl) - .cloned() - .unwrap_or_else(|| Value::Acl(Vec::new())) - }) { - acl - } else { - return Err(trc::StoreEvent::DataCorruption - .into_err() - .id(arguments.tag) - .ctx(trc::Key::Reason, "Invalid mailbox ACL") - .caused_by(trc::location!())); - }; - if let Some(item) = acl + if let Some(item) = mailbox + .acls .iter_mut() .find(|item| item.account_id == acl_account_id) { @@ -304,7 +275,9 @@ impl Session { if !rights.is_empty() { item.grants = rights; } else { - acl.retain(|item| item.account_id != acl_account_id); + mailbox + .acls + .retain(|item| item.account_id != acl_account_id); } } ModRightsOp::Add => { @@ -315,14 +288,16 @@ impl Session { item.grants.remove(right); } if item.grants.is_empty() { - acl.retain(|item| item.account_id != acl_account_id); + mailbox + .acls + .retain(|item| item.account_id != acl_account_id); } } } } else if !rights.is_empty() { match op { ModRightsOp::Add | ModRightsOp::Replace => { - acl.push(AclGrant { + mailbox.acls.push(AclGrant { account_id: acl_account_id, grants: rights, }); @@ -331,22 +306,22 @@ impl Session { } } - let grants = acl + let grants = mailbox + .acls .iter() .map(|r| trc::Value::from(r.account_id)) .collect::>(); // Write changes - let mailbox_id = mailbox.mailbox_id; let mut batch = BatchBuilder::new(); batch - .with_account_id(mailbox.account_id) + .with_account_id(mailbox_id.account_id) .with_collection(Collection::Mailbox) - .update_document(mailbox_id) + .update_document(mailbox_id.mailbox_id) .custom( - ObjectIndexBuilder::new(SCHEMA) - .with_changes(changes) - .with_current(values), + ObjectIndexBuilder::new() + .with_changes(mailbox) + .with_current(current_mailbox), ); if !batch.is_empty() { data.server @@ -355,15 +330,15 @@ impl Session { .await .imap_ctx(&arguments.tag, trc::location!())?; let mut changes = ChangeLogBuilder::new(); - changes.log_update(Collection::Mailbox, mailbox_id); + changes.log_update(Collection::Mailbox, mailbox_id.mailbox_id); let change_id = data .server - .commit_changes(mailbox.account_id, changes) + .commit_changes(mailbox_id.account_id, changes) .await .imap_ctx(&arguments.tag, trc::location!())?; data.server .broadcast_state_change( - StateChange::new(mailbox.account_id) + StateChange::new(mailbox_id.account_id) .with_change(DataType::Mailbox, change_id), ) .await; @@ -382,8 +357,8 @@ impl Session { Imap(trc::ImapEvent::SetAcl), SpanId = data.session_id, MailboxName = arguments.mailbox_name.clone(), - AccountId = mailbox.account_id, - MailboxId = mailbox.mailbox_id, + AccountId = mailbox_id.account_id, + MailboxId = mailbox_id.mailbox_id, Details = grants, Elapsed = op_start.elapsed() ); @@ -451,11 +426,15 @@ impl SessionData { &self, arguments: &Arguments, validate: bool, - ) -> trc::Result<(MailboxId, HashedValue>, Arc)> { + ) -> trc::Result<( + MailboxId, + HashedValue, + Arc, + )> { if let Some(mailbox) = self.get_mailbox_by_name(&arguments.mailbox_name) { if let Some(values) = self .server - .get_property::>>( + .get_property::>( mailbox.account_id, Collection::Mailbox, mailbox.mailbox_id, @@ -469,6 +448,7 @@ impl SessionData { || access_token.is_member(mailbox.account_id) || values .inner + .acls .effective_acl(&access_token) .contains(Acl::Administer) { diff --git a/crates/imap/src/op/append.rs b/crates/imap/src/op/append.rs index 73c54c4c..924173cb 100644 --- a/crates/imap/src/op/append.rs +++ b/crates/imap/src/op/append.rs @@ -7,18 +7,18 @@ use std::{sync::Arc, time::Instant}; use directory::Permission; -use email::ingest::{EmailIngest, IngestEmail, IngestSource}; +use email::message::ingest::{EmailIngest, IngestEmail, IngestSource}; use imap_proto::{ + Command, ResponseCode, StatusResponse, protocol::{append::Arguments, select::HighestModSeq}, receiver::Request, - Command, ResponseCode, StatusResponse, }; use crate::{ core::{ImapUidToId, SelectedMailbox, Session, SessionData}, spawn_op, }; -use common::{listener::SessionStream, MailboxId}; +use common::{MailboxId, listener::SessionStream}; use jmap_proto::types::{acl::Acl, keyword::Keyword, state::StateChange, type_state::DataType}; use mail_parser::MessageParser; diff --git a/crates/imap/src/op/copy_move.rs b/crates/imap/src/op/copy_move.rs index b3877187..262bfc56 100644 --- a/crates/imap/src/op/copy_move.rs +++ b/crates/imap/src/op/copy_move.rs @@ -8,8 +8,8 @@ use std::{sync::Arc, time::Instant}; use directory::Permission; use email::{ - ingest::EmailIngest, mailbox::{JUNK_ID, UidMailbox}, + message::{bayes::EmailBayesTrain, ingest::EmailIngest}, }; use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, protocol::copy_move::Arguments, @@ -21,7 +21,7 @@ use crate::{ spawn_op, }; use common::{MailboxId, listener::SessionStream}; -use jmap::email::{bayes::EmailBayesTrain, copy::EmailCopy, set::TagManager}; +use jmap::email::{copy::EmailCopy, set::TagManager}; use jmap_proto::{ error::set::SetErrorType, types::{ diff --git a/crates/imap/src/op/create.rs b/crates/imap/src/op/create.rs index 297581dc..ba2e7d52 100644 --- a/crates/imap/src/op/create.rs +++ b/crates/imap/src/op/create.rs @@ -11,20 +11,19 @@ use crate::{ op::ImapContext, spawn_op, }; -use common::{listener::SessionStream, Account, Mailbox}; +use common::{Account, Mailbox, config::jmap::settings::SpecialUse, listener::SessionStream}; use directory::Permission; -use email::mailbox::SCHEMA; use imap_proto::{ + Command, ResponseCode, StatusResponse, protocol::{create::Arguments, list::Attribute}, receiver::Request, - Command, ResponseCode, StatusResponse, }; use jmap::JmapMethods; use jmap_proto::{ - object::{index::ObjectIndexBuilder, Object}, + object::index::ObjectIndexBuilder, types::{ acl::Acl, collection::Collection, id::Id, property::Property, state::StateChange, - type_state::DataType, value::Value, + type_state::DataType, }, }; use store::{query::Filter, write::BatchBuilder}; @@ -83,16 +82,11 @@ impl SessionData { let mut parent_id = params.parent_mailbox_id.map(|id| id + 1).unwrap_or(0); let mut create_ids = Vec::with_capacity(params.path.len()); for (pos, &path_item) in params.path.iter().enumerate() { - let mut mailbox = Object::with_capacity(4) - .with_property(Property::Name, path_item) - .with_property(Property::ParentId, Value::Id(Id::from(parent_id))) - .with_property( - Property::Cid, - Value::UnsignedInt(rand::random::() as u64), - ); + let mut mailbox = email::mailbox::Mailbox::new(path_item).with_parent_id(parent_id); + if pos == params.path.len() - 1 { - if let Some(mailbox_role) = arguments.mailbox_role { - mailbox.set(Property::Role, mailbox_role); + if let Some(mailbox_role) = arguments.mailbox_role.map(attr_to_role) { + mailbox.role = mailbox_role; } } let mut batch = BatchBuilder::new(); @@ -100,7 +94,7 @@ impl SessionData { .with_account_id(params.account_id) .with_collection(Collection::Mailbox) .create_document() - .custom(ObjectIndexBuilder::new(SCHEMA).with_changes(mailbox)); + .custom(ObjectIndexBuilder::new().with_changes(mailbox)); let mailbox_id = self .server .store() @@ -248,7 +242,7 @@ impl SessionData { pub async fn validate_mailbox_create<'x>( &self, mailbox_name: &'x str, - mailbox_role: Option<&'x str>, + mailbox_role: Option, ) -> trc::Result> { // Remove leading and trailing separators let mut name = mailbox_name.trim(); @@ -392,12 +386,13 @@ impl SessionData { parent_mailbox_name, special_use: if let Some(mailbox_role) = mailbox_role { // Make sure role is unique + let role_name = attr_to_role(mailbox_role).as_str().unwrap_or_default(); if !self .server .filter( account_id, Collection::Mailbox, - vec![Filter::eq(Property::Role, mailbox_role)], + vec![Filter::eq(Property::Role, role_name)], ) .await .caused_by(trc::location!())? @@ -406,12 +401,10 @@ impl SessionData { { return Err(trc::ImapEvent::Error .into_err() - .details(format!( - "A mailbox with role '{mailbox_role}' already exists.", - )) + .details(format!("A mailbox with role '{role_name}' already exists.",)) .code(ResponseCode::UseAttr)); } - Attribute::try_from(mailbox_role).ok() + Some(mailbox_role) } else { None }, @@ -430,3 +423,16 @@ pub struct CreateParams<'x> { pub special_use: Option, pub is_rename: bool, } + +#[inline] +fn attr_to_role(attr: Attribute) -> SpecialUse { + match attr { + Attribute::Archive => SpecialUse::Archive, + Attribute::Drafts => SpecialUse::Drafts, + Attribute::Junk => SpecialUse::Junk, + Attribute::Sent => SpecialUse::Sent, + Attribute::Trash => SpecialUse::Trash, + Attribute::Important => SpecialUse::Important, + _ => SpecialUse::None, + } +} diff --git a/crates/imap/src/op/expunge.rs b/crates/imap/src/op/expunge.rs index a32599c3..8c8817fc 100644 --- a/crates/imap/src/op/expunge.rs +++ b/crates/imap/src/op/expunge.rs @@ -8,24 +8,24 @@ use std::{sync::Arc, time::Instant}; use ahash::AHashMap; use directory::Permission; -use email::mailbox::UidMailbox; +use email::{mailbox::UidMailbox, message::delete::EmailDeletion}; use imap_proto::{ + Command, ResponseCode, ResponseType, StatusResponse, parser::parse_sequence_set, receiver::{Request, Token}, - Command, ResponseCode, ResponseType, StatusResponse, }; use trc::AddContext; use crate::core::{SavedSearch, SelectedMailbox, Session, SessionData}; -use common::{listener::SessionStream, ImapId}; -use jmap::email::{delete::EmailDeletion, set::TagManager}; +use common::{ImapId, listener::SessionStream}; +use jmap::email::set::TagManager; use jmap_proto::types::{ acl::Acl, collection::Collection, id::Id, keyword::Keyword, property::Property, state::StateChange, type_state::DataType, }; use store::{ roaring::RoaringBitmap, - write::{assert::HashedValue, log::ChangeLogBuilder, BatchBuilder, F_VALUE}, + write::{BatchBuilder, F_VALUE, assert::HashedValue, log::ChangeLogBuilder}, }; use super::{ImapContext, ToModSeq}; diff --git a/crates/imap/src/op/fetch.rs b/crates/imap/src/op/fetch.rs index 6c14aa03..ae402724 100644 --- a/crates/imap/src/op/fetch.rs +++ b/crates/imap/src/op/fetch.rs @@ -13,19 +13,19 @@ use crate::{ use ahash::AHashMap; use common::listener::SessionStream; use directory::Permission; -use email::metadata::MessageMetadata; +use email::message::metadata::MessageMetadata; use imap_proto::{ + Command, ResponseCode, ResponseType, StatusResponse, parser::PushUnique, protocol::{ + Flag, expunge::Vanished, fetch::{ self, Arguments, Attribute, BodyContents, BodyPart, BodyPartExtension, BodyPartFields, DataItem, Envelope, FetchItem, Section, }, - Flag, }, receiver::Request, - Command, ResponseCode, ResponseType, StatusResponse, }; use jmap::{blob::download::BlobDownload, changes::get::ChangesLookup}; use jmap_proto::types::{ @@ -35,7 +35,7 @@ use jmap_proto::types::{ use mail_parser::{Address, GetHeader, HeaderName, Message, PartType}; use store::{ query::log::{Change, Query}, - write::{assert::HashedValue, BatchBuilder, Bincode, F_BITMAP, F_VALUE}, + write::{BatchBuilder, Bincode, F_BITMAP, F_VALUE, assert::HashedValue}, }; use trc::AddContext; diff --git a/crates/imap/src/op/rename.rs b/crates/imap/src/op/rename.rs index b0f53f43..e4aeb25c 100644 --- a/crates/imap/src/op/rename.rs +++ b/crates/imap/src/op/rename.rs @@ -12,19 +12,18 @@ use crate::{ }; use common::listener::SessionStream; use directory::Permission; -use email::mailbox::SCHEMA; use imap_proto::{ - protocol::rename::Arguments, receiver::Request, Command, ResponseCode, StatusResponse, + Command, ResponseCode, StatusResponse, protocol::rename::Arguments, receiver::Request, }; use jmap::auth::acl::EffectiveAcl; use jmap_proto::{ - object::{index::ObjectIndexBuilder, Object}, + object::index::ObjectIndexBuilder, types::{ - acl::Acl, collection::Collection, id::Id, property::Property, state::StateChange, - type_state::DataType, value::Value, + acl::Acl, collection::Collection, property::Property, state::StateChange, + type_state::DataType, }, }; -use store::write::{assert::HashedValue, BatchBuilder}; +use store::write::{BatchBuilder, assert::HashedValue}; use trc::AddContext; use super::ImapContext; @@ -94,7 +93,7 @@ impl SessionData { // Obtain mailbox let mailbox = self .server - .get_property::>>( + .get_property::>( params.account_id, Collection::Mailbox, mailbox_id, @@ -119,6 +118,7 @@ impl SessionData { if access_token.is_shared(params.account_id) && !mailbox .inner + .acls .effective_acl(&access_token) .contains(Acl::Modify) { @@ -146,17 +146,9 @@ impl SessionData { .with_account_id(params.account_id) .with_collection(Collection::Mailbox) .create_document() - .custom( - ObjectIndexBuilder::new(SCHEMA).with_changes( - Object::with_capacity(3) - .with_property(Property::Name, path_item) - .with_property(Property::ParentId, Value::Id(Id::from(parent_id))) - .with_property( - Property::Cid, - Value::UnsignedInt(rand::random::() as u64), - ), - ), - ); + .custom(ObjectIndexBuilder::new().with_changes( + email::mailbox::Mailbox::new(path_item).with_parent_id(parent_id), + )); let mailbox_id = self .server @@ -171,22 +163,18 @@ impl SessionData { } let mut batch = BatchBuilder::new(); + let mut new_mailbox = mailbox.inner.clone(); + new_mailbox.name = new_mailbox_name.to_string(); + new_mailbox.parent_id = parent_id; + new_mailbox.uid_validity = rand::random::(); batch .with_account_id(params.account_id) .with_collection(Collection::Mailbox) .update_document(mailbox_id) .custom( - ObjectIndexBuilder::new(SCHEMA) + ObjectIndexBuilder::new() .with_current(mailbox) - .with_changes( - Object::with_capacity(3) - .with_property(Property::Name, new_mailbox_name) - .with_property(Property::ParentId, Value::Id(Id::from(parent_id))) - .with_property( - Property::Cid, - Value::UnsignedInt(rand::random::() as u64), - ), - ), + .with_changes(new_mailbox), ); changes.log_update(Collection::Mailbox, mailbox_id); diff --git a/crates/imap/src/op/status.rs b/crates/imap/src/op/status.rs index f674b166..feb05533 100644 --- a/crates/imap/src/op/status.rs +++ b/crates/imap/src/op/status.rs @@ -19,13 +19,12 @@ use imap_proto::{ protocol::status::{Status, StatusItem, StatusItemType}, receiver::Request, }; -use jmap_proto::{ - object::Object, - types::{collection::Collection, id::Id, keyword::Keyword, property::Property, value::Value}, -}; +use jmap_proto::types::{collection::Collection, id::Id, keyword::Keyword, property::Property}; use store::{Deserialize, U32_LEN}; use store::{ - IndexKeyPrefix, IterateParams, roaring::RoaringBitmap, write::key::DeserializeBigEndian, + IndexKeyPrefix, IterateParams, ValueKey, + roaring::RoaringBitmap, + write::{ValueClass, key::DeserializeBigEndian}, }; use trc::AddContext; @@ -249,6 +248,7 @@ impl SessionData { for item in items_update { let result = match item { Status::Messages => mailbox_message_ids.as_ref().map(|v| v.len()).unwrap_or(0), +<<<<<<< HEAD Status::UidNext => self .get_uid_next(&mailbox) .await @@ -272,6 +272,44 @@ impl SessionData { .account_id(mailbox.account_id) .document_id(mailbox.mailbox_id) })?, +======= + Status::UidNext => { + (self + .server + .core + .storage + .data + .get_counter(ValueKey { + account_id: mailbox.account_id, + collection: Collection::Mailbox.into(), + document_id: mailbox.mailbox_id, + class: ValueClass::Property(Property::EmailIds.into()), + }) + .await + .caused_by(trc::location!())? + + 1) as u64 + } + Status::UidValidity => { + self.server + .get_property::( + mailbox.account_id, + Collection::Mailbox, + mailbox.mailbox_id, + &Property::Value, + ) + .await? + .ok_or_else(|| { + trc::StoreEvent::UnexpectedError + .into_err() + .details("Mailbox unavailable") + .ctx(trc::Key::Reason, "Failed to obtain uid validity") + .caused_by(trc::location!()) + .account_id(mailbox.account_id) + .document_id(mailbox.mailbox_id) + })? + .uid_validity as u64 + } +>>>>>>> b34a8804 (Improved object serialization) Status::Unseen => { if let (Some(message_ids), Some(mailbox_message_ids)) = (&message_ids, &mailbox_message_ids) diff --git a/crates/imap/src/op/store.rs b/crates/imap/src/op/store.rs index a77fd8ac..f16ccbba 100644 --- a/crates/imap/src/op/store.rs +++ b/crates/imap/src/op/store.rs @@ -7,33 +7,33 @@ use std::{sync::Arc, time::Instant}; use crate::{ - core::{message::MAX_RETRIES, SelectedMailbox, Session, SessionData}, + core::{SelectedMailbox, Session, SessionData, message::MAX_RETRIES}, spawn_op, }; use ahash::AHashSet; use common::listener::SessionStream; use directory::Permission; -use email::{ingest::EmailIngest, mailbox::UidMailbox}; +use email::{ + mailbox::UidMailbox, + message::{bayes::EmailBayesTrain, ingest::EmailIngest}, +}; use imap_proto::{ + Command, ResponseCode, ResponseType, StatusResponse, protocol::{ + Flag, ImapResponse, fetch::{DataItem, FetchItem}, store::{Arguments, Operation, Response}, - Flag, ImapResponse, }, receiver::Request, - Command, ResponseCode, ResponseType, StatusResponse, -}; -use jmap::{ - changes::get::ChangesLookup, - email::{bayes::EmailBayesTrain, set::TagManager}, }; +use jmap::{changes::get::ChangesLookup, email::set::TagManager}; use jmap_proto::types::{ acl::Acl, collection::Collection, id::Id, keyword::Keyword, property::Property, state::StateChange, type_state::DataType, }; use store::{ query::log::{Change, Query}, - write::{assert::HashedValue, log::ChangeLogBuilder, BatchBuilder, ValueClass, F_VALUE}, + write::{BatchBuilder, F_VALUE, ValueClass, assert::HashedValue, log::ChangeLogBuilder}, }; use trc::AddContext; diff --git a/crates/imap/src/op/subscribe.rs b/crates/imap/src/op/subscribe.rs index 671db792..d91f0f93 100644 --- a/crates/imap/src/op/subscribe.rs +++ b/crates/imap/src/op/subscribe.rs @@ -12,17 +12,12 @@ use crate::{ }; use common::listener::SessionStream; use directory::Permission; -use email::mailbox::SCHEMA; -use imap_proto::{receiver::Request, Command, ResponseCode, StatusResponse}; -use jmap::mailbox::set::MailboxSubscribe; +use imap_proto::{Command, ResponseCode, StatusResponse, receiver::Request}; use jmap_proto::{ - object::{index::ObjectIndexBuilder, Object}, - types::{ - collection::Collection, property::Property, state::StateChange, type_state::DataType, - value::Value, - }, + object::index::ObjectIndexBuilder, + types::{collection::Collection, property::Property, state::StateChange, type_state::DataType}, }; -use store::write::{assert::HashedValue, BatchBuilder}; +use store::write::{BatchBuilder, assert::HashedValue}; use super::ImapContext; @@ -102,7 +97,7 @@ impl SessionData { // Obtain mailbox let mailbox = self .server - .get_property::>>( + .get_property::>( account_id, Collection::Mailbox, mailbox_id, @@ -120,23 +115,29 @@ impl SessionData { })?; // Subscribe/unsubscribe to mailbox - if let Some(value) = mailbox.inner.mailbox_subscribe(self.account_id, subscribe) { + if (subscribe && !mailbox.inner.is_subscribed(self.account_id)) + || (!subscribe && mailbox.inner.is_subscribed(self.account_id)) + { // Build batch let mut changes = self .server .begin_changes(account_id) .imap_ctx(&tag, trc::location!())?; + let mut new_mailbox = mailbox.inner.clone(); + if subscribe { + new_mailbox.subscribers.push(self.account_id); + } else { + new_mailbox.remove_subscriber(self.account_id); + } let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Mailbox) .update_document(mailbox_id) .custom( - ObjectIndexBuilder::new(SCHEMA) + ObjectIndexBuilder::new() .with_current(mailbox) - .with_changes( - Object::with_capacity(1).with_property(Property::IsSubscribed, value), - ), + .with_changes(new_mailbox), ); changes.log_update(Collection::Mailbox, mailbox_id); diff --git a/crates/imap/src/op/thread.rs b/crates/imap/src/op/thread.rs index f65395ba..3dabbe7b 100644 --- a/crates/imap/src/op/thread.rs +++ b/crates/imap/src/op/thread.rs @@ -13,14 +13,14 @@ use crate::{ use ahash::AHashMap; use common::listener::SessionStream; use directory::Permission; -use email::cache::ThreadCache; +use email::thread::cache::ThreadCache; use imap_proto::{ + Command, StatusResponse, protocol::{ - thread::{Arguments, Response}, ImapResponse, + thread::{Arguments, Response}, }, receiver::Request, - Command, StatusResponse, }; use trc::AddContext; diff --git a/crates/jmap-proto/src/method/copy.rs b/crates/jmap-proto/src/method/copy.rs index e95a2138..6e09162e 100644 --- a/crates/jmap-proto/src/method/copy.rs +++ b/crates/jmap-proto/src/method/copy.rs @@ -9,14 +9,13 @@ use utils::map::vec_map::VecMap; use crate::{ error::set::SetError, - object::Object, - parser::{json::Parser, JsonObjectParser, Token}, - request::{method::MethodObject, reference::MaybeReference, RequestProperty}, + parser::{JsonObjectParser, Token, json::Parser}, + request::{RequestProperty, method::MethodObject, reference::MaybeReference}, types::{ blob::BlobId, id::Id, state::{State, StateChange}, - value::{SetValue, Value}, + value::{Object, SetValue, Value}, }, }; @@ -98,7 +97,7 @@ impl JsonObjectParser for CopyRequest { _ => { return Err(trc::JmapEvent::UnknownMethod .into_err() - .details(format!("{}/copy", parser.ctx))) + .details(format!("{}/copy", parser.ctx))); } }, account_id: Id::default(), diff --git a/crates/jmap-proto/src/method/get.rs b/crates/jmap-proto/src/method/get.rs index e7fbb6eb..79f16a63 100644 --- a/crates/jmap-proto/src/method/get.rs +++ b/crates/jmap-proto/src/method/get.rs @@ -5,14 +5,21 @@ */ use crate::{ - object::{blob, email, Object}, - parser::{json::Parser, JsonObjectParser, Token}, + object::{blob, email}, + parser::{JsonObjectParser, Token, json::Parser}, request::{ + RequestProperty, RequestPropertyParser, method::MethodObject, reference::{MaybeReference, ResultReference}, - RequestProperty, RequestPropertyParser, }, - types::{any_id::AnyId, blob::BlobId, id::Id, property::Property, state::State, value::Value}, + types::{ + any_id::AnyId, + blob::BlobId, + id::Id, + property::Property, + state::State, + value::{Object, Value}, + }, }; #[derive(Debug, Clone)] @@ -74,7 +81,7 @@ impl JsonObjectParser for GetRequest { _ => { return Err(trc::JmapEvent::UnknownMethod .into_err() - .details(format!("{}/get", parser.ctx))) + .details(format!("{}/get", parser.ctx))); } }, account_id: Id::default(), diff --git a/crates/jmap-proto/src/method/import.rs b/crates/jmap-proto/src/method/import.rs index 447892f9..d340f686 100644 --- a/crates/jmap-proto/src/method/import.rs +++ b/crates/jmap-proto/src/method/import.rs @@ -8,11 +8,10 @@ use utils::map::vec_map::VecMap; use crate::{ error::set::SetError, - object::Object, - parser::{json::Parser, JsonObjectParser, Token}, + parser::{JsonObjectParser, Token, json::Parser}, request::{ - reference::{MaybeReference, ResultReference}, RequestProperty, + reference::{MaybeReference, ResultReference}, }, response::Response, types::{ @@ -22,7 +21,7 @@ use crate::{ keyword::Keyword, property::Property, state::{State, StateChange}, - value::{SetValueMap, Value}, + value::{Object, SetValueMap, Value}, }, }; diff --git a/crates/jmap-proto/src/method/parse.rs b/crates/jmap-proto/src/method/parse.rs index c33d84fd..06ed576c 100644 --- a/crates/jmap-proto/src/method/parse.rs +++ b/crates/jmap-proto/src/method/parse.rs @@ -7,10 +7,14 @@ use utils::map::vec_map::VecMap; use crate::{ - object::Object, - parser::{json::Parser, Ignore, JsonObjectParser, Token}, + parser::{Ignore, JsonObjectParser, Token, json::Parser}, request::RequestProperty, - types::{blob::BlobId, id::Id, property::Property, value::Value}, + types::{ + blob::BlobId, + id::Id, + property::Property, + value::{Object, Value}, + }, }; #[derive(Debug, Clone)] diff --git a/crates/jmap-proto/src/method/set.rs b/crates/jmap-proto/src/method/set.rs index eb144e6f..a51508b5 100644 --- a/crates/jmap-proto/src/method/set.rs +++ b/crates/jmap-proto/src/method/set.rs @@ -9,12 +9,12 @@ use utils::map::{bitmap::Bitmap, vec_map::VecMap}; use crate::{ error::set::{InvalidProperty, SetError}, - object::{email_submission, mailbox, sieve, Object}, - parser::{json::Parser, JsonObjectParser, Token}, + object::{email_submission, mailbox, sieve}, + parser::{JsonObjectParser, Token, json::Parser}, request::{ + RequestProperty, RequestPropertyParser, method::MethodObject, reference::{MaybeReference, ResultReference}, - RequestProperty, RequestPropertyParser, }, response::Response, types::{ @@ -26,7 +26,7 @@ use crate::{ keyword::Keyword, property::{HeaderForm, ObjectProperty, Property, SetProperty}, state::{State, StateChange}, - value::{SetValue, SetValueMap, Value}, + value::{Object, SetValue, SetValueMap, Value}, }, }; @@ -114,7 +114,7 @@ impl JsonObjectParser for SetRequest { _ => { return Err(trc::JmapEvent::UnknownMethod .into_err() - .details(format!("{}/set", parser.ctx))) + .details(format!("{}/set", parser.ctx))); } }, account_id: Id::default(), @@ -168,9 +168,7 @@ impl JsonObjectParser for Object { where Self: Sized, { - let mut obj = Object { - properties: VecMap::with_capacity(8), - }; + let mut obj = Object(VecMap::with_capacity(8)); parser .next_token::()? @@ -348,7 +346,7 @@ impl JsonObjectParser for Object { SetValue::ResultReference(ResultReference::parse(parser)?) }; - obj.properties.append(key.property, value); + obj.0.append(key.property, value); } Ok(obj) @@ -536,7 +534,7 @@ impl SetResponse { (&mut self.created) .into_iter() .map(|(_, obj)| obj) - .find(|obj| obj.properties.get(&Property::Id) == Some(&Value::Id(id))) + .find(|obj| obj.0.get(&Property::Id) == Some(&Value::Id(id))) } pub fn has_changes(&self) -> bool { diff --git a/crates/jmap-proto/src/object/email_submission.rs b/crates/jmap-proto/src/object/email_submission.rs index 121feff1..3d816694 100644 --- a/crates/jmap-proto/src/object/email_submission.rs +++ b/crates/jmap-proto/src/object/email_submission.rs @@ -7,13 +7,14 @@ use utils::map::vec_map::VecMap; use crate::{ - parser::{json::Parser, JsonObjectParser}, - request::{reference::MaybeReference, RequestProperty, RequestPropertyParser}, - types::{id::Id, value::SetValue}, + parser::{JsonObjectParser, json::Parser}, + request::{RequestProperty, RequestPropertyParser, reference::MaybeReference}, + types::{ + id::Id, + value::{Object, SetValue}, + }, }; -use super::Object; - #[derive(Debug, Clone, Default)] pub struct SetArguments { pub on_success_update_email: Option, Object>>, diff --git a/crates/jmap-proto/src/object/index.rs b/crates/jmap-proto/src/object/index.rs index cebc6091..bc74bf31 100644 --- a/crates/jmap-proto/src/object/index.rs +++ b/crates/jmap-proto/src/object/index.rs @@ -4,155 +4,129 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{borrow::Cow, collections::HashSet}; - +use std::{collections::HashSet, fmt::Debug}; use store::{ + Deserialize, Serialize, write::{ - assert::HashedValue, BatchBuilder, BitmapClass, BitmapHash, IntoOperations, Operation, - TokenizeText, ValueClass, ValueOp, + BatchBuilder, BitmapClass, BitmapHash, DirectoryClass, IntoOperations, Operation, + TokenizeText, ValueClass, ValueOp, assert::HashedValue, }, - Serialize, }; -use crate::{ - error::set::SetError, - types::{id::Id, property::Property, value::Value}, -}; +use crate::types::{property::Property, value::AclGrant}; -use super::Object; - -#[derive(Debug, Clone, Default)] -pub struct ObjectIndexBuilder { - index: &'static [IndexProperty], - current: Option>>, - changes: Option>, -} - -#[derive(Debug, Clone, Copy, Default)] -pub enum IndexAs { +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IndexValue<'x> { Text { + field: u8, + value: &'x str, tokenize: bool, index: bool, }, - TextList { - tokenize: bool, - index: bool, + U32 { + field: u8, + value: Option, + }, + U64 { + field: u8, + value: Option, + }, + U32List { + field: u8, + value: &'x [u32], + }, + Tag { + field: u8, + is_set: bool, + }, + Quota { + used: u32, + }, + Acl { + value: &'x [AclGrant], }, - Integer, - IntegerList, - LongInteger, - HasProperty, - Acl, - #[default] - None, } -#[derive(Debug, Clone)] -pub struct IndexProperty { - property: Property, - index_as: IndexAs, - required: bool, - max_size: usize, +pub trait IndexableObject: Debug + Serialize + Deserialize + Eq { + fn index_values(&self) -> impl Iterator>; } -impl ObjectIndexBuilder { - pub fn new(index: &'static [IndexProperty]) -> Self { +#[derive(Debug)] +pub struct ObjectIndexBuilder { + tenant_id: Option, + current: Option>, + changes: Option, +} + +impl Default for ObjectIndexBuilder { + fn default() -> Self { + Self::new() + } +} + +impl ObjectIndexBuilder { + pub fn new() -> Self { Self { - index, current: None, changes: None, + tenant_id: None, } } - pub fn with_current(mut self, current: HashedValue>) -> Self { + pub fn with_current(mut self, current: HashedValue) -> Self { self.current = Some(current); self } - pub fn with_changes(mut self, changes: Object) -> Self { + pub fn with_changes(mut self, changes: T) -> Self { self.changes = Some(changes); self } - pub fn with_current_opt(mut self, current: Option>>) -> Self { + pub fn with_current_opt(mut self, current: Option>) -> Self { self.current = current; self } - pub fn get(&self, property: &Property) -> &Value { - self.changes - .as_ref() - .and_then(|c| c.properties.get(property)) - .or_else(|| { - self.current - .as_ref() - .and_then(|c| c.inner.properties.get(property)) - }) - .unwrap_or(&Value::Null) - } - - pub fn set(&mut self, property: Property, value: Value) { - if let Some(changes) = &mut self.changes { - changes.properties.set(property, value); - } - } - - pub fn validate(self) -> Result { - for item in self.index { - if item.required || item.max_size > 0 { - let error: Cow = match self.get(&item.property) { - Value::Null if item.required => "Property cannot be empty.".into(), - Value::Text(text) => { - if item.required && text.trim().is_empty() { - "Property cannot be empty.".into() - } else if item.max_size > 0 && text.len() > item.max_size { - format!("Property cannot be longer than {} bytes.", item.max_size) - .into() - } else { - continue; - } - } - _ => continue, - }; - return Err(SetError::invalid_properties() - .with_property(item.property.clone()) - .with_description(error)); - } - } - - Ok(self) - } - - pub fn changes(&self) -> Option<&Object> { + pub fn changes(&self) -> Option<&T> { self.changes.as_ref() } - pub fn changes_mut(&mut self) -> Option<&mut Object> { + pub fn changes_mut(&mut self) -> Option<&mut T> { self.changes.as_mut() } - pub fn current(&self) -> Option<&HashedValue>> { + pub fn current(&self) -> Option<&HashedValue> { self.current.as_ref() } + + pub fn with_tenant_id(mut self, tenant_id: Option) -> Self { + self.tenant_id = tenant_id; + self + } + + pub fn set_tenant_id(&mut self, tenant_id: u32) { + self.tenant_id = tenant_id.into(); + } } -impl IntoOperations for ObjectIndexBuilder { +impl IntoOperations for ObjectIndexBuilder { fn build(self, batch: &mut BatchBuilder) { match (self.current, self.changes) { (None, Some(changes)) => { // Insertion - build_batch(batch, self.index, &changes, true); + build_batch(batch, &changes, self.tenant_id, true); batch.set(Property::Value, changes.serialize()); } (Some(current), Some(changes)) => { // Update batch.assert_value(Property::Value, ¤t); - merge_batch(batch, self.index, current.inner, changes); + merge_batch(batch, current.inner, changes, self.tenant_id); } (Some(current), None) => { // Deletion batch.assert_value(Property::Value, ¤t); - build_batch(batch, self.index, ¤t.inner, false); + build_batch(batch, ¤t.inner, self.tenant_id, false); batch.clear(Property::Value); } (None, None) => unreachable!(), @@ -160,374 +134,81 @@ impl IntoOperations for ObjectIndexBuilder { } } -fn merge_batch( +fn build_batch( batch: &mut BatchBuilder, - index: &'static [IndexProperty], - mut current: Object, - changes: Object, -) { - let mut has_changes = false; - - for (property, value) in changes.properties { - let current_value = current.get(&property); - if current_value == &value { - continue; - } - - for index_property in index { - if index_property.property != property { - continue; - } - match index_property.index_as { - IndexAs::Text { tokenize, index } => { - // Remove current text from index - let mut add_tokens = HashSet::new(); - let mut remove_tokens = HashSet::new(); - if let Some(text) = current_value.as_string() { - if index { - batch.ops.push(Operation::Index { - field: property.clone().into(), - key: text.serialize(), - set: false, - }); - } - if tokenize { - text.tokenize_into(&mut remove_tokens); - } - } - - // Add new text to index - if let Some(text) = value.as_string() { - if index { - batch.ops.push(Operation::Index { - field: property.clone().into(), - key: text.serialize(), - set: true, - }); - } - if tokenize { - for token in text.to_tokens() { - if !remove_tokens.remove(&token) { - add_tokens.insert(token); - } - } - } - } - - // Update tokens - let field: u8 = property.clone().into(); - for (token, set) in [(add_tokens, true), (remove_tokens, false)] { - for token in token { - batch.ops.push(Operation::Bitmap { - class: BitmapClass::Text { - field, - token: BitmapHash::new(token), - }, - set, - }); - } - } - } - IndexAs::TextList { tokenize, index } => { - let mut add_tokens = HashSet::new(); - let mut remove_tokens = HashSet::new(); - let mut add_values = HashSet::new(); - let mut remove_values = HashSet::new(); - - // Remove current text from index - if let Some(current_values) = current_value.as_list() { - for current_value in current_values { - if let Some(text) = current_value.as_string() { - if index { - remove_values.insert(text); - } - if tokenize { - text.tokenize_into(&mut remove_tokens); - } - } - } - } - - // Add new text to index - if let Some(values) = value.as_list() { - for value in values { - if let Some(text) = value.as_string() { - if index && !remove_values.remove(text) { - add_values.insert(text); - } - if tokenize { - for token in text.to_tokens() { - if !remove_tokens.remove(&token) { - add_tokens.insert(token); - } - } - } - } - } - } - - // Update index - for (values, set) in [(add_values, true), (remove_values, false)] { - for value in values { - batch.ops.push(Operation::Index { - field: property.clone().into(), - key: value.serialize(), - set, - }); - } - } - - // Update tokens - let field: u8 = property.clone().into(); - for (token, set) in [(add_tokens, true), (remove_tokens, false)] { - for token in token { - batch.ops.push(Operation::Bitmap { - class: BitmapClass::Text { - field, - token: BitmapHash::new(token), - }, - set, - }); - } - } - } - index_as @ (IndexAs::Integer | IndexAs::LongInteger) => { - if let Some(current_value) = current_value.try_cast_uint() { - batch.ops.push(Operation::Index { - field: property.clone().into(), - key: current_value.into_index(index_as), - set: false, - }); - } - if let Some(value) = value.try_cast_uint() { - batch.ops.push(Operation::Index { - field: property.clone().into(), - key: value.into_index(index_as), - set: true, - }); - } - } - IndexAs::IntegerList => { - let mut add_values = HashSet::new(); - let mut remove_values = HashSet::new(); - - if let Some(current_values) = current_value.as_list() { - for current_value in current_values { - if let Some(current_value) = current_value.try_cast_uint() { - remove_values.insert(current_value); - } - } - } - if let Some(values) = value.as_list() { - for value in values { - if let Some(value) = value.try_cast_uint() { - if !remove_values.remove(&value) { - add_values.insert(value); - } - } - } - } - - for (values, set) in [(add_values, true), (remove_values, false)] { - for value in values { - batch.ops.push(Operation::Index { - field: property.clone().into(), - key: (value as u32).serialize(), - set, - }); - } - } - } - IndexAs::HasProperty => { - if current_value == &Value::Null { - batch.ops.push(Operation::Bitmap { - class: BitmapClass::Tag { - field: property.clone().into(), - value: ().into(), - }, - set: true, - }); - } else if value == Value::Null { - batch.ops.push(Operation::Bitmap { - class: BitmapClass::Tag { - field: property.clone().into(), - value: ().into(), - }, - set: false, - }); - } - } - IndexAs::Acl => { - match (current_value, &value) { - (Value::Acl(current_value), Value::Acl(value)) => { - // Remove deleted ACLs - for current_item in current_value { - if !value - .iter() - .any(|item| item.account_id == current_item.account_id) - { - batch - .ops - .push(Operation::acl(current_item.account_id, None)); - } - } - - // Update ACLs - for item in value { - let mut add_item = true; - for current_item in current_value { - if item.account_id == current_item.account_id { - if item.grants == current_item.grants { - add_item = false; - } - break; - } - } - if add_item { - batch.ops.push(Operation::acl( - item.account_id, - item.grants.bitmap.serialize().into(), - )); - } - } - } - (Value::Null, Value::Acl(values)) => { - // Add all ACLs - for item in values { - batch.ops.push(Operation::acl( - item.account_id, - item.grants.bitmap.serialize().into(), - )); - } - } - (Value::Acl(current_values), Value::Null) => { - // Remove all ACLs - for item in current_values { - batch.ops.push(Operation::acl(item.account_id, None)); - } - } - _ => {} - } - } - IndexAs::None => (), - } - } - if value != Value::Null { - current.set(property, value); - } else { - current.remove(&property); - } - has_changes = true; - } - - if has_changes { - batch.ops.push(Operation::Value { - class: Property::Value.into(), - op: ValueOp::Set(current.serialize().into()), - }); - } -} - -fn build_batch( - batch: &mut BatchBuilder, - index: &'static [IndexProperty], - object: &Object, + object: &T, + tenant_id: Option, set: bool, ) { - for item in index { - match (object.get(&item.property), item.index_as) { - (Value::Text(text), IndexAs::Text { tokenize, index }) => { - if index { - batch.ops.push(Operation::Index { - field: (&item.property).into(), - key: text.serialize(), - set, - }); - } - if tokenize { - let field: u8 = (&item.property).into(); - for token in text.as_str().to_tokens() { - batch.ops.push(Operation::Bitmap { - class: BitmapClass::Text { - field, - token: BitmapHash::new(token), - }, + for item in object.index_values() { + match item { + IndexValue::Text { + field, + value, + tokenize, + index, + } => { + if !value.is_empty() { + if index { + batch.ops.push(Operation::Index { + field, + key: value.serialize(), set, }); } - } - } - (Value::List(values), IndexAs::TextList { tokenize, index }) => { - let mut tokens = HashSet::new(); - let mut indexes = HashSet::new(); - for value in values { - if let Some(text) = value.as_string() { - if index { - indexes.insert(text); - } - if tokenize { - tokens.extend(text.to_tokens()); + if tokenize { + for token in value.to_tokens() { + batch.ops.push(Operation::Bitmap { + class: BitmapClass::Text { + field, + token: BitmapHash::new(token), + }, + set, + }); } } } - let field: u8 = (&item.property).into(); - for text in indexes { + } + IndexValue::U32 { field, value } => { + if let Some(value) = value { batch.ops.push(Operation::Index { field, - key: text.serialize(), + key: value.serialize(), set, }); } - for token in tokens { + } + IndexValue::U64 { field, value } => { + if let Some(value) = value { + batch.ops.push(Operation::Index { + field, + key: value.serialize(), + set, + }); + } + } + IndexValue::U32List { field, value } => { + for item in value { + batch.ops.push(Operation::Index { + field, + key: item.serialize(), + set, + }); + } + } + IndexValue::Tag { field, is_set } => { + if is_set { batch.ops.push(Operation::Bitmap { - class: BitmapClass::Text { + class: BitmapClass::Tag { field, - token: BitmapHash::new(token), + value: ().into(), }, set, }); } } - (Value::UnsignedInt(integer), IndexAs::Integer | IndexAs::LongInteger) => { - batch.ops.push(Operation::Index { - field: (&item.property).into(), - key: integer.into_index(item.index_as), - set, - }); - } - (Value::Bool(boolean), IndexAs::Integer) => { - batch.ops.push(Operation::Index { - field: (&item.property).into(), - key: (*boolean as u32).serialize(), - set, - }); - } - (Value::Id(id), IndexAs::Integer | IndexAs::LongInteger) => { - batch.ops.push(Operation::Index { - field: (&item.property).into(), - key: id.into_index(item.index_as), - set, - }); - } - (Value::List(values), IndexAs::IntegerList) => { - for value in values - .iter() - .map(|value| match value { - Value::UnsignedInt(integer) => *integer as u32, - Value::Id(id) => id.document_id(), - _ => unreachable!(), - }) - .collect::>() - { - batch.ops.push(Operation::Index { - field: (&item.property).into(), - key: value.into_index(item.index_as), - set, - }); - } - } - (Value::Acl(values), IndexAs::Acl) => { - for item in values { + IndexValue::Acl { value } => { + for item in value { batch.ops.push(Operation::acl( item.account_id, if set { @@ -538,77 +219,271 @@ fn build_batch( )); } } - (value, IndexAs::HasProperty) if value != &Value::Null => { - batch.ops.push(Operation::Bitmap { - class: BitmapClass::Tag { - field: (&item.property).into(), - value: ().into(), - }, - set, - }); + IndexValue::Quota { used } => { + let value = if set { used as i64 } else { -(used as i64) }; + + if let Some(account_id) = batch.last_account_id() { + batch.add(DirectoryClass::UsedQuota(account_id), value); + } + + if let Some(tenant_id) = tenant_id { + batch.add(DirectoryClass::UsedQuota(tenant_id), value); + } } - - _ => (), } } } -impl IndexProperty { - pub const fn new(property: Property) -> Self { - Self { - property, - required: false, - max_size: 0, - index_as: IndexAs::None, +fn merge_batch( + batch: &mut BatchBuilder, + current: T, + changes: T, + tenant_id: Option, +) { + let mut has_changes = current != changes; + + for (current, change) in current.index_values().zip(changes.index_values()) { + if current == change { + continue; } - } + has_changes = true; - pub const fn required(mut self) -> Self { - self.required = true; - self - } + match (current, change) { + ( + IndexValue::Text { + field, + value: old_value, + tokenize, + index, + }, + IndexValue::Text { + value: new_value, .. + }, + ) => { + // Remove current text from index + let mut add_tokens = HashSet::new(); + let mut remove_tokens = HashSet::new(); - pub const fn max_size(mut self, max_size: usize) -> Self { - self.max_size = max_size; - self - } + if !old_value.is_empty() { + if index { + batch.ops.push(Operation::Index { + field, + key: old_value.serialize(), + set: false, + }); + } + if tokenize { + old_value.tokenize_into(&mut remove_tokens); + } + } - pub const fn index_as(mut self, index_as: IndexAs) -> Self { - self.index_as = index_as; - self - } -} + // Add new text to index + if !new_value.is_empty() { + if index { + batch.ops.push(Operation::Index { + field, + key: new_value.serialize(), + set: true, + }); + } + if tokenize { + for token in new_value.to_tokens() { + if !remove_tokens.remove(&token) { + add_tokens.insert(token); + } + } + } + } -trait IntoIndex { - fn into_index(self, index_as: IndexAs) -> Vec; -} + // Update tokens + for (token, set) in [(add_tokens, true), (remove_tokens, false)] { + for token in token { + batch.ops.push(Operation::Bitmap { + class: BitmapClass::Text { + field, + token: BitmapHash::new(token), + }, + set, + }); + } + } + } + ( + IndexValue::U32 { + field, + value: old_value, + }, + IndexValue::U32 { + value: new_value, .. + }, + ) => { + if let Some(value) = old_value { + batch.ops.push(Operation::Index { + field, + key: value.serialize(), + set: false, + }); + } + if let Some(value) = new_value { + batch.ops.push(Operation::Index { + field, + key: value.serialize(), + set: true, + }); + } + } + ( + IndexValue::U64 { + field, + value: old_value, + }, + IndexValue::U64 { + value: new_value, .. + }, + ) => { + if let Some(value) = old_value { + batch.ops.push(Operation::Index { + field, + key: value.serialize(), + set: false, + }); + } + if let Some(value) = new_value { + batch.ops.push(Operation::Index { + field, + key: value.serialize(), + set: true, + }); + } + } + ( + IndexValue::U32List { + field, + value: old_value, + }, + IndexValue::U32List { + value: new_value, .. + }, + ) => { + let mut add_values = HashSet::new(); + let mut remove_values = HashSet::new(); -impl IntoIndex for &u64 { - fn into_index(self, index_as: IndexAs) -> Vec { - match index_as { - IndexAs::Integer => (*self as u32).serialize(), - IndexAs::LongInteger => self.serialize(), + for current_value in old_value { + remove_values.insert(current_value); + } + for value in new_value { + if !remove_values.remove(&value) { + add_values.insert(value); + } + } + + for (values, set) in [(add_values, true), (remove_values, false)] { + for value in values { + batch.ops.push(Operation::Index { + field, + key: value.serialize(), + set, + }); + } + } + } + ( + IndexValue::Tag { + field, + is_set: was_set, + }, + IndexValue::Tag { is_set, .. }, + ) => { + if was_set { + batch.ops.push(Operation::Bitmap { + class: BitmapClass::Tag { + field, + value: ().into(), + }, + set: false, + }); + } + if is_set { + batch.ops.push(Operation::Bitmap { + class: BitmapClass::Tag { + field, + value: ().into(), + }, + set: true, + }); + } + } + (IndexValue::Acl { value: old_acl }, IndexValue::Acl { value: new_acl }) => { + match (!old_acl.is_empty(), !new_acl.is_empty()) { + (true, true) => { + // Remove deleted ACLs + for current_item in old_acl { + if !new_acl + .iter() + .any(|item| item.account_id == current_item.account_id) + { + batch + .ops + .push(Operation::acl(current_item.account_id, None)); + } + } + + // Update ACLs + for item in new_acl { + let mut add_item = true; + for current_item in old_acl { + if item.account_id == current_item.account_id { + if item.grants == current_item.grants { + add_item = false; + } + break; + } + } + if add_item { + batch.ops.push(Operation::acl( + item.account_id, + item.grants.bitmap.serialize().into(), + )); + } + } + } + (false, true) => { + // Add all ACLs + for item in new_acl { + batch.ops.push(Operation::acl( + item.account_id, + item.grants.bitmap.serialize().into(), + )); + } + } + (true, false) => { + // Remove all ACLs + for item in old_acl { + batch.ops.push(Operation::acl(item.account_id, None)); + } + } + _ => {} + } + } + (IndexValue::Quota { used: old_used }, IndexValue::Quota { used: new_used }) => { + let value = new_used as i64 - old_used as i64; + if let Some(account_id) = batch.last_account_id() { + batch.add(DirectoryClass::UsedQuota(account_id), value); + } + + if let Some(tenant_id) = tenant_id { + batch.add(DirectoryClass::UsedQuota(tenant_id), value); + } + } _ => unreachable!(), } } -} -impl IntoIndex for &u32 { - fn into_index(self, index_as: IndexAs) -> Vec { - match index_as { - IndexAs::Integer | IndexAs::IntegerList => self.serialize(), - _ => unreachable!("index as {index_as:?} not supported for u32"), - } - } -} - -impl IntoIndex for &Id { - fn into_index(self, index_as: IndexAs) -> Vec { - match index_as { - IndexAs::Integer => self.document_id().serialize(), - IndexAs::LongInteger => self.id().serialize(), - _ => unreachable!("index as {index_as:?} not supported for Id"), - } + if has_changes { + batch.ops.push(Operation::Value { + class: Property::Value.into(), + op: ValueOp::Set(current.serialize().into()), + }); } } diff --git a/crates/jmap-proto/src/object/mod.rs b/crates/jmap-proto/src/object/mod.rs index 1787c558..a899ce17 100644 --- a/crates/jmap-proto/src/object/mod.rs +++ b/crates/jmap-proto/src/object/mod.rs @@ -4,6 +4,19 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use std::sync::Arc; + +use utils::{ + erased_serde, + json::{JsonPointerItem, JsonQueryable}, +}; + +use crate::types::{ + id::Id, + property::Property, + value::{Object, Value}, +}; + pub mod blob; pub mod email; pub mod email_submission; @@ -11,287 +24,64 @@ pub mod index; pub mod mailbox; pub mod sieve; -use std::slice::Iter; - -use store::{ - write::{DeserializeFrom, SerializeInto, ToBitmaps}, - Deserialize, Serialize, U64_LEN, -}; -use utils::{ - codec::leb128::{Leb128Iterator, Leb128Vec}, - map::{bitmap::Bitmap, vec_map::VecMap}, -}; - -use crate::types::{ - blob::BlobId, - date::UTCDate, - id::Id, - keyword::Keyword, - property::Property, - value::{AclGrant, Value}, -}; - -#[derive(Debug, Clone, Default, serde::Serialize, PartialEq, Eq)] -#[serde(transparent)] -pub struct Object { - pub properties: VecMap, +pub trait JsonObjectTrait: JsonQueryable + erased_serde::Serialize { + fn id(&self) -> Option; } -impl Object { - pub fn with_capacity(capacity: usize) -> Self { - Self { - properties: VecMap::with_capacity(capacity), - } +#[derive(Clone, Debug)] +pub struct JsonObject(Arc); + +impl JsonObject { + pub fn new(value: T) -> Self { + Self(Arc::new(value)) } - pub fn set(&mut self, property: Property, value: impl Into) -> bool { - self.properties.set(property, value.into()) - } - - pub fn append(&mut self, property: Property, value: impl Into) { - self.properties.append(property, value.into()); - } - - pub fn with_property(mut self, property: Property, value: impl Into) -> Self { - self.properties.append(property, value.into()); - self - } - - pub fn remove(&mut self, property: &Property) -> Value { - self.properties.remove(property).unwrap_or(Value::Null) - } - - pub fn get(&self, property: &Property) -> &Value { - self.properties.get(property).unwrap_or(&Value::Null) + #[inline] + pub fn id(&self) -> Option { + self.0.id() } } -impl ToBitmaps for Value { - fn to_bitmaps(&self, ops: &mut Vec, field: u8, set: bool) { - match self { - Value::Text(text) => text.as_str().to_bitmaps(ops, field, set), - Value::Keyword(keyword) => keyword.to_bitmaps(ops, field, set), - Value::UnsignedInt(int) => int.to_bitmaps(ops, field, set), - Value::List(items) => { - for item in items { - match item { - Value::Text(text) => text.as_str().to_bitmaps(ops, field, set), - Value::UnsignedInt(int) => int.to_bitmaps(ops, field, set), - Value::Keyword(keyword) => keyword.to_bitmaps(ops, field, set), - _ => (), - } +impl serde::Serialize for JsonObject { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + erased_serde::serialize(self.0.as_ref(), serializer) + } +} + +impl JsonObjectTrait for Object { + fn id(&self) -> Option { + self.get(&Property::Id).as_id().copied() + } +} + +impl JsonQueryable for Object { + fn eval_pointer<'x>( + &'x self, + mut pointer: std::slice::Iter, + results: &mut Vec<&'x dyn JsonQueryable>, + ) { + match pointer.next() { + Some(JsonPointerItem::String(n)) => { + if let Some(v) = self + .0 + .iter() + .find_map(|(k, v)| if k.as_str() == n { Some(v) } else { None }) + { + v.eval_pointer(pointer, results); } } - _ => (), - } - } -} - -impl ToBitmaps for Object { - fn to_bitmaps(&self, _ops: &mut Vec, _field: u8, _set: bool) { - unreachable!() - } -} - -impl ToBitmaps for &Object { - fn to_bitmaps(&self, _ops: &mut Vec, _field: u8, _set: bool) { - unreachable!() - } -} - -const TEXT: u8 = 0; -const UNSIGNED_INT: u8 = 1; -const BOOL_TRUE: u8 = 2; -const BOOL_FALSE: u8 = 3; -const ID: u8 = 4; -const DATE: u8 = 5; -const BLOB_ID: u8 = 6; -const BLOB: u8 = 7; -const KEYWORD: u8 = 8; -const LIST: u8 = 9; -const OBJECT: u8 = 10; -const ACL: u8 = 11; -const NULL: u8 = 12; - -impl Serialize for Value { - fn serialize(self) -> Vec { - let mut buf = Vec::with_capacity(1024); - self.serialize_into(&mut buf); - buf - } -} - -impl Deserialize for Value { - fn deserialize(bytes: &[u8]) -> trc::Result { - Self::deserialize_from(&mut bytes.iter()).ok_or_else(|| { - trc::StoreEvent::DataCorruption - .caused_by(trc::location!()) - .ctx(trc::Key::Value, bytes) - }) - } -} - -impl Serialize for Object { - fn serialize(self) -> Vec { - (&self).serialize() - } -} - -impl Serialize for &Object { - fn serialize(self) -> Vec { - let mut buf = Vec::with_capacity(1024); - self.serialize_into(&mut buf); - buf - } -} - -impl Deserialize for Object { - fn deserialize(bytes: &[u8]) -> trc::Result { - Object::deserialize_from(&mut bytes.iter()).ok_or_else(|| { - trc::StoreEvent::DataCorruption - .caused_by(trc::location!()) - .ctx(trc::Key::Value, bytes) - }) - } -} - -impl SerializeInto for Object { - fn serialize_into(&self, buf: &mut Vec) { - buf.push_leb128(self.properties.len()); - for (k, v) in &self.properties { - k.serialize_into(buf); - v.serialize_into(buf); - } - } -} - -impl SerializeInto for AclGrant { - fn serialize_into(&self, buf: &mut Vec) { - buf.push_leb128(self.account_id); - buf.extend_from_slice(self.grants.bitmap.to_be_bytes().as_slice()); - } -} - -impl DeserializeFrom for AclGrant { - fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option { - let account_id = bytes.next_leb128()?; - let mut grants = [0u8; U64_LEN]; - for byte in grants.iter_mut() { - *byte = *bytes.next()?; - } - - Some(Self { - account_id, - grants: Bitmap::from(u64::from_be_bytes(grants)), - }) - } -} - -impl DeserializeFrom for Object { - fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option> { - let len = bytes.next_leb128()?; - let mut properties = VecMap::with_capacity(len); - for _ in 0..len { - let key = Property::deserialize_from(bytes)?; - let value = Value::deserialize_from(bytes)?; - properties.append(key, value); - } - Some(Object { properties }) - } -} - -impl SerializeInto for Value { - fn serialize_into(&self, buf: &mut Vec) { - match self { - Value::Text(v) => { - buf.push(TEXT); - v.serialize_into(buf); - } - Value::UnsignedInt(v) => { - buf.push(UNSIGNED_INT); - v.serialize_into(buf); - } - Value::Bool(v) => { - buf.push(if *v { BOOL_TRUE } else { BOOL_FALSE }); - } - Value::Id(v) => { - buf.push(ID); - v.id().serialize_into(buf); - } - Value::Date(v) => { - buf.push(DATE); - (v.timestamp() as u64).serialize_into(buf); - } - Value::BlobId(v) => { - buf.push(BLOB_ID); - v.serialize_into(buf); - } - Value::Keyword(v) => { - buf.push(KEYWORD); - v.serialize_into(buf); - } - Value::List(v) => { - buf.push(LIST); - buf.push_leb128(v.len()); - for i in v { - i.serialize_into(buf); + Some(JsonPointerItem::Wildcard) => { + for v in self.0.values() { + v.eval_pointer(pointer.clone(), results); } } - Value::Object(v) => { - buf.push(OBJECT); - v.serialize_into(buf); - } - Value::Blob(v) => { - buf.push(BLOB); - v.serialize_into(buf); - } - Value::Acl(v) => { - buf.push(ACL); - buf.push_leb128(v.len()); - for i in v { - i.serialize_into(buf); - } - } - Value::Null => { - buf.push(NULL); + Some(JsonPointerItem::Root) | None => { + results.push(self); } - } - } -} - -impl DeserializeFrom for Value { - fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option { - match *bytes.next()? { - TEXT => Some(Value::Text(String::deserialize_from(bytes)?)), - UNSIGNED_INT => Some(Value::UnsignedInt(bytes.next_leb128()?)), - BOOL_TRUE => Some(Value::Bool(true)), - BOOL_FALSE => Some(Value::Bool(false)), - ID => Some(Value::Id(Id::new(bytes.next_leb128()?))), - DATE => Some(Value::Date(UTCDate::from_timestamp( - bytes.next_leb128::()? as i64, - ))), - BLOB_ID => Some(Value::BlobId(BlobId::deserialize_from(bytes)?)), - KEYWORD => Some(Value::Keyword(Keyword::deserialize_from(bytes)?)), - LIST => { - let len = bytes.next_leb128()?; - let mut items = Vec::with_capacity(len); - for _ in 0..len { - items.push(Value::deserialize_from(bytes)?); - } - Some(Value::List(items)) - } - OBJECT => Some(Value::Object(Object::deserialize_from(bytes)?)), - BLOB => Some(Value::Blob(Vec::deserialize_from(bytes)?)), - ACL => { - let len = bytes.next_leb128()?; - let mut items = Vec::with_capacity(len); - for _ in 0..len { - items.push(AclGrant::deserialize_from(bytes)?); - } - Some(Value::Acl(items)) - } - NULL => Some(Value::Null), - _ => None, + _ => {} } } } diff --git a/crates/jmap-proto/src/response/references.rs b/crates/jmap-proto/src/response/references.rs index 28ecffe4..2d6ac88e 100644 --- a/crates/jmap-proto/src/response/references.rs +++ b/crates/jmap-proto/src/response/references.rs @@ -11,16 +11,15 @@ use utils::map::vec_map::VecMap; use crate::{ error::set::SetError, method::{copy::CopyResponse, set::SetResponse, upload::DataSourceObject}, - object::Object, request::{ - reference::{MaybeReference, ResultReference}, RequestMethod, + reference::{MaybeReference, ResultReference}, }, types::{ any_id::AnyId, id::Id, property::Property, - value::{MaybePatchValue, SetValue, Value}, + value::{MaybePatchValue, Object, SetValue, Value}, }, }; @@ -154,8 +153,8 @@ impl Response { return Err(trc::JmapEvent::InvalidResultReference .into_err() .details(format!( - "Id reference {parent_id:?} points to invalid type." - ))); + "Id reference {parent_id:?} points to invalid type." + ))); } None => { graph @@ -193,7 +192,7 @@ impl Response { response .list .iter() - .filter_map(|obj| obj.properties.get(&property).cloned()) + .filter_map(|obj| obj.0.get(&property).cloned()) .collect(), ) } @@ -265,7 +264,7 @@ impl Response { obj: &mut Object, mut graph: Option<(&str, &mut HashMap>)>, ) -> trc::Result<()> { - for set_value in obj.properties.values_mut() { + for set_value in obj.0.values_mut() { match set_value { SetValue::IdReference(MaybeReference::Reference(parent_id)) => { if let Some(id) = self.created_ids.get(parent_id) { @@ -427,7 +426,7 @@ impl EvalObjectReferences for SetResponse { fn get_id(&self, id_ref: &str) -> Option { self.created .get(id_ref) - .and_then(|obj| obj.properties.get(&Property::Id)) + .and_then(|obj| obj.0.get(&Property::Id)) .and_then(|value| match value { Value::Id(id) => Value::Id(*id).into(), Value::BlobId(blob_id) => Value::BlobId(blob_id.clone()).into(), @@ -466,7 +465,7 @@ impl EvalResult { _ => { return Err(trc::JmapEvent::InvalidResultReference .into_err() - .details(format!("Failed to evaluate {rr} result reference."))) + .details(format!("Failed to evaluate {rr} result reference."))); } } } @@ -508,7 +507,7 @@ impl EvalResult { _ => { return Err(trc::JmapEvent::InvalidResultReference .into_err() - .details(format!("Failed to evaluate {rr} result reference."))) + .details(format!("Failed to evaluate {rr} result reference."))); } } } @@ -839,7 +838,7 @@ mod tests { .create .unwrap() .into_iter() - .map(|(p, mut v)| (p, v.properties.remove(&Property::ParentId).unwrap())) + .map(|(p, mut v)| (p, v.0.remove(&Property::ParentId).unwrap())) .collect::>(); assert_eq!( create.get("a").unwrap(), @@ -875,7 +874,7 @@ mod tests { .create .unwrap() .into_iter() - .map(|(p, mut v)| (p, v.properties.remove(&Property::ParentId).unwrap())) + .map(|(p, mut v)| (p, v.0.remove(&Property::ParentId).unwrap())) .collect::>(); assert_eq!( create.get("a1").unwrap(), diff --git a/crates/jmap-proto/src/types/date.rs b/crates/jmap-proto/src/types/date.rs index 923bc3f7..81345674 100644 --- a/crates/jmap-proto/src/types/date.rs +++ b/crates/jmap-proto/src/types/date.rs @@ -8,7 +8,7 @@ use std::fmt::Display; use store::Serialize; -use crate::parser::{json::Parser, JsonObjectParser}; +use crate::parser::{JsonObjectParser, json::Parser}; #[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] pub struct UTCDate { @@ -232,6 +232,12 @@ impl From for u64 { } } +impl From for UTCDate { + fn from(value: u64) -> Self { + UTCDate::from_timestamp(value as i64) + } +} + #[cfg(test)] mod tests { use crate::{parser::json::Parser, types::date::UTCDate}; diff --git a/crates/jmap-proto/src/types/property.rs b/crates/jmap-proto/src/types/property.rs index 4ebb5653..99a05fda 100644 --- a/crates/jmap-proto/src/types/property.rs +++ b/crates/jmap-proto/src/types/property.rs @@ -10,7 +10,7 @@ use mail_parser::HeaderName; use serde::Serialize; use store::write::{DeserializeFrom, SerializeInto}; -use crate::parser::{json::Parser, JsonObjectParser}; +use crate::parser::{JsonObjectParser, json::Parser}; use super::{acl::Acl, id::Id, keyword::Keyword, value::Value}; @@ -888,6 +888,127 @@ impl Display for Property { } } +impl Property { + pub fn as_str(&self) -> &str { + match self { + Property::Acl => "acl", + Property::Aliases => "aliases", + Property::Attachments => "attachments", + Property::Bcc => "bcc", + Property::BlobId => "blobId", + Property::BodyStructure => "bodyStructure", + Property::BodyValues => "bodyValues", + Property::Capabilities => "capabilities", + Property::Cc => "cc", + Property::Charset => "charset", + Property::Cid => "cid", + Property::DeliveryStatus => "deliveryStatus", + Property::Description => "description", + Property::DeviceClientId => "deviceClientId", + Property::Disposition => "disposition", + Property::DsnBlobIds => "dsnBlobIds", + Property::Email => "email", + Property::EmailId => "emailId", + Property::EmailIds => "emailIds", + Property::Envelope => "envelope", + Property::Expires => "expires", + Property::From => "from", + Property::FromDate => "fromDate", + Property::HasAttachment => "hasAttachment", + Property::Header(_) => "header", + Property::Headers => "headers", + Property::HtmlBody => "htmlBody", + Property::HtmlSignature => "htmlSignature", + Property::Id => "id", + Property::IdentityId => "identityId", + Property::InReplyTo => "inReplyTo", + Property::IsActive => "isActive", + Property::IsEnabled => "isEnabled", + Property::IsSubscribed => "isSubscribed", + Property::Keys => "keys", + Property::Keywords => "keywords", + Property::Language => "language", + Property::Location => "location", + Property::MailboxIds => "mailboxIds", + Property::MayDelete => "mayDelete", + Property::MdnBlobIds => "mdnBlobIds", + Property::Members => "members", + Property::MessageId => "messageId", + Property::MyRights => "myRights", + Property::Name => "name", + Property::ParentId => "parentId", + Property::PartId => "partId", + Property::Picture => "picture", + Property::Preview => "preview", + Property::Quota => "quota", + Property::ReceivedAt => "receivedAt", + Property::References => "references", + Property::ReplyTo => "replyTo", + Property::Role => "role", + Property::Secret => "secret", + Property::SendAt => "sendAt", + Property::Sender => "sender", + Property::SentAt => "sentAt", + Property::Size => "size", + Property::SortOrder => "sortOrder", + Property::Subject => "subject", + Property::SubParts => "subParts", + Property::TextBody => "textBody", + Property::TextSignature => "textSignature", + Property::ThreadId => "threadId", + Property::Timezone => "timezone", + Property::To => "to", + Property::ToDate => "toDate", + Property::TotalEmails => "totalEmails", + Property::TotalThreads => "totalThreads", + Property::Type => "type", + Property::Types => "types", + Property::UndoStatus => "undoStatus", + Property::UnreadEmails => "unreadEmails", + Property::UnreadThreads => "unreadThreads", + Property::Url => "url", + Property::VerificationCode => "verificationCode", + Property::Parameters => "parameters", + Property::Addresses => "addresses", + Property::P256dh => "p256dh", + Property::Auth => "auth", + Property::Value => "value", + Property::SmtpReply => "smtpReply", + Property::Delivered => "delivered", + Property::Displayed => "displayed", + Property::MailFrom => "mailFrom", + Property::RcptTo => "rcptTo", + Property::IsEncodingProblem => "isEncodingProblem", + Property::IsTruncated => "isTruncated", + Property::MayReadItems => "mayReadItems", + Property::MayAddItems => "mayAddItems", + Property::MayRemoveItems => "mayRemoveItems", + Property::MaySetSeen => "maySetSeen", + Property::MaySetKeywords => "maySetKeywords", + Property::MayCreateChild => "mayCreateChild", + Property::MayRename => "mayRename", + Property::MaySubmit => "maySubmit", + Property::ResourceType => "resourceType", + Property::Used => "used", + Property::HardLimit => "hardLimit", + Property::WarnLimit => "warnLimit", + Property::SoftLimit => "softLimit", + Property::Scope => "scope", + Property::Data(data) => match data { + DataProperty::AsText => "data:asText", + DataProperty::AsBase64 => "data:asBase64", + DataProperty::Default => "data", + }, + Property::Digest(digest) => match digest { + DigestProperty::Sha => "digest:sha", + DigestProperty::Sha256 => "digest:sha-256", + DigestProperty::Sha512 => "digest:sha-512", + }, + Property::_T(s) => s, + } + } +} + impl Display for SetProperty { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { self.property.fmt(f) @@ -934,11 +1055,7 @@ impl Display for HeaderProperty { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "header:{}", self.header)?; self.form.fmt(f)?; - if self.all { - write!(f, ":all") - } else { - Ok(()) - } + if self.all { write!(f, ":all") } else { Ok(()) } } } diff --git a/crates/jmap-proto/src/types/value.rs b/crates/jmap-proto/src/types/value.rs index 80373df4..93e6fba3 100644 --- a/crates/jmap-proto/src/types/value.rs +++ b/crates/jmap-proto/src/types/value.rs @@ -8,11 +8,13 @@ use std::{borrow::Cow, fmt::Display}; use mail_parser::{Addr, DateTime, Group}; use serde::Serialize; -use utils::map::bitmap::Bitmap; +use utils::{ + json::{JsonPointerItem, JsonQueryable}, + map::{bitmap::Bitmap, vec_map::VecMap}, +}; use crate::{ - object::Object, - parser::{json::Parser, Ignore, JsonObjectParser, Token}, + parser::{Ignore, JsonObjectParser, Token, json::Parser}, request::reference::{MaybeReference, ResultReference}, }; @@ -44,6 +46,9 @@ pub enum Value { Null, } +#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize)] +pub struct Object(pub VecMap); + #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize)] pub struct AclGrant { pub account_id: u32, @@ -82,13 +87,13 @@ impl Value { Ok(match token { Token::String(v) => v.into_value(), Token::DictStart => { - let mut properties = Object::with_capacity(4); + let mut properties = VecMap::with_capacity(4); while let Some(key) = parser.next_dict_key::()? { let property = key.into_property(); let value = Value::from_property(parser, &property)?; properties.append(property, value); } - Value::Object(properties) + Value::Object(Object(properties)) } Token::ArrayStart => { let mut values = Vec::with_capacity(4); @@ -467,20 +472,23 @@ impl> From> for Value { impl From> for Value { fn from(value: Addr<'_>) -> Self { - Value::Object( - Object::with_capacity(2) - .with_property(Property::Name, value.name) - .with_property(Property::Email, value.address.unwrap_or_default()), - ) + Value::Object(Object( + VecMap::with_capacity(2) + .with_append(Property::Name, Value::from(value.name)) + .with_append( + Property::Email, + Value::from(value.address.unwrap_or_default()), + ), + )) } } impl From> for Value { fn from(group: Group<'_>) -> Self { - Value::Object( - Object::with_capacity(2) - .with_property(Property::Name, group.name) - .with_property( + Value::Object(Object( + VecMap::with_capacity(2) + .with_append(Property::Name, Value::from(group.name)) + .with_append( Property::Addresses, Value::List( group @@ -490,6 +498,78 @@ impl From> for Value { .collect::>(), ), ), - ) + )) + } +} + +impl Object { + pub fn with_capacity(capacity: usize) -> Self { + Self(VecMap::with_capacity(capacity)) + } + + pub fn set(&mut self, property: Property, value: impl Into) -> bool { + self.0.set(property, value.into()) + } + + pub fn append(&mut self, property: Property, value: impl Into) { + self.0.append(property, value.into()); + } + + pub fn with_property(mut self, property: Property, value: impl Into) -> Self { + self.0.append(property, value.into()); + self + } + + pub fn remove(&mut self, property: &Property) -> Value { + self.0.remove(property).unwrap_or(Value::Null) + } + + pub fn get(&self, property: &Property) -> &Value { + self.0.get(property).unwrap_or(&Value::Null) + } +} + +impl JsonQueryable for Value { + fn eval_pointer<'x>( + &'x self, + mut pointer: std::slice::Iter, + results: &mut Vec<&'x dyn JsonQueryable>, + ) { + match pointer.next() { + Some(JsonPointerItem::String(n)) => { + if let Value::Object(map) = self { + if let Some(v) = map + .0 + .iter() + .find_map(|(k, v)| if k.as_str() == n { Some(v) } else { None }) + { + v.eval_pointer(pointer, results); + } + } + } + Some(JsonPointerItem::Number(n)) => { + if let Value::List(values) = self { + if let Some(v) = values.get(*n as usize) { + v.eval_pointer(pointer, results); + } + } + } + Some(JsonPointerItem::Wildcard) => match self { + Value::List(values) => { + for v in values { + v.eval_pointer(pointer.clone(), results); + } + } + Value::Object(map) => { + for v in map.0.values() { + v.eval_pointer(pointer.clone(), results); + } + } + _ => {} + }, + Some(JsonPointerItem::Root) | None => { + results.push(self); + } + } } } diff --git a/crates/jmap/src/api/form.rs b/crates/jmap/src/api/form.rs index 7b1da34c..f9bc27ad 100644 --- a/crates/jmap/src/api/form.rs +++ b/crates/jmap/src/api/form.rs @@ -8,24 +8,25 @@ use std::{borrow::Cow, fmt::Write, future::Future}; use chrono::Utc; use common::{ + KV_RATE_LIMIT_CONTACT, Server, config::network::{ContactForm, FieldOrDefault}, - ip_to_bytes, psl, Server, KV_RATE_LIMIT_CONTACT, + ip_to_bytes, psl, }; -use email::delivery::{IngestMessage, LocalDeliveryStatus, MailDelivery}; +use email::message::delivery::{IngestMessage, LocalDeliveryStatus, MailDelivery}; use hyper::StatusCode; use mail_auth::common::cache::NoCache; use mail_builder::{ + MessageBuilder, headers::{ - address::{Address, EmailAddress}, HeaderType, + address::{Address, EmailAddress}, }, mime::make_boundary, - MessageBuilder, }; use serde_json::json; use store::{ - write::{now, BatchBuilder, BlobOp}, Serialize, + write::{BatchBuilder, BlobOp, now}, }; use trc::AddContext; use utils::BlobHash; @@ -34,8 +35,8 @@ use x509_parser::nom::AsBytes; use crate::auth::oauth::FormData; use super::{ - http::{HttpSessionData, ToHttpResponse}, HttpResponse, JsonResponse, + http::{HttpSessionData, ToHttpResponse}, }; pub trait FormHandler: Sync + Send { diff --git a/crates/jmap/src/api/management/enterprise/undelete.rs b/crates/jmap/src/api/management/enterprise/undelete.rs index ecbad57b..88686a3e 100644 --- a/crates/jmap/src/api/management/enterprise/undelete.rs +++ b/crates/jmap/src/api/management/enterprise/undelete.rs @@ -10,12 +10,12 @@ use std::str::FromStr; -use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; -use common::{auth::AccessToken, enterprise::undelete::DeletedBlob, Server}; +use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; +use common::{Server, auth::AccessToken, enterprise::undelete::DeletedBlob}; use directory::backend::internal::manage::ManageDirectory; use email::{ - ingest::{EmailIngest, IngestEmail, IngestSource}, mailbox::INBOX_ID, + message::ingest::{EmailIngest, IngestEmail, IngestSource}, }; use hyper::Method; use jmap_proto::types::collection::Collection; @@ -24,13 +24,13 @@ use serde_json::json; use std::future::Future; use store::write::{BatchBuilder, BlobOp, ValueClass}; use trc::AddContext; -use utils::{url_params::UrlParams, BlobHash}; +use utils::{BlobHash, url_params::UrlParams}; use crate::{ api::{ + HttpRequest, HttpResponse, JsonResponse, http::{HttpSessionData, ToHttpResponse}, management::decode_path_element, - HttpRequest, HttpResponse, JsonResponse, }, blob::download::BlobDownload, }; diff --git a/crates/jmap/src/api/management/stores.rs b/crates/jmap/src/api/management/stores.rs index cffa9257..0c28e03f 100644 --- a/crates/jmap/src/api/management/stores.rs +++ b/crates/jmap/src/api/management/stores.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; use common::{ auth::AccessToken, ipc::{HousekeeperEvent, PurgeType}, @@ -12,27 +12,24 @@ use common::{ *, }; use directory::{ - backend::internal::manage::{self, ManageDirectory}, Permission, + backend::internal::manage::{self, ManageDirectory}, }; -use email::{ - ingest::EmailIngest, - mailbox::{UidMailbox, SCHEMA}, -}; +use email::{mailbox::UidMailbox, message::ingest::EmailIngest}; use hyper::Method; use jmap_proto::{ - object::{index::ObjectIndexBuilder, Object}, - types::{collection::Collection, property::Property, value::Value}, + object::index::ObjectIndexBuilder, + types::{collection::Collection, property::Property}, }; use serde_json::json; -use store::write::{assert::HashedValue, BatchBuilder, ValueClass, F_VALUE}; +use store::write::{BatchBuilder, F_VALUE, ValueClass, assert::HashedValue}; use trc::AddContext; use utils::url_params::UrlParams; use crate::{ api::{ - http::{HttpSessionData, ToHttpResponse}, HttpRequest, HttpResponse, JsonResponse, + http::{HttpSessionData, ToHttpResponse}, }, services::index::Indexer, }; @@ -340,7 +337,7 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u .unwrap_or_default() { let mailbox = server - .get_property::>>( + .get_property::>( account_id, Collection::Mailbox, mailbox_id, @@ -349,19 +346,17 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u .await .caused_by(trc::location!())? .ok_or_else(|| trc::ImapEvent::Error.into_err().caused_by(trc::location!()))?; - + let mut new_mailbox = mailbox.inner.clone(); + new_mailbox.uid_validity = rand::random::(); let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Mailbox) .update_document(mailbox_id) .custom( - ObjectIndexBuilder::new(SCHEMA) + ObjectIndexBuilder::new() .with_current(mailbox) - .with_changes(Object::with_capacity(1).with_property( - Property::Cid, - Value::UnsignedInt(rand::random::() as u64), - )), + .with_changes(new_mailbox), ) .clear(Property::EmailIds); server diff --git a/crates/jmap/src/auth/acl.rs b/crates/jmap/src/auth/acl.rs index 006e2232..439eb4e1 100644 --- a/crates/jmap/src/auth/acl.rs +++ b/crates/jmap/src/auth/acl.rs @@ -6,14 +6,13 @@ use std::future::Future; -use common::{auth::AccessToken, Server}; +use common::{Server, auth::AccessToken}; use directory::{ - backend::internal::{manage::ChangedPrincipals, PrincipalField}, QueryBy, Type, + backend::internal::{PrincipalField, manage::ChangedPrincipals}, }; use jmap_proto::{ error::set::SetError, - object::Object, types::{ acl::Acl, collection::Collection, @@ -21,12 +20,7 @@ use jmap_proto::{ value::{AclGrant, MaybePatchValue, Value}, }, }; -use store::{ - query::acl::AclQuery, - roaring::RoaringBitmap, - write::{assert::HashedValue, ValueClass}, - ValueKey, -}; +use store::{ValueKey, query::acl::AclQuery, roaring::RoaringBitmap, write::ValueClass}; use trc::AddContext; use utils::map::bitmap::Bitmap; @@ -72,8 +66,8 @@ pub trait AclMethods: Sync + Send { fn acl_set( &self, - changes: &mut Object, - current: Option<&HashedValue>>, + changes: &mut Vec, + current: Option<&[AclGrant]>, acl_changes: MaybePatchValue, ) -> impl Future> + Send; @@ -86,8 +80,8 @@ pub trait AclMethods: Sync + Send { fn refresh_acls( &self, - changes: &Object, - current: &Option>>, + changes: &[AclGrant], + current: Option<&[AclGrant]>, ) -> impl Future + Send; fn map_acl_set( @@ -255,38 +249,23 @@ impl AclMethods for Server { async fn acl_set( &self, - changes: &mut Object, - current: Option<&HashedValue>>, + changes: &mut Vec, + current: Option<&[AclGrant]>, acl_changes: MaybePatchValue, ) -> Result<(), SetError> { match acl_changes { MaybePatchValue::Value(Value::List(values)) => { - changes - .properties - .set(Property::Acl, Value::Acl(self.map_acl_set(values).await?)); + *changes = self.map_acl_set(values).await?; } MaybePatchValue::Patch(patch) => { let (mut patch, is_update) = self.map_acl_patch(patch).await?; - let acl = if let Value::Acl(acl) = - changes - .properties - .get_mut_or_insert_with(Property::Acl, || { - current - .and_then(|current| { - current.inner.properties.get(&Property::Acl).cloned() - }) - .unwrap_or_else(|| Value::Acl(Vec::new())) - }) { - acl - } else { - return Err(SetError::invalid_properties() - .with_property(Property::Acl) - .with_description("Invalid ACL value found.")); - }; + if let Some(changes_) = current { + *changes = changes_.to_vec(); + } if let Some(is_set) = is_update { if !patch.grants.is_empty() { - if let Some(acl_item) = acl + if let Some(acl_item) = changes .iter_mut() .find(|item| item.account_id == patch.account_id) { @@ -296,30 +275,30 @@ impl AclMethods for Server { } else { acl_item.grants.remove(item); if acl_item.grants.is_empty() { - acl.retain(|item| item.account_id != patch.account_id); + changes.retain(|item| item.account_id != patch.account_id); } } } else if is_set { - acl.push(patch); + changes.push(patch); } } } else if !patch.grants.is_empty() { - if let Some(acl_item) = acl + if let Some(acl_item) = changes .iter_mut() .find(|item| item.account_id == patch.account_id) { acl_item.grants = patch.grants; } else { - acl.push(patch); + changes.push(patch); } } else { - acl.retain(|item| item.account_id != patch.account_id); + changes.retain(|item| item.account_id != patch.account_id); } } _ => { return Err(SetError::invalid_properties() .with_property(Property::Acl) - .with_description("Invalid ACL property.")) + .with_description("Invalid ACL property.")); } } Ok(()) @@ -336,7 +315,7 @@ impl AclMethods for Server { access_token.is_member(item.account_id) && item.grants.contains(Acl::Administer) }) { - let mut acl_obj = Object::with_capacity(value.len() / 2); + let mut acl_obj = jmap_proto::types::value::Object::with_capacity(value.len() / 2); for item in value { if let Some(mut principal) = self .core @@ -361,62 +340,53 @@ impl AclMethods for Server { } } - async fn refresh_acls( - &self, - changes: &Object, - current: &Option>>, - ) { - if let Value::Acl(acl_changes) = changes.get(&Property::Acl) { - let mut changed_principals = ChangedPrincipals::new(); - if let Some(Value::Acl(acl_current)) = current - .as_ref() - .and_then(|current| current.inner.properties.get(&Property::Acl)) - { - for current_item in acl_current { - let mut invalidate = true; - for change_item in acl_changes { - if change_item.account_id == current_item.account_id { - invalidate = change_item.grants != current_item.grants; - break; - } - } - if invalidate { - changed_principals.add_change( - current_item.account_id, - Type::Individual, - PrincipalField::EnabledPermissions, - ); - } - } - + async fn refresh_acls(&self, acl_changes: &[AclGrant], current: Option<&[AclGrant]>) { + let mut changed_principals = ChangedPrincipals::new(); + if let Some(acl_current) = current { + for current_item in acl_current { + let mut invalidate = true; for change_item in acl_changes { - let mut invalidate = true; - for current_item in acl_current { - if change_item.account_id == current_item.account_id { - invalidate = change_item.grants != current_item.grants; - break; - } - } - if invalidate { - changed_principals.add_change( - change_item.account_id, - Type::Individual, - PrincipalField::EnabledPermissions, - ); + if change_item.account_id == current_item.account_id { + invalidate = change_item.grants != current_item.grants; + break; } } - } else { - for value in acl_changes { + if invalidate { changed_principals.add_change( - value.account_id, + current_item.account_id, Type::Individual, PrincipalField::EnabledPermissions, ); } } - self.increment_token_revision(changed_principals).await; + for change_item in acl_changes { + let mut invalidate = true; + for current_item in acl_current { + if change_item.account_id == current_item.account_id { + invalidate = change_item.grants != current_item.grants; + break; + } + } + if invalidate { + changed_principals.add_change( + change_item.account_id, + Type::Individual, + PrincipalField::EnabledPermissions, + ); + } + } + } else { + for value in acl_changes { + changed_principals.add_change( + value.account_id, + Type::Individual, + PrincipalField::EnabledPermissions, + ); + } } + + self.increment_token_revision(changed_principals).await; } async fn map_acl_set(&self, acl_set: Vec) -> Result, SetError> { @@ -497,14 +467,12 @@ pub trait EffectiveAcl { fn effective_acl(&self, access_token: &AccessToken) -> Bitmap; } -impl EffectiveAcl for Object { +impl EffectiveAcl for Vec { fn effective_acl(&self, access_token: &AccessToken) -> Bitmap { let mut acl = Bitmap::::new(); - if let Some(Value::Acl(permissions)) = self.properties.get(&Property::Acl) { - for item in permissions { - if access_token.is_member(item.account_id) { - acl.union(&item.grants); - } + for item in self { + if access_token.is_member(item.account_id) { + acl.union(&item.grants); } } diff --git a/crates/jmap/src/blob/get.rs b/crates/jmap/src/blob/get.rs index cdc3b115..d9ac3297 100644 --- a/crates/jmap/src/blob/get.rs +++ b/crates/jmap/src/blob/get.rs @@ -4,21 +4,21 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{auth::AccessToken, Server}; +use common::{Server, auth::AccessToken}; use email::mailbox::UidMailbox; use jmap_proto::{ method::{ get::{GetRequest, GetResponse}, lookup::{BlobInfo, BlobLookupRequest, BlobLookupResponse}, }, - object::{blob::GetArguments, Object}, + object::blob::GetArguments, types::{ + MaybeUnparsable, collection::Collection, id::Id, property::{DataProperty, DigestProperty, Property}, type_state::DataType, - value::Value, - MaybeUnparsable, + value::{Object, Value}, }, }; use mail_builder::encoders::base64::base64_encode; diff --git a/crates/jmap/src/email/body.rs b/crates/jmap/src/email/body.rs index 31860dd0..611240cc 100644 --- a/crates/jmap/src/email/body.rs +++ b/crates/jmap/src/email/body.rs @@ -4,10 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use email::metadata::{MessageMetadataContents, MetadataPartType}; -use jmap_proto::{ - object::Object, - types::{blob::BlobId, property::Property, value::Value}, +use email::message::metadata::{MessageMetadataContents, MetadataPartType}; +use jmap_proto::types::{ + blob::BlobId, + property::Property, + value::{Object, Value}, }; use mail_parser::{HeaderValue, MessagePart, MimeHeaders, PartType}; diff --git a/crates/jmap/src/email/copy.rs b/crates/jmap/src/email/copy.rs index 7be91279..5bb1b064 100644 --- a/crates/jmap/src/email/copy.rs +++ b/crates/jmap/src/email/copy.rs @@ -5,14 +5,19 @@ */ use common::{ - auth::{AccessToken, ResourceToken}, Server, + auth::{AccessToken, ResourceToken}, }; + use email::{ - index::{EmailIndexBuilder, TrimTextValue, VisitValues, MAX_ID_LENGTH, MAX_SORT_FIELD_LENGTH}, - ingest::{EmailIngest, IngestedEmail, LogEmailInsert}, - mailbox::{MailboxFnc, UidMailbox}, - metadata::MessageMetadata, + mailbox::{UidMailbox, manage::MailboxFnc}, + message::{ + index::{ + EmailIndexBuilder, MAX_ID_LENGTH, MAX_SORT_FIELD_LENGTH, TrimTextValue, VisitValues, + }, + ingest::{EmailIngest, IngestedEmail, LogEmailInsert}, + metadata::MessageMetadata, + }, }; use jmap_proto::{ error::set::SetError, @@ -21,9 +26,9 @@ use jmap_proto::{ set::{self, SetRequest}, }, request::{ + Call, RequestMethod, method::{MethodFunction, MethodName, MethodObject}, reference::MaybeReference, - Call, RequestMethod, }, response::references::EvalObjectReferences, types::{ @@ -39,14 +44,14 @@ use jmap_proto::{ value::{MaybePatchValue, Value}, }, }; -use mail_parser::{parsers::fields::thread::thread_name, HeaderName, HeaderValue}; +use mail_parser::{HeaderName, HeaderValue, parsers::fields::thread::thread_name}; use store::{ - write::{ - log::{Changes, LogInsert}, - BatchBuilder, Bincode, MaybeDynamicId, TagValue, TaskQueueClass, ValueClass, F_BITMAP, - F_VALUE, - }, BlobClass, + write::{ + BatchBuilder, Bincode, F_BITMAP, F_VALUE, MaybeDynamicId, TagValue, TaskQueueClass, + ValueClass, + log::{Changes, LogInsert}, + }, }; use trc::AddContext; use utils::map::vec_map::VecMap; @@ -140,7 +145,7 @@ impl EmailCopy for Server { let mut keywords = Vec::new(); let mut received_at = None; - for (property, value) in create.properties { + for (property, value) in create.0 { let value = match response.eval_object_references(value) { Ok(value) => value, Err(err) => { diff --git a/crates/jmap/src/email/crypto.rs b/crates/jmap/src/email/crypto.rs index fc6f5129..a3840c1e 100644 --- a/crates/jmap/src/email/crypto.rs +++ b/crates/jmap/src/email/crypto.rs @@ -6,18 +6,21 @@ use std::{future::Future, sync::Arc}; -use crate::api::{http::ToHttpResponse, HttpResponse, JsonResponse}; -use common::{auth::AccessToken, Server}; +use crate::api::{HttpResponse, JsonResponse, http::ToHttpResponse}; +use common::{Server, auth::AccessToken}; use directory::backend::internal::manage; -use email::crypto::{ - try_parse_certs, EncryptMessage, EncryptMessageError, EncryptionMethod, EncryptionParams, - EncryptionType, +use email::message::crypto::{ + EncryptMessage, EncryptMessageError, EncryptionMethod, EncryptionParams, EncryptionType, + try_parse_certs, }; use jmap_proto::types::{collection::Collection, property::Property}; use mail_builder::encoders::base64::base64_encode_mime; use mail_parser::MessageParser; use serde_json::json; -use store::{write::{BatchBuilder, Bincode, F_CLEAR, F_VALUE}, Serialize}; +use store::{ + Serialize, + write::{BatchBuilder, Bincode, F_CLEAR, F_VALUE}, +}; pub trait CryptoHandler: Sync + Send { fn handle_crypto_get( diff --git a/crates/jmap/src/email/get.rs b/crates/jmap/src/email/get.rs index 4684c890..efcc62d5 100644 --- a/crates/jmap/src/email/get.rs +++ b/crates/jmap/src/email/get.rs @@ -4,15 +4,16 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{auth::AccessToken, Server}; +use common::{Server, auth::AccessToken}; + use email::{ - cache::ThreadCache, mailbox::UidMailbox, - metadata::{MessageMetadata, MetadataPartType}, + message::metadata::{MessageMetadata, MetadataPartType}, + thread::cache::ThreadCache, }; use jmap_proto::{ method::get::{GetRequest, GetResponse}, - object::{email::GetArguments, Object}, + object::email::GetArguments, types::{ acl::Acl, blob::BlobId, @@ -21,11 +22,11 @@ use jmap_proto::{ id::Id, keyword::Keyword, property::{HeaderForm, Property}, - value::Value, + value::{Object, Value}, }, }; use mail_parser::HeaderName; -use store::{write::Bincode, BlobClass}; +use store::{BlobClass, write::Bincode}; use trc::{AddContext, StoreEvent}; use crate::{ diff --git a/crates/jmap/src/email/headers.rs b/crates/jmap/src/email/headers.rs index 0ceee69d..3e84c0e1 100644 --- a/crates/jmap/src/email/headers.rs +++ b/crates/jmap/src/email/headers.rs @@ -6,14 +6,12 @@ use std::borrow::Cow; -use jmap_proto::{ - object::Object, - types::{ - property::{HeaderForm, HeaderProperty, Property}, - value::Value, - }, +use jmap_proto::types::{ + property::{HeaderForm, HeaderProperty, Property}, + value::{Object, Value}, }; use mail_builder::{ + MessageBuilder, headers::{ address::{Address, EmailAddress, GroupedAddresses}, date::Date, @@ -22,9 +20,8 @@ use mail_builder::{ text::Text, url::URL, }, - MessageBuilder, }; -use mail_parser::{parsers::MessageStream, Addr, Header, HeaderName, HeaderValue}; +use mail_parser::{Addr, Header, HeaderName, HeaderValue, parsers::MessageStream}; pub trait IntoForm { fn into_form(self, form: &HeaderForm) -> Value; @@ -169,10 +166,12 @@ impl IntoForm for HeaderValue<'_> { ( HeaderValue::Address(mail_parser::Address::List(addrlist)), HeaderForm::GroupedAddresses, - ) => Value::List(vec![Object::with_capacity(2) - .with_property(Property::Name, Value::Null) - .with_property(Property::Addresses, addrlist) - .into()]), + ) => Value::List(vec![ + Object::with_capacity(2) + .with_property(Property::Name, Value::Null) + .with_property(Property::Addresses, addrlist) + .into(), + ]), ( HeaderValue::Address(mail_parser::Address::Group(grouplist)), HeaderForm::GroupedAddresses, @@ -188,12 +187,12 @@ impl<'x> ValueToHeader<'x> for Value { let mut obj = self.try_unwrap_object()?; Some(GroupedAddresses { name: obj - .properties + .0 .remove(&Property::Name) .and_then(|n| n.try_unwrap_string()) .map(|n| n.into()), addresses: obj - .properties + .0 .remove(&Property::Addresses)? .try_into_address_list()?, }) @@ -212,15 +211,11 @@ impl<'x> ValueToHeader<'x> for Value { let mut obj = self.try_unwrap_object()?; Some(EmailAddress { name: obj - .properties + .0 .remove(&Property::Name) .and_then(|n| n.try_unwrap_string()) .map(|n| n.into()), - email: obj - .properties - .remove(&Property::Email)? - .try_unwrap_string()? - .into(), + email: obj.0.remove(&Property::Email)?.try_unwrap_string()?.into(), }) } } diff --git a/crates/jmap/src/email/import.rs b/crates/jmap/src/email/import.rs index 1aa12b79..f59c2ea8 100644 --- a/crates/jmap/src/email/import.rs +++ b/crates/jmap/src/email/import.rs @@ -4,10 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{auth::AccessToken, Server}; +use common::{Server, auth::AccessToken}; use email::{ - ingest::{EmailIngest, IngestEmail, IngestSource}, - mailbox::MailboxFnc, + mailbox::manage::MailboxFnc, + message::ingest::{EmailIngest, IngestEmail, IngestSource}, }; use jmap_proto::{ error::set::{SetError, SetErrorType}, diff --git a/crates/jmap/src/email/mod.rs b/crates/jmap/src/email/mod.rs index 0993bbc4..cb46e16f 100644 --- a/crates/jmap/src/email/mod.rs +++ b/crates/jmap/src/email/mod.rs @@ -4,11 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -pub mod bayes; pub mod body; pub mod copy; pub mod crypto; -pub mod delete; pub mod get; pub mod headers; pub mod import; diff --git a/crates/jmap/src/email/parse.rs b/crates/jmap/src/email/parse.rs index 3b55a484..17abbdfe 100644 --- a/crates/jmap/src/email/parse.rs +++ b/crates/jmap/src/email/parse.rs @@ -4,15 +4,17 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{auth::AccessToken, Server}; -use email::index::PREVIEW_LENGTH; +use common::{Server, auth::AccessToken}; +use email::message::index::PREVIEW_LENGTH; use jmap_proto::{ method::parse::{ParseEmailRequest, ParseEmailResponse}, - object::Object, - types::{property::Property, value::Value}, + types::{ + property::Property, + value::{Object, Value}, + }, }; use mail_parser::{ - decoders::html::html_to_text, parsers::preview::preview_text, MessageParser, PartType, + MessageParser, PartType, decoders::html::html_to_text, parsers::preview::preview_text, }; use std::future::Future; use utils::map::vec_map::VecMap; diff --git a/crates/jmap/src/email/query.rs b/crates/jmap/src/email/query.rs index f75bc62b..4f656d68 100644 --- a/crates/jmap/src/email/query.rs +++ b/crates/jmap/src/email/query.rs @@ -4,8 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{auth::AccessToken, Server}; -use email::cache::ThreadCache; +use common::{Server, auth::AccessToken}; +use email::thread::cache::ThreadCache; use jmap_proto::{ method::query::{Comparator, Filter, QueryRequest, QueryResponse, SortProperty}, object::email::QueryArguments, @@ -15,14 +15,14 @@ use mail_parser::HeaderName; use nlp::language::Language; use std::future::Future; use store::{ + ValueKey, fts::{Field, FilterGroup, FtsFilter, IntoFilterGroup}, query::{self}, roaring::RoaringBitmap, write::ValueClass, - ValueKey, }; -use crate::{auth::acl::AclMethods, JmapMethods}; +use crate::{JmapMethods, auth::acl::AclMethods}; pub trait EmailQuery: Sync + Send { fn email_query( @@ -175,7 +175,7 @@ impl EmailQuery for Server { other => { return Err(trc::JmapEvent::UnsupportedFilter .into_err() - .details(other.to_string())) + .details(other.to_string())); } } } @@ -274,7 +274,7 @@ impl EmailQuery for Server { other => { return Err(trc::JmapEvent::UnsupportedFilter .into_err() - .details(other.to_string())) + .details(other.to_string())); } } } @@ -354,7 +354,7 @@ impl EmailQuery for Server { other => { return Err(trc::JmapEvent::UnsupportedSort .into_err() - .details(other.to_string())) + .details(other.to_string())); } }); } diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index 44460821..469113d5 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -6,10 +6,13 @@ use std::{borrow::Cow, collections::HashMap, slice::IterMut}; -use common::{auth::AccessToken, Server}; +use common::{Server, auth::AccessToken}; use email::{ - ingest::{EmailIngest, IngestEmail, IngestSource}, - mailbox::{MailboxFnc, UidMailbox}, + mailbox::{UidMailbox, manage::MailboxFnc}, + message::{ + delete::EmailDeletion, + ingest::{EmailIngest, IngestEmail, IngestSource}, + }, }; use jmap_proto::{ error::set::{SetError, SetErrorType}, @@ -26,35 +29,32 @@ use jmap_proto::{ }, }; use mail_builder::{ + MessageBuilder, headers::{ - address::Address, content_type::ContentType, date::Date, message_id::MessageId, raw::Raw, - text::Text, HeaderType, + HeaderType, address::Address, content_type::ContentType, date::Date, message_id::MessageId, + raw::Raw, text::Text, }, mime::{BodyPart, MimePart}, - MessageBuilder, }; use mail_parser::MessageParser; use store::{ + Serialize, ahash::AHashSet, roaring::RoaringBitmap, write::{ - assert::HashedValue, log::ChangeLogBuilder, BatchBuilder, DeserializeFrom, SerializeInto, - ToBitmaps, ValueClass, F_BITMAP, F_CLEAR, F_VALUE, + BatchBuilder, DeserializeFrom, F_BITMAP, F_CLEAR, F_VALUE, SerializeInto, ToBitmaps, + ValueClass, assert::HashedValue, log::ChangeLogBuilder, }, - Serialize, }; use trc::AddContext; use crate::{ - api::http::HttpSessionData, auth::acl::AclMethods, blob::download::BlobDownload, - changes::state::StateManager, JmapMethods, + JmapMethods, api::http::HttpSessionData, auth::acl::AclMethods, blob::download::BlobDownload, + changes::state::StateManager, }; use std::future::Future; -use super::{ - delete::EmailDeletion, - headers::{BuildHeader, ValueToHeader}, -}; +use super::headers::{BuildHeader, ValueToHeader}; pub trait EmailSet: Sync + Send { fn email_set( @@ -112,7 +112,7 @@ impl EmailSet for Server { // Process creates 'create: for (id, mut object) in request.unwrap_create() { let has_body_structure = object - .properties + .0 .keys() .any(|key| matches!(key, Property::BodyStructure)); let mut builder = MessageBuilder::new(); @@ -121,33 +121,25 @@ impl EmailSet for Server { let mut received_at = None; // Parse body values - let body_values = object - .properties - .remove(&Property::BodyValues) - .and_then(|obj| { - if let SetValue::Value(Value::Object(obj)) = obj { - let mut values = HashMap::with_capacity(obj.properties.len()); - for (key, value) in obj.properties { - if let (Property::_T(id), Value::Object(mut bv)) = (key, value) { - values.insert( - id, - bv.properties - .remove(&Property::Value)? - .try_unwrap_string()?, - ); - } else { - return None; - } + let body_values = object.0.remove(&Property::BodyValues).and_then(|obj| { + if let SetValue::Value(Value::Object(obj)) = obj { + let mut values = HashMap::with_capacity(obj.0.len()); + for (key, value) in obj.0 { + if let (Property::_T(id), Value::Object(mut bv)) = (key, value) { + values.insert(id, bv.0.remove(&Property::Value)?.try_unwrap_string()?); + } else { + return None; } - Some(values) - } else { - None } - }); + Some(values) + } else { + None + } + }); let mut size_attachments = 0; // Parse properties - for (property, value) in object.properties { + for (property, value) in object.0 { let value = match response.eval_object_references(value) { Ok(value) => value, Err(err) => { @@ -306,7 +298,7 @@ impl EmailSet for Server { let mut headers: Vec<(Cow, HeaderType)> = Vec::new(); if let Some(obj) = value.try_unwrap_object() { - for (body_property, value) in obj.properties { + for (body_property, value) in obj.0 { match (body_property, value) { (Property::Type, Value::Text(value)) => { content_type = value.into(); @@ -797,7 +789,7 @@ impl EmailSet for Server { .with_account_id(account_id) .with_collection(Collection::Email); - for (property, value) in object.properties { + for (property, value) in object.0 { let value = match response.eval_object_references(value) { Ok(value) => value, Err(err) => { @@ -1097,9 +1089,8 @@ enum LastTag { None, } -impl< - T: PartialEq + Clone + ToBitmaps + SerializeInto + Serialize + DeserializeFrom + Sync + Send, - > TagManager +impl + TagManager { pub fn new(current: HashedValue>) -> Self { Self { diff --git a/crates/jmap/src/email/snippet.rs b/crates/jmap/src/email/snippet.rs index a9b302f7..5cd6b007 100644 --- a/crates/jmap/src/email/snippet.rs +++ b/crates/jmap/src/email/snippet.rs @@ -4,8 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{auth::AccessToken, Server}; -use email::metadata::{MessageMetadata, MetadataPartType}; +use common::{Server, auth::AccessToken}; +use email::message::metadata::{MessageMetadata, MetadataPartType}; use jmap_proto::{ method::{ query::Filter, @@ -13,8 +13,8 @@ use jmap_proto::{ }, types::{acl::Acl, collection::Collection, property::Property}, }; -use mail_parser::{decoders::html::html_to_text, GetHeader, HeaderName, PartType}; -use nlp::language::{search_snippet::generate_snippet, stemmer::Stemmer, Language}; +use mail_parser::{GetHeader, HeaderName, PartType, decoders::html::html_to_text}; +use nlp::language::{Language, search_snippet::generate_snippet, stemmer::Stemmer}; use store::{backend::MAX_TOKEN_LENGTH, write::Bincode}; use crate::{auth::acl::AclMethods, blob::download::BlobDownload}; diff --git a/crates/jmap/src/identity/get.rs b/crates/jmap/src/identity/get.rs index 1e75eeed..4734c2bd 100644 --- a/crates/jmap/src/identity/get.rs +++ b/crates/jmap/src/identity/get.rs @@ -5,16 +5,17 @@ */ use common::Server; -use directory::{backend::internal::PrincipalField, QueryBy}; +use directory::{QueryBy, backend::internal::PrincipalField}; +use email::identity::{EmailAddress, Identity}; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, - object::Object, - types::{collection::Collection, property::Property, value::Value}, -}; -use store::{ - roaring::RoaringBitmap, - write::{BatchBuilder, F_VALUE}, + types::{ + collection::Collection, + property::Property, + value::{Object, Value}, + }, }; +use store::{Serialize, roaring::RoaringBitmap, write::BatchBuilder}; use trc::AddContext; use utils::sanitize_email; @@ -79,7 +80,7 @@ impl IdentityGet for Server { continue; } let mut identity = if let Some(identity) = self - .get_property::>( + .get_property::( account_id, Collection::Identity, document_id, @@ -101,17 +102,32 @@ impl IdentityGet for Server { Property::MayDelete => { result.append(Property::MayDelete, Value::Bool(true)); } - Property::TextSignature | Property::HtmlSignature => { + Property::Name => { + result.append(Property::Name, std::mem::take(&mut identity.name)); + } + Property::Email => { + result.append(Property::Email, std::mem::take(&mut identity.email)); + } + Property::TextSignature => { result.append( - property.clone(), - identity - .properties - .remove(property) - .unwrap_or(Value::Text(String::new())), + Property::TextSignature, + std::mem::take(&mut identity.text_signature), ); } + Property::HtmlSignature => { + result.append( + Property::HtmlSignature, + std::mem::take(&mut identity.html_signature), + ); + } + Property::Bcc => { + result.append(Property::Bcc, email_to_value(identity.bcc.take())); + } + Property::ReplyTo => { + result.append(Property::ReplyTo, email_to_value(identity.reply_to.take())); + } property => { - result.append(property.clone(), identity.remove(property)); + result.append(property.clone(), Value::Null); } } } @@ -169,12 +185,14 @@ impl IdentityGet for Server { } else { name.clone() }; - batch.create_document_with_id(document_id).value( + batch.create_document_with_id(document_id).set( Property::Value, - Object::with_capacity(4) - .with_property(Property::Name, name) - .with_property(Property::Email, email), - F_VALUE, + Identity { + name, + email, + ..Default::default() + } + .serialize(), ); identity_ids.insert(document_id); } @@ -188,3 +206,22 @@ impl IdentityGet for Server { Ok(identity_ids) } } + +fn email_to_value(email: Option>) -> Value { + if let Some(email) = email { + Value::List( + email + .into_iter() + .map(|email| { + Value::Object( + Object::with_capacity(2) + .with_property(Property::Name, email.name) + .with_property(Property::Email, email.email), + ) + }) + .collect(), + ) + } else { + Value::Null + } +} diff --git a/crates/jmap/src/identity/set.rs b/crates/jmap/src/identity/set.rs index aad44a2c..64d158d7 100644 --- a/crates/jmap/src/identity/set.rs +++ b/crates/jmap/src/identity/set.rs @@ -5,11 +5,11 @@ */ use common::Server; -use directory::{backend::internal::PrincipalField, QueryBy}; +use directory::{QueryBy, backend::internal::PrincipalField}; +use email::identity::{EmailAddress, Identity}; use jmap_proto::{ error::set::SetError, method::set::{RequestArguments, SetRequest, SetResponse}, - object::Object, response::references::EvalObjectReferences, types::{ collection::Collection, @@ -18,7 +18,8 @@ use jmap_proto::{ }, }; use std::future::Future; -use store::write::{log::ChangeLogBuilder, BatchBuilder, F_CLEAR, F_VALUE}; +use store::Serialize; +use store::write::{BatchBuilder, F_CLEAR, F_VALUE, log::ChangeLogBuilder}; use trc::AddContext; use utils::sanitize_email; @@ -45,26 +46,19 @@ impl IdentitySet for Server { // Process creates let mut changes = ChangeLogBuilder::new(); 'create: for (id, object) in request.unwrap_create() { - let mut identity = Object::with_capacity(object.properties.len()); + let mut identity = Identity::default(); - for (property, value) in object.properties { - match response - .eval_object_references(value) - .and_then(|value| validate_identity_value(&property, value, None)) - { - Ok(Value::Null) => (), - Ok(value) => { - identity.set(property, value); - } - Err(err) => { - response.not_created.append(id, err); - continue 'create; - } + for (property, value) in object.0 { + if let Err(err) = response.eval_object_references(value).and_then(|value| { + validate_identity_value(&property, value, &mut identity, true) + }) { + response.not_created.append(id, err); + continue 'create; } } // Validate email address - if let Value::Text(email) = identity.get(&Property::Email) { + if !identity.email.is_empty() { if !self .core .storage @@ -72,7 +66,7 @@ impl IdentitySet for Server { .query(QueryBy::Id(account_id), false) .await? .unwrap_or_default() - .has_str_value(PrincipalField::Emails, email) + .has_str_value(PrincipalField::Emails, &identity.email) { response.not_created.append( id, @@ -100,7 +94,7 @@ impl IdentitySet for Server { .with_account_id(account_id) .with_collection(Collection::Identity) .create_document() - .value(Property::Value, identity, F_VALUE); + .set(Property::Value, identity.serialize()); let document_id = self .store() .write_expect_id(batch) @@ -122,7 +116,7 @@ impl IdentitySet for Server { // Obtain identity let document_id = id.document_id(); let mut identity = if let Some(identity) = self - .get_property::>( + .get_property::( account_id, Collection::Identity, document_id, @@ -136,22 +130,13 @@ impl IdentitySet for Server { continue 'update; }; - for (property, value) in object.properties { - match response - .eval_object_references(value) - .and_then(|value| validate_identity_value(&property, value, Some(&identity))) - { - Ok(Value::Null) => { - identity.remove(&property); - } - Ok(value) => { - identity.set(property, value); - } - Err(err) => { - response.not_updated.append(id, err); - continue 'update; - } - }; + for (property, value) in object.0 { + if let Err(err) = response.eval_object_references(value).and_then(|value| { + validate_identity_value(&property, value, &mut identity, false) + }) { + response.not_updated.append(id, err); + continue 'update; + } } // Update record @@ -160,7 +145,7 @@ impl IdentitySet for Server { .with_account_id(account_id) .with_collection(Collection::Identity) .update_document(document_id) - .value(Property::Value, identity, F_VALUE); + .set(Property::Value, identity.serialize()); self.store() .write(batch) .await @@ -203,35 +188,50 @@ impl IdentitySet for Server { fn validate_identity_value( property: &Property, value: MaybePatchValue, - current: Option<&Object>, -) -> Result { - Ok(match (property, value) { + identity: &mut Identity, + is_create: bool, +) -> Result<(), SetError> { + match (property, value) { (Property::Name, MaybePatchValue::Value(Value::Text(value))) if value.len() < 255 => { - Value::Text(value) + identity.name = value; } (Property::Email, MaybePatchValue::Value(Value::Text(value))) - if current.is_none() && value.len() < 255 => + if is_create && value.len() < 255 => { - Value::Text(sanitize_email(&value).ok_or_else(|| { + identity.email = sanitize_email(&value).ok_or_else(|| { SetError::invalid_properties() .with_property(Property::Email) .with_description("Invalid e-mail address.") - })?) + })?; + } + (Property::TextSignature, MaybePatchValue::Value(Value::Text(value))) + if value.len() < 2048 => + { + identity.text_signature = value; + } + (Property::HtmlSignature, MaybePatchValue::Value(Value::Text(value))) + if value.len() < 2048 => + { + identity.html_signature = value; } - ( - Property::TextSignature | Property::HtmlSignature, - MaybePatchValue::Value(Value::Text(value)), - ) if value.len() < 2048 => Value::Text(value), (Property::ReplyTo | Property::Bcc, MaybePatchValue::Value(Value::List(value))) => { - for addr in &value { + let mut addresses = Vec::with_capacity(value.len()); + for addr in value { + let mut address = EmailAddress { + name: None, + email: String::new(), + }; let mut is_valid = false; if let Value::Object(obj) = addr { - for (key, value) in &obj.properties { + for (key, value) in obj.0 { match (key, value) { (Property::Email, Value::Text(value)) if value.len() < 255 => { - is_valid = true + is_valid = true; + address.email = value; + } + (Property::Name, Value::Text(value)) if value.len() < 255 => { + address.name = Some(value); } - (Property::Name, Value::Text(value)) if value.len() < 255 => (), (Property::Name, Value::Null) => (), _ => { is_valid = false; @@ -241,28 +241,42 @@ fn validate_identity_value( } } - if !is_valid { + if is_valid && !address.email.is_empty() { + addresses.push(address); + } else { return Err(SetError::invalid_properties() .with_property(property.clone()) .with_description("Invalid e-mail address object.")); } } - Value::List(value) + match property { + Property::ReplyTo => { + identity.reply_to = Some(addresses); + } + Property::Bcc => { + identity.bcc = Some(addresses); + } + _ => unreachable!(), + } } - ( - Property::Name - | Property::TextSignature - | Property::HtmlSignature - | Property::ReplyTo - | Property::Bcc, - MaybePatchValue::Value(Value::Null), - ) => Value::Null, - + (Property::Name, MaybePatchValue::Value(Value::Null)) => { + identity.name.clear(); + } + (Property::TextSignature, MaybePatchValue::Value(Value::Null)) => { + identity.text_signature.clear(); + } + (Property::HtmlSignature, MaybePatchValue::Value(Value::Null)) => { + identity.html_signature.clear(); + } + (Property::ReplyTo, MaybePatchValue::Value(Value::Null)) => identity.reply_to = None, + (Property::Bcc, MaybePatchValue::Value(Value::Null)) => identity.bcc = None, (property, _) => { return Err(SetError::invalid_properties() .with_property(property.clone()) .with_description("Field could not be set.")); } - }) + } + + Ok(()) } diff --git a/crates/jmap/src/mailbox/get.rs b/crates/jmap/src/mailbox/get.rs index 05283e21..8b5de409 100644 --- a/crates/jmap/src/mailbox/get.rs +++ b/crates/jmap/src/mailbox/get.rs @@ -4,12 +4,16 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{auth::AccessToken, Server}; -use email::mailbox::MailboxFnc; +use common::{Server, auth::AccessToken}; +use email::mailbox::{Mailbox, manage::MailboxFnc}; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, - object::Object, - types::{acl::Acl, collection::Collection, property::Property, value::Value}, + types::{ + acl::Acl, + collection::Collection, + property::Property, + value::{Object, Value}, + }, }; use crate::{ @@ -95,7 +99,7 @@ impl MailboxGet for Server { let mut values = if fetch_properties { match self - .get_property::>( + .get_property::( account_id, Collection::Mailbox, document_id, @@ -109,8 +113,9 @@ impl MailboxGet for Server { continue; } } + .into() } else { - Object::with_capacity(0) + None }; let mut mailbox = Object::with_capacity(properties.len()); @@ -118,21 +123,32 @@ impl MailboxGet for Server { for property in &properties { let value = match property { Property::Id => Value::Id(id), - Property::Name | Property::Role => values.remove(property), - Property::SortOrder => values - .properties - .remove(property) - .unwrap_or(Value::UnsignedInt(0)), - Property::ParentId => values - .properties - .remove(property) - .map(|parent_id| match parent_id { - Value::Id(value) if value.document_id() > 0 => { - Value::Id((value.document_id() - 1).into()) - } - _ => Value::Null, - }) - .unwrap_or_default(), + Property::Name => { + Value::Text(std::mem::take(&mut values.as_mut().unwrap().name)) + } + Property::Role => { + if let Some(role) = values.as_ref().unwrap().role.as_str() { + Value::Text(role.to_string()) + } else { + Value::Null + } + } + Property::SortOrder => Value::UnsignedInt( + values + .as_ref() + .unwrap() + .sort_order + .unwrap_or_default() + .into(), + ), + Property::ParentId => { + let parent_id = values.as_ref().unwrap().parent_id; + if parent_id > 0 { + Value::Id((parent_id - 1).into()) + } else { + Value::Null + } + } Property::TotalEmails => Value::UnsignedInt( self.get_tag( account_id, @@ -173,7 +189,7 @@ impl MailboxGet for Server { ), Property::MyRights => { if access_token.is_shared(account_id) { - let acl = values.effective_acl(access_token); + let acl = values.as_ref().unwrap().acls.effective_acl(access_token); Object::with_capacity(9) .with_property(Property::MayReadItems, acl.contains(Acl::ReadItems)) .with_property(Property::MayAddItems, acl.contains(Acl::AddItems)) @@ -208,31 +224,21 @@ impl MailboxGet for Server { .into() } } - Property::IsSubscribed => values - .properties - .remove(property) - .map(|parent_id| match parent_id { - Value::List(values) - if values - .contains(&Value::Id(access_token.primary_id().into())) => - { - Value::Bool(true) - } - _ => Value::Bool(false), - }) - .unwrap_or(Value::Bool(false)), + Property::IsSubscribed => { + if values + .as_ref() + .unwrap() + .subscribers + .contains(&access_token.primary_id()) + { + Value::Bool(true) + } else { + Value::Bool(false) + } + } Property::Acl => { - self.acl_get( - values - .properties - .get(&Property::Acl) - .and_then(|v| v.as_acl()) - .map(|v| &v[..]) - .unwrap_or_else(|| &[]), - access_token, - account_id, - ) - .await + self.acl_get(&values.as_ref().unwrap().acls, access_token, account_id) + .await } _ => Value::Null, diff --git a/crates/jmap/src/mailbox/query.rs b/crates/jmap/src/mailbox/query.rs index 4f039e06..22e6645c 100644 --- a/crates/jmap/src/mailbox/query.rs +++ b/crates/jmap/src/mailbox/query.rs @@ -4,12 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{auth::AccessToken, Server}; -use email::mailbox::MailboxFnc; +use common::{Server, auth::AccessToken}; +use email::mailbox::{Mailbox, manage::MailboxFnc}; use jmap_proto::{ method::query::{Comparator, Filter, QueryRequest, QueryResponse, SortProperty}, - object::{mailbox::QueryArguments, Object}, - types::{acl::Acl, collection::Collection, property::Property, value::Value}, + object::mailbox::QueryArguments, + types::{acl::Acl, collection::Collection, property::Property}, }; use store::{ ahash::{AHashMap, AHashSet}, @@ -17,7 +17,7 @@ use store::{ roaring::RoaringBitmap, }; -use crate::{auth::acl::AclMethods, JmapMethods, UpdateResults}; +use crate::{JmapMethods, UpdateResults, auth::acl::AclMethods}; use std::future::Future; pub trait MailboxQuery: Sync + Send { @@ -93,7 +93,7 @@ impl MailboxQuery for Server { other => { return Err(trc::JmapEvent::UnsupportedFilter .into_err() - .details(other.to_string())) + .details(other.to_string())); } } } @@ -117,7 +117,7 @@ impl MailboxQuery for Server { || (response.total.is_some_and(|total| total > 0) && filter_as_tree)) { for (document_id, value) in self - .get_properties::, _, _>( + .get_properties::( account_id, Collection::Mailbox, &mailbox_ids, @@ -125,13 +125,8 @@ impl MailboxQuery for Server { ) .await? { - let parent_id = value - .properties - .get(&Property::ParentId) - .and_then(|id| id.as_id().map(|id| id.document_id())) - .unwrap_or(0); - hierarchy.insert(document_id + 1, parent_id); - tree.entry(parent_id) + hierarchy.insert(document_id + 1, value.parent_id); + tree.entry(value.parent_id) .or_insert_with(AHashSet::default) .insert(document_id + 1); } @@ -199,7 +194,7 @@ impl MailboxQuery for Server { other => { return Err(trc::JmapEvent::UnsupportedSort .into_err() - .details(other.to_string())) + .details(other.to_string())); } }); } diff --git a/crates/jmap/src/mailbox/set.rs b/crates/jmap/src/mailbox/set.rs index bdd45b7b..ba3bb73b 100644 --- a/crates/jmap/src/mailbox/set.rs +++ b/crates/jmap/src/mailbox/set.rs @@ -4,13 +4,16 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{auth::AccessToken, Server}; +use common::{Server, auth::AccessToken, config::jmap::settings::SpecialUse}; use directory::Permission; -use email::mailbox::{MailboxFnc, SCHEMA}; +use email::{ + mailbox::{Mailbox, manage::MailboxFnc}, + message::delete::EmailDeletion, +}; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{SetRequest, SetResponse}, - object::{index::ObjectIndexBuilder, mailbox::SetArguments, Object}, + object::{index::ObjectIndexBuilder, mailbox::SetArguments}, response::references::EvalObjectReferences, types::{ acl::Acl, @@ -19,27 +22,27 @@ use jmap_proto::{ property::Property, state::StateChange, type_state::DataType, - value::{MaybePatchValue, SetValue, Value}, + value::{MaybePatchValue, Object, SetValue, Value}, }, }; use store::{ query::Filter, roaring::RoaringBitmap, write::{ + BatchBuilder, F_BITMAP, F_CLEAR, F_VALUE, assert::{AssertValue, HashedValue}, log::ChangeLogBuilder, - BatchBuilder, F_BITMAP, F_CLEAR, F_VALUE, }, }; +use utils::config::utils::ParseValue; use crate::{ - auth::acl::{AclMethods, EffectiveAcl}, - email::delete::EmailDeletion, JmapMethods, + auth::acl::{AclMethods, EffectiveAcl}, }; #[allow(unused_imports)] -use email::mailbox::{UidMailbox, INBOX_ID, JUNK_ID, TRASH_ID}; +use email::mailbox::{INBOX_ID, JUNK_ID, TRASH_ID, UidMailbox}; use std::future::Future; pub struct SetContext<'x> { @@ -70,9 +73,9 @@ pub trait MailboxSet: Sync + Send { fn mailbox_set_item( &self, changes_: Object, - update: Option<(u32, HashedValue>)>, + update: Option<(u32, HashedValue)>, ctx: &SetContext, - ) -> impl Future>> + Send; + ) -> impl Future, SetError>>> + Send; } impl MailboxSet for Server { @@ -106,15 +109,11 @@ impl MailboxSet for Server { .with_account_id(account_id) .with_collection(Collection::Mailbox); - if let Value::Id(parent_id) = - builder.changes().unwrap().get(&Property::ParentId) - { - let parent_id = parent_id.document_id(); - if parent_id > 0 { - batch - .update_document(parent_id - 1) - .assert_value(Property::Value, AssertValue::Some); - } + let parent_id = builder.changes().unwrap().parent_id; + if parent_id > 0 { + batch + .update_document(parent_id - 1) + .assert_value(Property::Value, AssertValue::Some); } batch.create_document().custom(builder); @@ -166,7 +165,7 @@ impl MailboxSet for Server { // Obtain mailbox let document_id = id.document_id(); if let Some(mailbox) = self - .get_property::>>( + .get_property::>( account_id, Collection::Mailbox, document_id, @@ -176,7 +175,7 @@ impl MailboxSet for Server { { // Validate ACL if ctx.is_shared { - let acl = mailbox.inner.effective_acl(access_token); + let acl = mailbox.inner.acls.effective_acl(access_token); if !acl.contains(Acl::Modify) { ctx.response.not_updated.append( id, @@ -184,7 +183,7 @@ impl MailboxSet for Server { .with_description("You are not allowed to modify this mailbox."), ); continue 'update; - } else if object.properties.contains_key(&Property::Acl) + } else if object.0.contains_key(&Property::Acl) && !acl.contains(Acl::Administer) { ctx.response.not_updated.append( @@ -207,15 +206,11 @@ impl MailboxSet for Server { .with_account_id(account_id) .with_collection(Collection::Mailbox); - if let Value::Id(parent_id) = - builder.changes().unwrap().get(&Property::ParentId) - { - let parent_id = parent_id.document_id(); - if parent_id > 0 { - batch - .update_document(parent_id - 1) - .assert_value(Property::Value, AssertValue::Some); - } + let parent_id = builder.changes().unwrap().parent_id; + if parent_id > 0 { + batch + .update_document(parent_id - 1) + .assert_value(Property::Value, AssertValue::Some); } batch.update_document(document_id).custom(builder); @@ -432,7 +427,7 @@ impl MailboxSet for Server { // Obtain mailbox if let Some(mailbox) = self - .get_property::>>( + .get_property::>( account_id, Collection::Mailbox, document_id, @@ -442,7 +437,7 @@ impl MailboxSet for Server { { // Validate ACLs if access_token.is_shared(account_id) { - let acl = mailbox.inner.effective_acl(access_token); + let acl = mailbox.inner.acls.effective_acl(access_token); if !acl.contains(Acl::Administer) { if !acl.contains(Acl::Delete) { return Ok(Err(SetError::forbidden() @@ -461,7 +456,7 @@ impl MailboxSet for Server { .with_collection(Collection::Mailbox) .delete_document(document_id) .value(Property::EmailIds, (), F_VALUE | F_CLEAR) - .custom(ObjectIndexBuilder::new(SCHEMA).with_current(mailbox)); + .custom(ObjectIndexBuilder::new().with_current(mailbox)); match self.core.storage.data.write(batch.build()).await { Ok(_) => { @@ -484,23 +479,27 @@ impl MailboxSet for Server { async fn mailbox_set_item( &self, changes_: Object, - update: Option<(u32, HashedValue>)>, + update: Option<(u32, HashedValue)>, ctx: &SetContext<'_>, - ) -> trc::Result> { + ) -> trc::Result, SetError>> { // Parse properties - let mut changes = Object::with_capacity(changes_.properties.len()); - for (property, value) in changes_.properties { + let mut changes = update + .as_ref() + .map(|(_, obj)| obj.inner.clone()) + .unwrap_or_else(|| Mailbox::new(String::new())); + let mut has_acl_changes = false; + for (property, value) in changes_.0 { let value = match ctx.response.eval_object_references(value) { Ok(value) => value, Err(err) => { return Ok(Err(err)); } }; - let value = match (&property, value) { + match (&property, value) { (Property::Name, MaybePatchValue::Value(Value::Text(value))) => { let value = value.trim(); if !value.is_empty() && value.len() < self.core.jmap.mailbox_name_max_len { - Value::Text(value.to_string()) + changes.name = value.to_string(); } else { return Ok(Err(SetError::invalid_properties() .with_property(Property::Name) @@ -523,47 +522,45 @@ impl MailboxSet for Server { return Ok(Err(SetError::invalid_properties() .with_description("Parent ID does not exist."))); } - - Value::Id((parent_id + 1).into()) + changes.parent_id = parent_id + 1; + } + (Property::ParentId, MaybePatchValue::Value(Value::Null)) => { + changes.parent_id = 0; } - (Property::ParentId, MaybePatchValue::Value(Value::Null)) => Value::Id(0u64.into()), (Property::IsSubscribed, MaybePatchValue::Value(Value::Bool(subscribe))) => { - if let Some((_, current_fields)) = update.as_ref() { - if let Some(value) = current_fields - .inner - .mailbox_subscribe(ctx.access_token.primary_id(), subscribe) - { - value - } else { - continue; + let account_id = ctx.access_token.primary_id(); + if subscribe { + if !changes.subscribers.contains(&account_id) { + changes.subscribers.push(account_id); } - } else if subscribe { - Value::List(vec![Value::Id(ctx.access_token.primary_id().into())]) } else { - continue; + changes.subscribers.retain(|id| *id != account_id); } } (Property::Role, MaybePatchValue::Value(Value::Text(value))) => { - let role = value.trim().to_lowercase(); - if [ - "inbox", "trash", "spam", "junk", "drafts", "archive", "sent", - ] - .contains(&role.as_str()) - { - Value::Text(role) + let role = value.trim(); + if let Ok(role) = SpecialUse::parse_value(role) { + changes.role = role; } else { return Ok(Err(SetError::invalid_properties() .with_property(Property::Role) .with_description(format!("Invalid role {role:?}.")))); } } - (Property::Role, MaybePatchValue::Value(Value::Null)) => Value::Null, + (Property::Role, MaybePatchValue::Value(Value::Null)) => { + changes.role = SpecialUse::None; + } (Property::SortOrder, MaybePatchValue::Value(Value::UnsignedInt(value))) => { - Value::UnsignedInt(value) + changes.sort_order = Some(value as u32); } (Property::Acl, value) => { + has_acl_changes = true; match self - .acl_set(&mut changes, update.as_ref().map(|(_, obj)| obj), value) + .acl_set( + &mut changes.acls, + update.as_ref().map(|(_, obj)| obj.inner.acls.as_slice()), + value, + ) .await { Ok(_) => continue, @@ -576,234 +573,157 @@ impl MailboxSet for Server { _ => { return Ok(Err(SetError::invalid_properties() .with_property(property) - .with_description("Invalid property or value.".to_string()))) + .with_description("Invalid property or value.".to_string()))); } - }; - - changes.append(property, value); + } } // Validate depth and circular parent-child relationship - if let Value::Id(mailbox_parent_id) = changes.get(&Property::ParentId) { - let current_mailbox_id = update - .as_ref() - .map_or(u32::MAX, |(mailbox_id, _)| *mailbox_id + 1); - let mut mailbox_parent_id = mailbox_parent_id.document_id(); - let mut success = false; - for depth in 0..self.core.jmap.mailbox_max_depth { - if mailbox_parent_id == current_mailbox_id { - return Ok(Err(SetError::invalid_properties() - .with_property(Property::ParentId) - .with_description("Mailbox cannot be a parent of itself."))); - } else if mailbox_parent_id == 0 { - if depth == 0 && ctx.is_shared { - return Ok(Err(SetError::forbidden() - .with_description("You are not allowed to create root folders."))); - } - success = true; - break; - } - let parent_document_id = mailbox_parent_id - 1; - - if let Some(mut fields) = self - .get_property::>( - ctx.account_id, - Collection::Mailbox, - parent_document_id, - Property::Value, - ) - .await? - { - if depth == 0 - && ctx.is_shared - && !fields - .effective_acl(ctx.access_token) - .contains_any([Acl::CreateChild, Acl::Administer].into_iter()) - { - return Ok(Err(SetError::forbidden().with_description( - "You are not allowed to create sub mailboxes under this mailbox.", - ))); - } - - mailbox_parent_id = fields - .properties - .remove(&Property::ParentId) - .and_then(|v| v.try_unwrap_id().map(|id| id.document_id())) - .unwrap_or(0); - } else if ctx.mailbox_ids.contains(parent_document_id) { - // Parent mailbox is probably created within the same request - success = true; - break; - } else { - return Ok(Err(SetError::invalid_properties() - .with_property(Property::ParentId) - .with_description("Mailbox parent does not exist."))); - } - } - - if !success { + let mut mailbox_parent_id = changes.parent_id; + let current_mailbox_id = update + .as_ref() + .map_or(u32::MAX, |(mailbox_id, _)| *mailbox_id + 1); + let mut success = false; + for depth in 0..self.core.jmap.mailbox_max_depth { + if mailbox_parent_id == current_mailbox_id { return Ok(Err(SetError::invalid_properties() .with_property(Property::ParentId) - .with_description( - "Mailbox parent-child relationship is too deep.", - ))); + .with_description("Mailbox cannot be a parent of itself."))); + } else if mailbox_parent_id == 0 { + if depth == 0 && ctx.is_shared { + return Ok(Err(SetError::forbidden() + .with_description("You are not allowed to create root folders."))); + } + success = true; + break; + } + let parent_document_id = mailbox_parent_id - 1; + + if let Some(fields) = self + .get_property::( + ctx.account_id, + Collection::Mailbox, + parent_document_id, + Property::Value, + ) + .await? + { + if depth == 0 + && ctx.is_shared + && !fields + .acls + .effective_acl(ctx.access_token) + .contains_any([Acl::CreateChild, Acl::Administer].into_iter()) + { + return Ok(Err(SetError::forbidden().with_description( + "You are not allowed to create sub mailboxes under this mailbox.", + ))); + } + + mailbox_parent_id = fields.parent_id; + } else if ctx.mailbox_ids.contains(parent_document_id) { + // Parent mailbox is probably created within the same request + success = true; + break; + } else { + return Ok(Err(SetError::invalid_properties() + .with_property(Property::ParentId) + .with_description("Mailbox parent does not exist."))); } - } else if update.is_none() { - // Set parentId if the field is missing - changes.append(Property::ParentId, Value::Id(0u64.into())); } - // Generate IMAP UID validity - if update.is_none() { - changes.append( - Property::Cid, - Value::UnsignedInt(rand::random::() as u64), - ); + if !success { + return Ok(Err(SetError::invalid_properties() + .with_property(Property::ParentId) + .with_description( + "Mailbox parent-child relationship is too deep.", + ))); } // Verify that the mailbox role is unique. - if let Value::Text(mailbox_role) = changes.get(&Property::Role) { - if update + if !matches!(changes.role, SpecialUse::None) + && update .as_ref() - .map(|(_, update)| update.inner.get(&Property::Role)) - .and_then(|v| v.as_string()) - .unwrap_or_default() - != mailbox_role + .is_none_or(|(_, m)| m.inner.role != changes.role) + { + if !self + .filter( + ctx.account_id, + Collection::Mailbox, + vec![Filter::eq( + Property::Role, + changes.role.as_str().unwrap_or_default(), + )], + ) + .await? + .results + .is_empty() { - if !self - .filter( - ctx.account_id, - Collection::Mailbox, - vec![Filter::eq(Property::Role, mailbox_role.as_str())], - ) - .await? - .results - .is_empty() - { - return Ok(Err(SetError::invalid_properties() - .with_property(Property::Role) - .with_description(format!( - "A mailbox with role '{}' already exists.", - mailbox_role - )))); - } + return Ok(Err(SetError::invalid_properties() + .with_property(Property::Role) + .with_description(format!( + "A mailbox with role '{}' already exists.", + changes.role.as_str().unwrap_or_default() + )))); + } - // Role of internal folders cannot be modified - if update.as_ref().is_some_and(|(document_id, _)| { - *document_id == INBOX_ID || *document_id == TRASH_ID - }) { - return Ok(Err(SetError::invalid_properties() - .with_property(Property::Role) - .with_description( - "You are not allowed to change the role of Inbox or Trash folders.", - ))); - } + // Role of internal folders cannot be modified + if update.as_ref().is_some_and(|(document_id, _)| { + *document_id == INBOX_ID || *document_id == TRASH_ID + }) { + return Ok(Err(SetError::invalid_properties() + .with_property(Property::Role) + .with_description( + "You are not allowed to change the role of Inbox or Trash folders.", + ))); } } // Verify that the mailbox name is unique. - if let Value::Text(mailbox_name) = changes.get(&Property::Name) { + if !changes.name.is_empty() { // Obtain parent mailbox id - if let Some(parent_mailbox_id) = if let Some(mailbox_parent_id) = &changes - .properties - .get(&Property::ParentId) - .and_then(|id| id.as_id().map(|id| id.document_id())) - { - (*mailbox_parent_id).into() - } else if let Some((_, current_fields)) = &update { - if current_fields - .inner - .properties - .get(&Property::Name) - .and_then(|n| n.as_string()) - != Some(mailbox_name) - { - current_fields - .inner - .properties - .get(&Property::ParentId) - .and_then(|id| id.as_id().map(|id| id.document_id())) - .unwrap_or_default() - .into() - } else { - None - } - } else { - 0.into() - } { - if !self + if update + .as_ref() + .is_none_or(|(_, m)| m.inner.name != changes.name) + && !self .filter( ctx.account_id, Collection::Mailbox, vec![ - Filter::eq(Property::Name, mailbox_name.as_str()), - Filter::eq(Property::ParentId, parent_mailbox_id), + Filter::eq(Property::Name, changes.name.as_str()), + Filter::eq(Property::ParentId, changes.parent_id), ], ) .await? .results .is_empty() - { - return Ok(Err(SetError::invalid_properties() - .with_property(Property::Name) - .with_description(format!( - "A mailbox with name '{}' already exists.", - mailbox_name - )))); - } + { + return Ok(Err(SetError::invalid_properties() + .with_property(Property::Name) + .with_description(format!( + "A mailbox with name '{}' already exists.", + changes.name + )))); } + } else { + return Ok(Err(SetError::invalid_properties() + .with_property(Property::Name) + .with_description("Mailbox name cannot be empty."))); } // Refresh ACLs let current = update.map(|(_, current)| current); - if changes.properties.contains_key(&Property::Acl) { - self.refresh_acls(&changes, ¤t).await; + if has_acl_changes { + self.refresh_acls( + &changes.acls, + current.as_ref().map(|m| m.inner.acls.as_slice()), + ) + .await; } // Validate - Ok(ObjectIndexBuilder::new(SCHEMA) + Ok(Ok(ObjectIndexBuilder::new() .with_changes(changes) - .with_current_opt(current) - .validate()) - } -} - -pub trait MailboxSubscribe { - fn mailbox_subscribe(&self, account_id: u32, subscribed: bool) -> Option; -} - -impl MailboxSubscribe for Object { - fn mailbox_subscribe(&self, account_id: u32, subscribe: bool) -> Option { - let account_id = Value::Id(account_id.into()); - if let Value::List(subscriptions) = self.get(&Property::IsSubscribed) { - if subscribe { - if !subscriptions.contains(&account_id) { - let mut current_subscriptions = subscriptions.clone(); - current_subscriptions.push(account_id); - Value::List(current_subscriptions).into() - } else { - None - } - } else if subscriptions.contains(&account_id) { - if subscriptions.len() > 1 { - Value::List( - subscriptions - .iter() - .filter(|id| *id != &account_id) - .cloned() - .collect(), - ) - .into() - } else { - Value::Null.into() - } - } else { - None - } - } else if subscribe { - Value::List(vec![account_id]).into() - } else { - None - } + .with_current_opt(current))) } } diff --git a/crates/jmap/src/principal/get.rs b/crates/jmap/src/principal/get.rs index 0950ad58..3b5f9957 100644 --- a/crates/jmap/src/principal/get.rs +++ b/crates/jmap/src/principal/get.rs @@ -5,11 +5,15 @@ */ use common::Server; -use directory::{backend::internal::PrincipalField, QueryBy}; +use directory::{QueryBy, backend::internal::PrincipalField}; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, - object::Object, - types::{collection::Collection, property::Property, state::State, value::Value}, + types::{ + collection::Collection, + property::Property, + state::State, + value::{Object, Value}, + }, }; use std::future::Future; diff --git a/crates/jmap/src/push/get.rs b/crates/jmap/src/push/get.rs index 8ec88ede..76b6aa69 100644 --- a/crates/jmap/src/push/get.rs +++ b/crates/jmap/src/push/get.rs @@ -4,22 +4,24 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use base64::{engine::general_purpose, Engine}; use common::{ + Server, auth::AccessToken, ipc::{StateEvent, UpdateSubscription}, - Server, }; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, - object::Object, - types::{collection::Collection, property::Property, type_state::DataType, value::Value}, + types::{ + collection::Collection, + date::UTCDate, + property::Property, + value::{Object, Value}, + }, }; use store::{ - write::{now, ValueClass}, BitmapKey, ValueKey, + write::{ValueClass, now}, }; -use utils::map::bitmap::Bitmap; use super::{EncryptionKeys, PushSubscription}; use std::future::Future; @@ -80,7 +82,7 @@ impl PushSubscriptionFetch for Server { continue; } let mut push = if let Some(push) = self - .get_property::>( + .get_property::( account_id, Collection::PushSubscription, document_id, @@ -104,8 +106,31 @@ impl PushSubscriptionFetch for Server { "The 'url' and 'keys' properties are not readable".to_string(), )); } + Property::DeviceClientId => { + result.append( + Property::DeviceClientId, + std::mem::take(&mut push.device_client_id), + ); + } + Property::Types => { + let mut types = Vec::new(); + for typ in push.types.into_iter() { + types.push(Value::Text(typ.to_string())); + } + result.append(Property::Types, Value::List(types)); + } + Property::Expires => { + if push.expires > 0 { + result.append( + Property::Expires, + Value::Date(UTCDate::from_timestamp(push.expires as i64)), + ); + } else { + result.append(Property::Expires, Value::Null); + } + } property => { - result.append(property.clone(), push.remove(property)); + result.append(property.clone(), Value::Null); } } } @@ -131,11 +156,11 @@ impl PushSubscriptionFetch for Server { let current_time = now(); for document_id in document_ids { - let mut subscription = self + let subscription = self .core .storage .data - .get_value::>(ValueKey { + .get_value::(ValueKey { account_id, collection: Collection::PushSubscription.into(), document_id, @@ -149,103 +174,29 @@ impl PushSubscriptionFetch for Server { .document_id(document_id) })?; - let expires = subscription - .properties - .get(&Property::Expires) - .and_then(|p| p.as_date()) - .ok_or_else(|| { - trc::StoreEvent::UnexpectedError - .caused_by(trc::location!()) - .document_id(document_id) - })? - .timestamp() as u64; - if expires > current_time { - let keys = if let Some((auth, p256dh)) = subscription - .properties - .remove(&Property::Keys) - .and_then(|value| value.try_unwrap_object()) - .and_then(|mut obj| { - ( - obj.properties - .remove(&Property::Auth) - .and_then(|value| value.try_unwrap_string())?, - obj.properties - .remove(&Property::P256dh) - .and_then(|value| value.try_unwrap_string())?, - ) - .into() - }) { - EncryptionKeys { - p256dh: general_purpose::URL_SAFE - .decode(&p256dh) - .unwrap_or_default(), - auth: general_purpose::URL_SAFE.decode(&auth).unwrap_or_default(), - } - .into() - } else { - None - }; - let verification_code = subscription - .properties - .remove(&Property::Value) - .and_then(|p| p.try_unwrap_string()) - .ok_or_else(|| { - trc::StoreEvent::UnexpectedError - .caused_by(trc::location!()) - .document_id(document_id) - })?; - let url = subscription - .properties - .remove(&Property::Url) - .and_then(|p| p.try_unwrap_string()) - .ok_or_else(|| { - trc::StoreEvent::UnexpectedError - .caused_by(trc::location!()) - .document_id(document_id) - })?; - - if subscription - .properties - .get(&Property::VerificationCode) - .and_then(|p| p.as_string()) - .is_some_and(|v| v == verification_code) - { - let types = if let Some(Value::List(value)) = - subscription.properties.remove(&Property::Types) - { - if !value.is_empty() { - let mut type_states = Bitmap::new(); - for type_state in value { - if let Some(type_state) = type_state - .as_string() - .and_then(|type_state| DataType::try_from(type_state).ok()) - { - type_states.insert(type_state); - } - } - type_states - } else { - Bitmap::all() - } - } else { - Bitmap::all() - }; - + if subscription.expires > current_time { + if subscription.verified { // Add verified subscription subscriptions.push(UpdateSubscription::Verified(PushSubscription { id: document_id, - url, - expires, - types, - keys, + url: subscription.url, + expires: subscription.expires, + types: subscription.types, + keys: subscription.keys.map(|keys| EncryptionKeys { + p256dh: keys.p256dh, + auth: keys.auth, + }), })); } else { // Add unverified subscription subscriptions.push(UpdateSubscription::Unverified { id: document_id, - url, - code: verification_code, - keys, + url: subscription.url, + code: subscription.verification_code, + keys: subscription.keys.map(|keys| EncryptionKeys { + p256dh: keys.p256dh, + auth: keys.auth, + }), }); } } diff --git a/crates/jmap/src/push/set.rs b/crates/jmap/src/push/set.rs index 29427de3..2358bb3b 100644 --- a/crates/jmap/src/push/set.rs +++ b/crates/jmap/src/push/set.rs @@ -6,26 +6,28 @@ use base64::{Engine, engine::general_purpose}; use common::{Server, auth::AccessToken}; +use email::push::{Keys, PushSubscription}; use jmap_proto::{ error::set::SetError, method::set::{RequestArguments, SetRequest, SetResponse}, - object::Object, response::references::EvalObjectReferences, types::{ collection::Collection, date::UTCDate, property::Property, type_state::DataType, - value::{MaybePatchValue, Value}, + value::{MaybePatchValue, Object, Value}, }, }; use rand::distr::Alphanumeric; use std::future::Future; use store::{ + Serialize, rand::{Rng, rng}, write::{BatchBuilder, F_CLEAR, F_VALUE, now}, }; use trc::AddContext; +use utils::map::bitmap::Bitmap; use crate::services::state::StateManager; @@ -56,7 +58,7 @@ impl PushSubscriptionSet for Server { // Process creates 'create: for (id, object) in request.unwrap_create() { - let mut push = Object::with_capacity(object.properties.len()); + let mut push = PushSubscription::default(); if push_ids.len() as usize >= self.core.jmap.push_max_total { response.not_created.append(id, SetError::forbidden().with_description( @@ -65,25 +67,17 @@ impl PushSubscriptionSet for Server { continue 'create; } - for (property, value) in object.properties { - match response + for (property, value) in object.0 { + if let Err(err) = response .eval_object_references(value) - .and_then(|value| validate_push_value(&property, value, None)) + .and_then(|value| validate_push_value(&property, value, &mut push, true)) { - Ok(Value::Null) => (), - Ok(value) => { - push.set(property, value); - } - Err(err) => { - response.not_created.append(id, err); - continue 'create; - } + response.not_created.append(id, err); + continue 'create; } } - if !push.properties.contains_key(&Property::DeviceClientId) - || !push.properties.contains_key(&Property::Url) - { + if push.device_client_id.is_empty() || push.url.is_empty() { response.not_created.append( id, SetError::invalid_properties() @@ -94,25 +88,17 @@ impl PushSubscriptionSet for Server { } // Add expiry time if missing - let expires = if let Some(expires) = push.properties.get(&Property::Expires) { - expires.clone() - } else { - let expires = Value::Date(UTCDate::from_timestamp(now() as i64 + EXPIRES_MAX)); - push.append(Property::Expires, expires.clone()); - expires - }; + if push.expires == 0 { + push.expires = now() + EXPIRES_MAX as u64; + } + let expires = UTCDate::from_timestamp(push.expires as i64); // Generate random verification code - push.append( - Property::Value, - Value::Text( - rng() - .sample_iter(Alphanumeric) - .take(VERIFICATION_CODE_LEN) - .map(char::from) - .collect::(), - ), - ); + push.verification_code = rng() + .sample_iter(Alphanumeric) + .take(VERIFICATION_CODE_LEN) + .map(char::from) + .collect::(); // Insert record let mut batch = BatchBuilder::new(); @@ -120,7 +106,7 @@ impl PushSubscriptionSet for Server { .with_account_id(account_id) .with_collection(Collection::PushSubscription) .create_document() - .value(Property::Value, push, F_VALUE); + .set(Property::Value, push.serialize()); let document_id = self .store() .write_expect_id(batch) @@ -147,7 +133,7 @@ impl PushSubscriptionSet for Server { // Obtain push subscription let document_id = id.document_id(); let mut push = if let Some(push) = self - .get_property::>( + .get_property::( account_id, Collection::PushSubscription, document_id, @@ -161,22 +147,14 @@ impl PushSubscriptionSet for Server { continue 'update; }; - for (property, value) in object.properties { - match response + for (property, value) in object.0 { + if let Err(err) = response .eval_object_references(value) - .and_then(|value| validate_push_value(&property, value, Some(&push))) + .and_then(|value| validate_push_value(&property, value, &mut push, true)) { - Ok(Value::Null) => { - push.remove(&property); - } - Ok(value) => { - push.set(property, value); - } - Err(err) => { - response.not_updated.append(id, err); - continue 'update; - } - }; + response.not_updated.append(id, err); + continue 'update; + } } // Update record @@ -185,7 +163,7 @@ impl PushSubscriptionSet for Server { .with_account_id(account_id) .with_collection(Collection::PushSubscription) .update_document(document_id) - .value(Property::Value, push, F_VALUE); + .set(Property::Value, push.serialize()); self.store() .write(batch) .await @@ -226,78 +204,90 @@ impl PushSubscriptionSet for Server { fn validate_push_value( property: &Property, value: MaybePatchValue, - current: Option<&Object>, -) -> Result { - Ok(match (property, value) { + push: &mut PushSubscription, + is_create: bool, +) -> Result<(), SetError> { + match (property, value) { (Property::DeviceClientId, MaybePatchValue::Value(Value::Text(value))) - if current.is_none() && value.len() < 255 => + if is_create && value.len() < 255 => { - Value::Text(value) + push.device_client_id = value; } (Property::Url, MaybePatchValue::Value(Value::Text(value))) - if current.is_none() && value.len() < 512 && value.starts_with("https://") => + if is_create && value.len() < 512 && value.starts_with("https://") => { - Value::Text(value) + push.url = value; } (Property::Keys, MaybePatchValue::Value(Value::Object(value))) - if current.is_none() - && value.properties.len() == 2 - && matches!(value.get(&Property::Auth), Value::Text(auth) if auth.len() < 1024 && - general_purpose::URL_SAFE.decode(auth).is_ok()) - && matches!(value.get(&Property::P256dh), Value::Text(p256dh) if p256dh.len() < 1024 && - general_purpose::URL_SAFE.decode(p256dh).is_ok()) => + if is_create && value.0.len() == 2 => { - Value::Object(value) + if let (Some(auth), Some(p256dh)) = ( + value + .get(&Property::Auth) + .as_string() + .and_then(|v| general_purpose::URL_SAFE.decode(v).ok()), + value + .get(&Property::P256dh) + .as_string() + .and_then(|v| general_purpose::URL_SAFE.decode(v).ok()), + ) { + push.keys = Some(Keys { auth, p256dh }); + } else { + return Err(SetError::invalid_properties() + .with_property(property.clone()) + .with_description("Failed to decode keys.")); + } } (Property::Expires, MaybePatchValue::Value(Value::Date(value))) => { let current_time = now() as i64; let expires = value.timestamp(); - Value::Date(UTCDate::from_timestamp( - if expires > current_time && (expires - current_time) > EXPIRES_MAX { - current_time + EXPIRES_MAX - } else { - expires - }, - )) + push.expires = if expires > current_time && (expires - current_time) > EXPIRES_MAX { + current_time + EXPIRES_MAX + } else { + expires + } as u64; } (Property::Expires, MaybePatchValue::Value(Value::Null)) => { - Value::Date(UTCDate::from_timestamp(now() as i64 + EXPIRES_MAX)) + push.expires = now() + EXPIRES_MAX as u64; } - (Property::Types, MaybePatchValue::Value(Value::List(value))) - if value.iter().all(|value| { - value + (Property::Types, MaybePatchValue::Value(Value::List(value))) => { + push.types.clear(); + + for item in value { + if let Some(dt) = item .as_string() .and_then(|value| DataType::try_from(value).ok()) - .is_some() - }) => - { - Value::List(value) + { + push.types.insert(dt); + } else { + return Err(SetError::invalid_properties() + .with_property(property.clone()) + .with_description("Invalid data type.")); + } + } } - (Property::VerificationCode, MaybePatchValue::Value(Value::Text(value))) - if current.is_some() => - { - if current - .as_ref() - .unwrap() - .properties - .get(&Property::Value) - .is_some_and(|v| matches!(v, Value::Text(v) if v == &value)) - { - Value::Text(value) + (Property::VerificationCode, MaybePatchValue::Value(Value::Text(value))) if !is_create => { + if push.verification_code == value { + push.verified = true; } else { return Err(SetError::invalid_properties() .with_property(property.clone()) .with_description("Verification code does not match.".to_string())); } } - ( - Property::Keys | Property::Types | Property::VerificationCode, - MaybePatchValue::Value(Value::Null), - ) => Value::Null, + (Property::Keys, MaybePatchValue::Value(Value::Null)) => { + push.keys = None; + } + (Property::Types, MaybePatchValue::Value(Value::Null)) => { + push.types = Bitmap::all(); + } + (Property::VerificationCode, MaybePatchValue::Value(Value::Null)) => {} (property, _) => { return Err(SetError::invalid_properties() .with_property(property.clone()) .with_description("Field could not be set.")); } - }) + } + + Ok(()) } diff --git a/crates/jmap/src/quota/get.rs b/crates/jmap/src/quota/get.rs index 33f3d367..6823c5a8 100644 --- a/crates/jmap/src/quota/get.rs +++ b/crates/jmap/src/quota/get.rs @@ -4,11 +4,16 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{auth::AccessToken, Server}; +use common::{Server, auth::AccessToken}; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, - object::Object, - types::{id::Id, property::Property, state::State, type_state::DataType, value::Value}, + types::{ + id::Id, + property::Property, + state::State, + type_state::DataType, + value::{Object, Value}, + }, }; use std::future::Future; diff --git a/crates/jmap/src/services/housekeeper.rs b/crates/jmap/src/services/housekeeper.rs index 590baac0..4d92faad 100644 --- a/crates/jmap/src/services/housekeeper.rs +++ b/crates/jmap/src/services/housekeeper.rs @@ -12,10 +12,10 @@ use std::{ }; use common::{ + Inner, KV_LOCK_HOUSEKEEPER, Server, config::telemetry::OtelMetrics, core::BuildServer, ipc::{HousekeeperEvent, PurgeType}, - Inner, Server, KV_LOCK_HOUSEKEEPER, }; #[cfg(feature = "enterprise")] @@ -24,12 +24,13 @@ use common::telemetry::{ tracers::store::TracingStore, }; +use email::message::delete::EmailDeletion; use smtp::reporting::SmtpReporting; -use store::{write::now, PurgeStore}; +use store::{PurgeStore, write::now}; use tokio::sync::mpsc; use trc::{Collector, MetricType, PurgeEvent}; -use crate::{email::delete::EmailDeletion, JmapMethods, LONG_SLUMBER}; +use crate::{JmapMethods, LONG_SLUMBER}; #[derive(PartialEq, Eq)] struct Action { @@ -154,96 +155,99 @@ pub fn spawn_housekeeper(inner: Arc, mut rx: mpsc::Receiver match event { - HousekeeperEvent::ReloadSettings => { - let server = inner.build_server(); + Ok(Some(event)) => { + match event { + HousekeeperEvent::ReloadSettings => { + let server = inner.build_server(); - // Reload OTEL push metrics - match &server.core.metrics.otel { - Some(otel) if !queue.has_action(&ActionClass::OtelMetrics) => { - OtelMetrics::enable_errors(); + // Reload OTEL push metrics + match &server.core.metrics.otel { + Some(otel) if !queue.has_action(&ActionClass::OtelMetrics) => { + OtelMetrics::enable_errors(); - queue.schedule( - Instant::now() + otel.interval, - ActionClass::OtelMetrics, - ); - } - _ => {} - } - - // SPDX-SnippetBegin - // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd - // SPDX-License-Identifier: LicenseRef-SEL - #[cfg(feature = "enterprise")] - if let Some(enterprise) = &server.core.enterprise { - if !queue.has_action(&ActionClass::RenewLicense) { - queue.schedule( - Instant::now() + enterprise.license.renew_in(), - ActionClass::RenewLicense, - ); - } - - if let Some(metrics_store) = enterprise.metrics_store.as_ref() { - if !queue.has_action(&ActionClass::InternalMetrics) { queue.schedule( - Instant::now() + metrics_store.interval.time_to_next(), - ActionClass::InternalMetrics, + Instant::now() + otel.interval, + ActionClass::OtelMetrics, ); } + _ => {} } - if !enterprise.metrics_alerts.is_empty() - && !queue.has_action(&ActionClass::AlertMetrics) - { - queue.schedule(Instant::now(), ActionClass::AlertMetrics); + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + if let Some(enterprise) = &server.core.enterprise { + if !queue.has_action(&ActionClass::RenewLicense) { + queue.schedule( + Instant::now() + enterprise.license.renew_in(), + ActionClass::RenewLicense, + ); + } + + if let Some(metrics_store) = enterprise.metrics_store.as_ref() { + if !queue.has_action(&ActionClass::InternalMetrics) { + queue.schedule( + Instant::now() + metrics_store.interval.time_to_next(), + ActionClass::InternalMetrics, + ); + } + } + + if !enterprise.metrics_alerts.is_empty() + && !queue.has_action(&ActionClass::AlertMetrics) + { + queue.schedule(Instant::now(), ActionClass::AlertMetrics); + } } + // SPDX-SnippetEnd + + // Reload ACME certificates + tokio::spawn(async move { + for provider in server.core.acme.providers.values() { + match server.init_acme(provider).await { + Ok(renew_at) => { + server + .inner + .ipc + .housekeeper_tx + .send(HousekeeperEvent::AcmeReschedule { + provider_id: provider.id.clone(), + renew_at: Instant::now() + renew_at, + }) + .await + .ok(); + } + Err(err) => { + trc::error!(err.details( + "Failed to reload ACME certificate manager." + )); + } + }; + } + }); } - // SPDX-SnippetEnd + HousekeeperEvent::AcmeReschedule { + provider_id, + renew_at, + } => { + let action = ActionClass::Acme(provider_id); + queue.remove_action(&action); + queue.schedule(renew_at, action); + } + HousekeeperEvent::Purge(purge) => { + let server = inner.build_server(); + tokio::spawn(async move { + server.purge(purge, 0).await; + }); + } + HousekeeperEvent::Exit => { + trc::event!(Housekeeper(trc::HousekeeperEvent::Stop)); - // Reload ACME certificates - tokio::spawn(async move { - for provider in server.core.acme.providers.values() { - match server.init_acme(provider).await { - Ok(renew_at) => { - server - .inner - .ipc - .housekeeper_tx - .send(HousekeeperEvent::AcmeReschedule { - provider_id: provider.id.clone(), - renew_at: Instant::now() + renew_at, - }) - .await - .ok(); - } - Err(err) => { - trc::error!(err - .details("Failed to reload ACME certificate manager.")); - } - }; - } - }); + return; + } } - HousekeeperEvent::AcmeReschedule { - provider_id, - renew_at, - } => { - let action = ActionClass::Acme(provider_id); - queue.remove_action(&action); - queue.schedule(renew_at, action); - } - HousekeeperEvent::Purge(purge) => { - let server = inner.build_server(); - tokio::spawn(async move { - server.purge(purge, 0).await; - }); - } - HousekeeperEvent::Exit => { - trc::event!(Housekeeper(trc::HousekeeperEvent::Stop)); - - return; - } - }, + } Ok(None) => { trc::event!(Housekeeper(trc::HousekeeperEvent::Stop)); return; @@ -431,8 +435,11 @@ pub fn spawn_housekeeper(inner: Arc, mut rx: mpsc::Receiver { - trc::error!(err - .details("Failed to obtain account count")); + trc::error!( + err.details( + "Failed to obtain account count" + ) + ); } } @@ -444,8 +451,11 @@ pub fn spawn_housekeeper(inner: Arc, mut rx: mpsc::Receiver { - trc::error!(err - .details("Failed to obtain domain count")); + trc::error!( + err.details( + "Failed to obtain domain count" + ) + ); } } } @@ -462,12 +472,14 @@ pub fn spawn_housekeeper(inner: Arc, mut rx: mpsc::Receiver {} Err(err) => { - trc::error!(trc::EventType::Server( - trc::ServerEvent::ThreadError, - ) - .reason(err) - .caused_by(trc::location!()) - .details("Join Error")); + trc::error!( + trc::EventType::Server( + trc::ServerEvent::ThreadError, + ) + .reason(err) + .caused_by(trc::location!()) + .details("Join Error") + ); } } }); @@ -695,9 +707,10 @@ impl Purge for Server { PurgeType::Lookup { store, prefix } => { if let Some(prefix) = prefix { if let Err(err) = store.key_delete_prefix(&prefix).await { - trc::error!(err - .details("Failed to delete key prefix") - .ctx(trc::Key::Key, prefix)); + trc::error!( + err.details("Failed to delete key prefix") + .ctx(trc::Key::Key, prefix) + ); } } else if let Err(err) = store.purge_in_memory_store().await { trc::error!(err.details("Failed to purge in-memory store")); @@ -726,9 +739,10 @@ impl Purge for Server { .remove_lock(KV_LOCK_HOUSEKEEPER, lock_name) .await { - trc::error!(err - .details("Failed to delete task lock.") - .details(lock_type)); + trc::error!( + err.details("Failed to delete task lock.") + .details(lock_type) + ); } } } diff --git a/crates/jmap/src/services/index.rs b/crates/jmap/src/services/index.rs index 07f7889d..c0075b70 100644 --- a/crates/jmap/src/services/index.rs +++ b/crates/jmap/src/services/index.rs @@ -6,29 +6,30 @@ use std::{sync::Arc, time::Instant}; -use common::{core::BuildServer, Inner, Server, KV_LOCK_EMAIL_TASK}; +use common::{Inner, KV_LOCK_EMAIL_TASK, Server, core::BuildServer}; use directory::{ - backend::internal::{manage::ManageDirectory, PrincipalField}, Type, + backend::internal::{PrincipalField, manage::ManageDirectory}, }; -use email::{index::IndexMessageText, metadata::MessageMetadata}; +use email::message::{bayes::EmailBayesTrain, index::IndexMessageText, metadata::MessageMetadata}; use jmap_proto::types::{collection::Collection, property::Property}; use store::{ + IterateParams, Serialize, U32_LEN, U64_LEN, ValueKey, ahash::AHashMap, fts::index::FtsDocument, roaring::RoaringBitmap, write::{ + BatchBuilder, Bincode, BlobOp, MaybeDynamicId, TaskQueueClass, ValueClass, key::{DeserializeBigEndian, KeySerializer}, - now, BatchBuilder, Bincode, BlobOp, MaybeDynamicId, TaskQueueClass, ValueClass, + now, }, - IterateParams, Serialize, ValueKey, U32_LEN, U64_LEN, }; use std::future::Future; use trc::{AddContext, TaskQueueEvent}; -use utils::{BlobHash, BLOB_HASH_LEN}; +use utils::{BLOB_HASH_LEN, BlobHash}; -use crate::{blob::download::BlobDownload, email::bayes::EmailBayesTrain}; +use crate::blob::download::BlobDownload; #[derive(Debug, Clone)] pub struct EmailTask { @@ -113,7 +114,7 @@ impl Indexer for Server { let entry = EmailTask::deserialize(key)?; if locked_seq_ids .get(&entry.seq) - .is_none_or( |expires| now >= *expires) + .is_none_or(|expires| now >= *expires) { entries.push(entry); } @@ -123,9 +124,10 @@ impl Indexer for Server { ) .await .map_err(|err| { - trc::error!(err - .caused_by(trc::location!()) - .details("Failed to iterate over index emails")); + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to iterate over index emails") + ); }); // Add entries to the index @@ -184,10 +186,11 @@ impl Indexer for Server { .with_document_id(event.document_id) .index_message(&message); if let Err(err) = self.core.storage.fts.index(document).await { - trc::error!(err - .account_id(event.account_id) - .document_id(event.document_id) - .details("Failed to index email in FTS index")); + trc::error!( + err.account_id(event.account_id) + .document_id(event.document_id) + .details("Failed to index email in FTS index") + ); continue; } @@ -217,11 +220,12 @@ impl Indexer for Server { } Err(err) => { - trc::error!(err - .account_id(event.account_id) - .document_id(event.document_id) - .caused_by(trc::location!()) - .details("Failed to retrieve email metadata")); + trc::error!( + err.account_id(event.account_id) + .document_id(event.document_id) + .caused_by(trc::location!()) + .details("Failed to retrieve email metadata") + ); continue; } @@ -250,10 +254,11 @@ impl Indexer for Server { ) .await { - trc::error!(err - .account_id(event.account_id) - .document_id(event.document_id) - .details("Failed to remove index email from queue.")); + trc::error!( + err.account_id(event.account_id) + .document_id(event.document_id) + .details("Failed to remove index email from queue.") + ); } } @@ -285,10 +290,11 @@ impl Indexer for Server { result } Err(err) => { - trc::error!(err - .account_id(event.account_id) - .document_id(event.document_id) - .details("Failed to lock email task")); + trc::error!( + err.account_id(event.account_id) + .document_id(event.document_id) + .details("Failed to lock email task") + ); false } @@ -301,10 +307,11 @@ impl Indexer for Server { .remove_lock(KV_LOCK_EMAIL_TASK, &event.lock_key()) .await { - trc::error!(err - .details("Failed to unlock email task") - .ctx(trc::Key::Key, event.seq) - .caused_by(trc::location!())); + trc::error!( + err.details("Failed to unlock email task") + .ctx(trc::Key::Key, event.seq) + .caused_by(trc::location!()) + ); } } diff --git a/crates/jmap/src/sieve/get.rs b/crates/jmap/src/sieve/get.rs index 6e199e21..54a3eaff 100644 --- a/crates/jmap/src/sieve/get.rs +++ b/crates/jmap/src/sieve/get.rs @@ -5,10 +5,14 @@ */ use common::Server; +use email::sieve::SieveScript; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, - object::Object, - types::{collection::Collection, property::Property, value::Value}, + types::{ + collection::Collection, + property::Property, + value::{Object, Value}, + }, }; use store::BlobClass; @@ -66,8 +70,8 @@ impl SieveScriptGet for Server { response.not_found.push(id.into()); continue; } - let mut push = if let Some(push) = self - .get_property::>( + let mut sieve = if let Some(sieve) = self + .get_property::( account_id, Collection::SieveScript, document_id, @@ -75,7 +79,7 @@ impl SieveScriptGet for Server { ) .await? { - push + sieve } else { response.not_found.push(id.into()); continue; @@ -86,24 +90,20 @@ impl SieveScriptGet for Server { Property::Id => { result.append(Property::Id, Value::Id(id)); } - Property::Name | Property::IsActive => { - result.append(property.clone(), push.remove(property)); + Property::Name => { + result.append(Property::Name, Value::Text(std::mem::take(&mut sieve.name))); + } + Property::IsActive => { + result.append(Property::IsActive, Value::Bool(sieve.is_active)); } Property::BlobId => { - result.append( - Property::BlobId, - match push.remove(&Property::BlobId) { - Value::BlobId(mut blob_id) => { - blob_id.class = BlobClass::Linked { - account_id, - collection: Collection::SieveScript.into(), - document_id, - }; - Value::BlobId(blob_id) - } - other => other, - }, - ); + let mut blob_id = sieve.blob_id.clone(); + blob_id.class = BlobClass::Linked { + account_id, + collection: Collection::SieveScript.into(), + document_id, + }; + result.append(Property::BlobId, Value::BlobId(blob_id)); } property => { result.append(property.clone(), Value::Null); diff --git a/crates/jmap/src/sieve/set.rs b/crates/jmap/src/sieve/set.rs index 3e0e9a55..6942ffaa 100644 --- a/crates/jmap/src/sieve/set.rs +++ b/crates/jmap/src/sieve/set.rs @@ -8,14 +8,11 @@ use common::{ Server, auth::{AccessToken, ResourceToken}, }; +use email::sieve::SieveScript; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{SetRequest, SetResponse}, - object::{ - Object, - index::{IndexAs, IndexProperty, ObjectIndexBuilder}, - sieve::SetArguments, - }, + object::{index::ObjectIndexBuilder, sieve::SetArguments}, request::reference::MaybeReference, response::references::EvalObjectReferences, types::{ @@ -23,7 +20,7 @@ use jmap_proto::{ collection::Collection, id::Id, property::Property, - value::{MaybePatchValue, SetValue, Value}, + value::{MaybePatchValue, Object, SetValue, Value}, }, }; use rand::distr::Alphanumeric; @@ -32,10 +29,7 @@ use store::{ BlobClass, query::Filter, rand::{Rng, rng}, - write::{ - BatchBuilder, BlobOp, DirectoryClass, F_CLEAR, F_VALUE, assert::HashedValue, - log::ChangeLogBuilder, - }, + write::{BatchBuilder, BlobOp, F_CLEAR, F_VALUE, assert::HashedValue, log::ChangeLogBuilder}, }; use trc::AddContext; @@ -48,17 +42,6 @@ pub struct SetContext<'x> { response: SetResponse, } -pub static SCHEMA: &[IndexProperty] = &[ - IndexProperty::new(Property::Name) - .index_as(IndexAs::Text { - tokenize: true, - index: true, - }) - .max_size(255) - .required(), - IndexProperty::new(Property::IsActive).index_as(IndexAs::Integer), -]; - pub trait SieveScriptSet: Sync + Send { fn sieve_script_set( &self, @@ -78,10 +61,12 @@ pub trait SieveScriptSet: Sync + Send { fn sieve_set_item( &self, changes_: Object, - update: Option<(u32, HashedValue>)>, + update: Option<(u32, HashedValue)>, ctx: &SetContext, session_id: u64, - ) -> impl Future>), SetError>>> + Send; + ) -> impl Future< + Output = trc::Result, Option>), SetError>>, + > + Send; fn sieve_activate_script( &self, @@ -121,18 +106,24 @@ impl SieveScriptSet for Server { { Ok((mut builder, Some(blob))) => { // Store blob - let blob_id = builder.changes_mut().unwrap().blob_id_mut().unwrap(); + let blob_id = &mut builder.changes_mut().unwrap().blob_id; blob_id.hash = self.put_blob(account_id, &blob, false).await?.hash; - let script_size = blob_id.section.as_ref().unwrap().size; let mut blob_id = blob_id.clone(); + // Increment tenant quota + #[cfg(feature = "enterprise")] + if self.core.is_enterprise_edition() { + if let Some(tenant) = ctx.resource_token.tenant { + builder.set_tenant_id(tenant.id); + } + } + // Write record let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::SieveScript) .create_document() - .add(DirectoryClass::UsedQuota(account_id), script_size as i64) .set( BlobOp::Link { hash: blob_id.hash.clone(), @@ -141,14 +132,6 @@ impl SieveScriptSet for Server { ) .custom(builder); - // Increment tenant quota - #[cfg(feature = "enterprise")] - if self.core.is_enterprise_edition() { - if let Some(tenant) = ctx.resource_token.tenant { - batch.add(DirectoryClass::UsedQuota(tenant.id), script_size as i64); - } - } - let document_id = self .store() .write_expect_id(batch) @@ -199,7 +182,7 @@ impl SieveScriptSet for Server { // Obtain sieve script let document_id = id.document_id(); if let Some(sieve) = self - .get_property::>>( + .get_property::>( account_id, Collection::SieveScript, document_id, @@ -207,16 +190,7 @@ impl SieveScriptSet for Server { ) .await? { - let prev_blob_id = sieve - .inner - .blob_id() - .ok_or_else(|| { - trc::StoreEvent::NotFound - .into_err() - .caused_by(trc::location!()) - .document_id(document_id) - })? - .clone(); + let prev_blob_id = sieve.inner.blob_id.clone(); match self .sieve_set_item( @@ -237,31 +211,15 @@ impl SieveScriptSet for Server { let blob_id = if let Some(blob) = blob { // Store blob - let blob_id = builder.changes_mut().unwrap().blob_id_mut().unwrap(); + let blob_id = &mut builder.changes_mut().unwrap().blob_id; blob_id.hash = self.put_blob(account_id, &blob, false).await?.hash; - let script_size = blob_id.section.as_ref().unwrap().size as i64; - let prev_script_size = - prev_blob_id.section.as_ref().unwrap().size as i64; let blob_id = blob_id.clone(); - // Update quota - let update_quota = match script_size.cmp(&prev_script_size) { - std::cmp::Ordering::Greater => script_size - prev_script_size, - std::cmp::Ordering::Less => -prev_script_size + script_size, - std::cmp::Ordering::Equal => 0, - }; - if update_quota != 0 { - batch.add(DirectoryClass::UsedQuota(account_id), update_quota); - - // Update tenant quota - #[cfg(feature = "enterprise")] - if self.core.is_enterprise_edition() { - if let Some(tenant) = ctx.resource_token.tenant { - batch.add( - DirectoryClass::UsedQuota(tenant.id), - update_quota, - ); - } + // Update tenant quota + #[cfg(feature = "enterprise")] + if self.core.is_enterprise_edition() { + if let Some(tenant) = ctx.resource_token.tenant { + builder.set_tenant_id(tenant.id); } } @@ -393,7 +351,7 @@ impl SieveScriptSet for Server { // Fetch record let account_id = resource_token.account_id; let obj = self - .get_property::>>( + .get_property::>( account_id, Collection::SieveScript, document_id, @@ -408,42 +366,29 @@ impl SieveScriptSet for Server { })?; // Make sure the script is not active - if fail_if_active - && matches!( - obj.inner.properties.get(&Property::IsActive), - Some(Value::Bool(true)) - ) - { + if fail_if_active && obj.inner.is_active { return Ok(false); } + let blob_hash = obj.inner.blob_id.hash.clone(); + let mut builder = ObjectIndexBuilder::new().with_current(obj); + // Update tenant quota + #[cfg(feature = "enterprise")] + if self.core.is_enterprise_edition() { + if let Some(tenant) = resource_token.tenant { + builder.set_tenant_id(tenant.id); + } + } + // Delete record let mut batch = BatchBuilder::new(); - let blob_id = obj.inner.blob_id().ok_or_else(|| { - trc::StoreEvent::NotFound - .into_err() - .caused_by(trc::location!()) - .document_id(document_id) - })?; - let updated_quota = -(blob_id.section.as_ref().unwrap().size as i64); batch .with_account_id(account_id) .with_collection(Collection::SieveScript) .delete_document(document_id) .value(Property::EmailIds, (), F_VALUE | F_CLEAR) - .clear(BlobOp::Link { - hash: blob_id.hash.clone(), - }) - .add(DirectoryClass::UsedQuota(account_id), updated_quota) - .custom(ObjectIndexBuilder::new(SCHEMA).with_current(obj)); - - // Update tenant quota - #[cfg(feature = "enterprise")] - if self.core.is_enterprise_edition() { - if let Some(tenant) = resource_token.tenant { - batch.add(DirectoryClass::UsedQuota(tenant.id), updated_quota); - } - } + .clear(BlobOp::Link { hash: blob_hash }) + .custom(builder); self.store() .write(batch) @@ -456,12 +401,14 @@ impl SieveScriptSet for Server { async fn sieve_set_item( &self, changes_: Object, - update: Option<(u32, HashedValue>)>, + update: Option<(u32, HashedValue)>, ctx: &SetContext<'_>, session_id: u64, - ) -> trc::Result>), SetError>> { + ) -> trc::Result, Option>), SetError>> { // Vacation script cannot be modified - if matches!(update.as_ref().and_then(|(_, obj)| obj.inner.properties.get(&Property::Name)), Some(Value::Text ( value )) if value.eq_ignore_ascii_case("vacation")) + if update + .as_ref() + .is_some_and(|(_, obj)| obj.inner.name.eq_ignore_ascii_case("vacation")) { return Ok(Err(SetError::forbidden().with_description(concat!( "The 'vacation' script cannot be modified, ", @@ -470,16 +417,19 @@ impl SieveScriptSet for Server { } // Parse properties - let mut changes = Object::with_capacity(changes_.properties.len()); + let mut changes = update + .as_ref() + .map(|(_, obj)| obj.inner.clone()) + .unwrap_or_default(); let mut blob_id = None; - for (property, value) in changes_.properties { + for (property, value) in changes_.0 { let value = match ctx.response.eval_object_references(value) { Ok(value) => value, Err(err) => { return Ok(Err(err)); } }; - let value = match (&property, value) { + match (&property, value) { (Property::Name, MaybePatchValue::Value(Value::Text(value))) => { if value.len() > self.core.jmap.sieve_max_script_name { return Ok(Err(SetError::invalid_properties() @@ -493,10 +443,7 @@ impl SieveScriptSet for Server { ))); } else if update .as_ref() - .and_then(|(_, obj)| obj.inner.properties.get(&Property::Name)) - .is_none_or( - |p| matches!(p, Value::Text (prev_value ) if prev_value != &value), - ) + .is_none_or(|(_, obj)| obj.inner.name != value) { if let Some(id) = self .filter( @@ -517,7 +464,7 @@ impl SieveScriptSet for Server { } } - Value::Text(value) + changes.name = value; } (Property::BlobId, MaybePatchValue::Value(Value::BlobId(value))) => { blob_id = value.into(); @@ -531,28 +478,21 @@ impl SieveScriptSet for Server { .with_property(property) .with_description("Invalid property or value.".to_string()))); } - }; - changes.append(property, value); + } } if update.is_none() { // Add name if missing - if !matches!(changes.properties.get(&Property::Name), Some(Value::Text ( value )) if !value.is_empty()) - { - changes.set( - Property::Name, - Value::Text( - rng() - .sample_iter(Alphanumeric) - .take(15) - .map(char::from) - .collect::(), - ), - ); + if changes.name.is_empty() { + changes.name = rng() + .sample_iter(Alphanumeric) + .take(15) + .map(char::from) + .collect::(); } // Set script as inactive - changes.set(Property::IsActive, Value::Bool(false)); + changes.is_active = false; } let blob_update = if let Some(blob_id) = blob_id { @@ -582,10 +522,7 @@ impl SieveScriptSet for Server { // Compile script match self.core.sieve.untrusted_compiler.compile(&bytes) { Ok(script) => { - changes.set( - Property::BlobId, - BlobId::default().with_section_size(bytes.len()), - ); + changes.blob_id = BlobId::default().with_section_size(bytes.len()); bytes.extend(bincode::serialize(&script).unwrap_or_default()); bytes.into() } @@ -617,11 +554,12 @@ impl SieveScriptSet for Server { }; // Validate - Ok(ObjectIndexBuilder::new(SCHEMA) - .with_changes(changes) - .with_current_opt(update.map(|(_, current)| current)) - .validate() - .map(|obj| (obj, blob_update))) + Ok(Ok(( + ObjectIndexBuilder::new() + .with_changes(changes) + .with_current_opt(update.map(|(_, current)| current)), + blob_update, + ))) } async fn sieve_activate_script( @@ -658,7 +596,7 @@ impl SieveScriptSet for Server { // Deactivate scripts for document_id in active_ids { if let Some(sieve) = self - .get_property::>>( + .get_property::>( account_id, Collection::SieveScript, document_id, @@ -666,14 +604,14 @@ impl SieveScriptSet for Server { ) .await? { + let mut new_sieve = sieve.inner.clone(); + new_sieve.is_active = false; batch .update_document(document_id) .value(Property::EmailIds, (), F_VALUE | F_CLEAR) .custom( - ObjectIndexBuilder::new(SCHEMA) - .with_changes( - Object::with_capacity(1).with_property(Property::IsActive, false), - ) + ObjectIndexBuilder::new() + .with_changes(new_sieve) .with_current(sieve), ); changed_ids.push((document_id, false)); @@ -683,7 +621,7 @@ impl SieveScriptSet for Server { // Activate script if let Some(document_id) = activate_id { if let Some(sieve) = self - .get_property::>>( + .get_property::>( account_id, Collection::SieveScript, document_id, @@ -691,11 +629,11 @@ impl SieveScriptSet for Server { ) .await? { + let mut new_sieve = sieve.inner.clone(); + new_sieve.is_active = true; batch.update_document(document_id).custom( - ObjectIndexBuilder::new(SCHEMA) - .with_changes( - Object::with_capacity(1).with_property(Property::IsActive, true), - ) + ObjectIndexBuilder::new() + .with_changes(new_sieve) .with_current(sieve), ); changed_ids.push((document_id, true)); @@ -718,25 +656,3 @@ impl SieveScriptSet for Server { Ok(changed_ids) } } - -pub trait ObjectBlobId { - fn blob_id(&self) -> Option<&BlobId>; - fn blob_id_mut(&mut self) -> Option<&mut BlobId>; -} - -impl ObjectBlobId for Object { - fn blob_id(&self) -> Option<&BlobId> { - self.properties - .get(&Property::BlobId) - .and_then(|v| v.as_blob_id()) - } - - fn blob_id_mut(&mut self) -> Option<&mut BlobId> { - self.properties - .get_mut(&Property::BlobId) - .and_then(|v| match v { - Value::BlobId(blob_id) => Some(blob_id), - _ => None, - }) - } -} diff --git a/crates/jmap/src/submission/get.rs b/crates/jmap/src/submission/get.rs index 6881e2da..20345d5d 100644 --- a/crates/jmap/src/submission/get.rs +++ b/crates/jmap/src/submission/get.rs @@ -5,10 +5,16 @@ */ use common::Server; +use email::submission::{Address, Delivered, EmailSubmission, Envelope, UndoStatus}; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, - object::Object, - types::{collection::Collection, property::Property, value::Value}, + types::{ + collection::Collection, + date::UTCDate, + id::Id, + property::Property, + value::{Object, Value}, + }, }; use smtp::queue::{self, spool::SmtpSpool}; use std::future::Future; @@ -71,8 +77,8 @@ impl EmailSubmissionGet for Server { response.not_found.push(id.into()); continue; } - let mut push = if let Some(push) = self - .get_property::>( + let mut submission = if let Some(submission) = self + .get_property::( account_id, Collection::EmailSubmission, document_id, @@ -80,79 +86,77 @@ impl EmailSubmissionGet for Server { ) .await? { - push + submission } else { response.not_found.push(id.into()); continue; }; // Obtain queueId - let queued_message = self - .read_message(push.get(&Property::MessageId).as_uint().unwrap_or(u64::MAX)) - .await; + if let Some(queue_id) = submission.queue_id { + if let Some(mut queued_message) = self.read_message(queue_id).await { + for rcpt in std::mem::take(&mut queued_message.recipients) { + let rcpt_status = submission + .delivery_status + .get_mut_or_insert(rcpt.address_lcase); + rcpt_status.delivered = match &rcpt.status { + queue::Status::Scheduled | queue::Status::TemporaryFailure(_) => { + Delivered::Queued + } + queue::Status::Completed(_) => Delivered::Yes, + queue::Status::PermanentFailure(_) => Delivered::No, + }; + rcpt_status.smtp_reply = match &rcpt.status { + queue::Status::Completed(reply) => { + reply.response.to_string().replace('\n', " ") + } + queue::Status::TemporaryFailure(reply) + | queue::Status::PermanentFailure(reply) => { + reply.response.to_string().replace('\n', " ") + } + queue::Status::Scheduled => "250 2.1.5 Queued".to_string(), + }; + } + submission.undo_status = UndoStatus::Pending; + } + } let mut result = Object::with_capacity(properties.len()); for property in &properties { let value = match property { Property::Id => Value::Id(id), Property::DeliveryStatus => { - match (queued_message.as_ref(), push.remove(property)) { - (Some(message), Value::Object(mut status)) => { - for rcpt in &message.recipients { - status.set( - Property::_T(rcpt.address.clone()), - Object::with_capacity(3) - .with_property( - Property::Delivered, - match &rcpt.status { - queue::Status::Scheduled - | queue::Status::TemporaryFailure(_) => { - "queued" - } - queue::Status::Completed(_) => "yes", - queue::Status::PermanentFailure(_) => "no", - }, - ) - .with_property( - Property::SmtpReply, - match &rcpt.status { - queue::Status::Completed(reply) => reply - .response - .to_string() - .replace('\n', " "), - queue::Status::TemporaryFailure(reply) - | queue::Status::PermanentFailure(reply) => { - reply - .response - .to_string() - .replace('\n', " ") - } - queue::Status::Scheduled => { - "250 2.1.5 Queued".to_string() - } - }, - ) - .with_property(Property::Displayed, "unknown"), - ); - } + let mut status = Object::with_capacity(submission.delivery_status.len()); - Value::Object(status) - } - (_, value) => value, + for (rcpt, delivery_status) in + std::mem::take(&mut submission.delivery_status) + { + status.set( + Property::_T(rcpt), + Object::with_capacity(3) + .with_property( + Property::Delivered, + delivery_status.delivered.as_str().to_string(), + ) + .with_property(Property::SmtpReply, delivery_status.smtp_reply) + .with_property(Property::Displayed, "unknown"), + ); } + + Value::Object(status) } Property::UndoStatus => { - if queued_message.is_some() { - Value::Text("pending".to_string()) - } else { - push.remove(property) - } + Value::Text(submission.undo_status.as_str().to_string()) + } + Property::EmailId => { + Value::Id(Id::from_parts(submission.thread_id, submission.email_id)) + } + Property::IdentityId => Value::Id(Id::from(submission.identity_id)), + Property::ThreadId => Value::Id(Id::from(submission.thread_id)), + Property::Envelope => build_envelope(std::mem::take(&mut submission.envelope)), + Property::SendAt => { + Value::Date(UTCDate::from_timestamp(submission.send_at as i64)) } - Property::EmailId - | Property::IdentityId - | Property::ThreadId - | Property::Envelope - | Property::SendAt => push.remove(property), Property::MdnBlobIds | Property::DsnBlobIds => Value::List(vec![]), _ => Value::Null, }; @@ -165,3 +169,32 @@ impl EmailSubmissionGet for Server { Ok(response) } } + +fn build_envelope(envelope: Envelope) -> Value { + Object::with_capacity(2) + .with_property(Property::MailFrom, build_address(envelope.mail_from)) + .with_property( + Property::RcptTo, + Value::List(envelope.rcpt_to.into_iter().map(build_address).collect()), + ) + .into() +} + +fn build_address(envelope: Address) -> Value { + Object::with_capacity(2) + .with_property(Property::Email, Value::Text(envelope.email)) + .with_property( + Property::Parameters, + if let Some(params) = envelope.parameters { + Value::Object(Object( + params + .into_iter() + .map(|(k, v)| (Property::_T(k), v.into())) + .collect(), + )) + } else { + Value::Null + }, + ) + .into() +} diff --git a/crates/jmap/src/submission/query.rs b/crates/jmap/src/submission/query.rs index fb1592e3..9ce3a2cb 100644 --- a/crates/jmap/src/submission/query.rs +++ b/crates/jmap/src/submission/query.rs @@ -5,6 +5,7 @@ */ use common::Server; +use email::submission::UndoStatus; use jmap_proto::{ method::query::{ Comparator, Filter, QueryRequest, QueryResponse, RequestArguments, SortProperty, @@ -54,9 +55,12 @@ impl EmailSubmissionQuery for Server { } filters.push(query::Filter::End); } - Filter::UndoStatus(undo_status) => { - filters.push(query::Filter::eq(Property::UndoStatus, undo_status)) - } + Filter::UndoStatus(undo_status) => filters.push(query::Filter::eq( + Property::UndoStatus, + UndoStatus::parse(&undo_status) + .unwrap_or(UndoStatus::Pending) + .as_index(), + )), Filter::Before(before) => filters.push(query::Filter::lt( Property::SendAt, before.timestamp() as u64, @@ -71,7 +75,7 @@ impl EmailSubmissionQuery for Server { other => { return Err(trc::JmapEvent::UnsupportedFilter .into_err() - .details(other.to_string())) + .details(other.to_string())); } } } @@ -103,7 +107,7 @@ impl EmailSubmissionQuery for Server { other => { return Err(trc::JmapEvent::UnsupportedSort .into_err() - .details(other.to_string())) + .details(other.to_string())); } }); } diff --git a/crates/jmap/src/submission/set.rs b/crates/jmap/src/submission/set.rs index 5a182aac..abbc978e 100644 --- a/crates/jmap/src/submission/set.rs +++ b/crates/jmap/src/submission/set.rs @@ -7,29 +7,29 @@ use std::{collections::HashMap, sync::Arc}; use common::{ - listener::{stream::NullIo, ServerInstance}, Server, + listener::{ServerInstance, stream::NullIo}, +}; +use email::{ + identity::Identity, + message::metadata::MessageMetadata, + submission::{Address, Delivered, DeliveryStatus, EmailSubmission, UndoStatus}, }; -use email::metadata::MessageMetadata; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{self, SetRequest, SetResponse}, - object::{ - email_submission::SetArguments, - index::{IndexAs, IndexProperty, ObjectIndexBuilder}, - Object, - }, + object::{email_submission::SetArguments, index::ObjectIndexBuilder}, request::{ + Call, RequestMethod, method::{MethodFunction, MethodName, MethodObject}, reference::MaybeReference, - Call, RequestMethod, }, response::references::EvalObjectReferences, types::{ collection::Collection, - date::UTCDate, + id::Id, property::Property, - value::{MaybePatchValue, SetValue, Value}, + value::{MaybePatchValue, Object, SetValue, Value}, }, }; use mail_parser::{HeaderName, HeaderValue}; @@ -37,25 +37,14 @@ use smtp::{ core::{Session, SessionData, State}, queue::spool::SmtpSpool, }; -use smtp_proto::{request::parser::Rfc5321Parser, MailFrom, RcptTo}; -use store::write::{assert::HashedValue, log::ChangeLogBuilder, now, BatchBuilder, Bincode}; +use smtp_proto::{MailFrom, RcptTo, request::parser::Rfc5321Parser}; +use store::write::{BatchBuilder, Bincode, assert::HashedValue, log::ChangeLogBuilder, now}; use trc::AddContext; use utils::{map::vec_map::VecMap, sanitize_email}; use crate::blob::download::BlobDownload; use std::future::Future; -pub static SCHEMA: &[IndexProperty] = &[ - IndexProperty::new(Property::UndoStatus).index_as(IndexAs::Text { - tokenize: false, - index: true, - }), - IndexProperty::new(Property::EmailId).index_as(IndexAs::LongInteger), - IndexProperty::new(Property::IdentityId).index_as(IndexAs::Integer), - IndexProperty::new(Property::ThreadId).index_as(IndexAs::Integer), - IndexProperty::new(Property::SendAt).index_as(IndexAs::LongInteger), -]; - pub trait EmailSubmissionSet: Sync + Send { fn email_submission_set( &self, @@ -70,7 +59,7 @@ pub trait EmailSubmissionSet: Sync + Send { response: &SetResponse, instance: &Arc, object: Object, - ) -> impl Future, SetError>>> + Send; + ) -> impl Future>> + Send; } impl EmailSubmissionSet for Server { @@ -96,7 +85,7 @@ impl EmailSubmissionSet for Server { // Add id mapping success_email_ids.insert( id.clone(), - *submission.get(&Property::EmailId).as_id().unwrap(), + Id::from_parts(submission.thread_id, submission.email_id), ); // Insert record @@ -105,7 +94,7 @@ impl EmailSubmissionSet for Server { .with_account_id(account_id) .with_collection(Collection::EmailSubmission) .create_document() - .custom(ObjectIndexBuilder::new(SCHEMA).with_changes(submission)); + .custom(ObjectIndexBuilder::new().with_changes(submission)); let document_id = self .store() .write_expect_id(batch) @@ -131,7 +120,7 @@ impl EmailSubmissionSet for Server { // Obtain submission let document_id = id.document_id(); let submission = if let Some(submission) = self - .get_property::>>( + .get_property::>( account_id, Collection::EmailSubmission, document_id, @@ -148,7 +137,7 @@ impl EmailSubmissionSet for Server { let mut queue_id = u64::MAX; let mut undo_status = None; - for (property, value) in object.properties { + for (property, value) in object.0 { let value = match response.eval_object_references(value) { Ok(value) => value, Err(err) => { @@ -159,11 +148,11 @@ impl EmailSubmissionSet for Server { if let ( Property::UndoStatus, MaybePatchValue::Value(Value::Text(undo_status_)), - Value::UnsignedInt(queue_id_), - ) = (&property, value, submission.inner.get(&Property::MessageId)) + Some(queue_id_), + ) = (&property, value, submission.inner.queue_id) { undo_status = undo_status_.into(); - queue_id = *queue_id_; + queue_id = queue_id_; } else { response.not_updated.append( id, @@ -183,18 +172,17 @@ impl EmailSubmissionSet for Server { queue_message.remove(self, message_due).await; // Update record + let mut new_submission = submission.inner.clone(); + new_submission.undo_status = UndoStatus::Canceled; let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::EmailSubmission) .update_document(document_id) .custom( - ObjectIndexBuilder::new(SCHEMA) + ObjectIndexBuilder::new() .with_current(submission) - .with_changes( - Object::with_capacity(1) - .with_property(Property::UndoStatus, undo_status), - ), + .with_changes(new_submission), ); self.store() .write(batch) @@ -233,7 +221,7 @@ impl EmailSubmissionSet for Server { for id in will_destroy { let document_id = id.document_id(); if let Some(submission) = self - .get_property::>>( + .get_property::>( account_id, Collection::EmailSubmission, document_id, @@ -247,7 +235,7 @@ impl EmailSubmissionSet for Server { .with_account_id(account_id) .with_collection(Collection::EmailSubmission) .delete_document(document_id) - .custom(ObjectIndexBuilder::new(SCHEMA).with_current(submission)); + .custom(ObjectIndexBuilder::new().with_current(submission)); self.store() .write(batch) .await @@ -328,14 +316,14 @@ impl EmailSubmissionSet for Server { response: &SetResponse, instance: &Arc, object: Object, - ) -> trc::Result, SetError>> { - let mut submission = Object::with_capacity(object.properties.len()); - let mut email_id = u32::MAX; - let mut identity_id = u32::MAX; + ) -> trc::Result> { + let mut submission = EmailSubmission::default(); + submission.email_id = u32::MAX; + submission.identity_id = u32::MAX; let mut mail_from = None; let mut rcpt_to: Vec> = Vec::new(); - for (property, value) in object.properties { + for (property, value) in object.0 { let value = match response.eval_object_references(value) { Ok(value) => value, Err(err) => { @@ -343,23 +331,21 @@ impl EmailSubmissionSet for Server { } }; - let value = match (&property, value) { + match (&property, value) { (Property::EmailId, MaybePatchValue::Value(Value::Id(value))) => { - submission.append(Property::ThreadId, Value::Id(value.prefix_id().into())); - email_id = value.document_id(); - Value::Id(value) + submission.email_id = value.document_id(); + submission.thread_id = value.prefix_id(); } (Property::IdentityId, MaybePatchValue::Value(Value::Id(value))) => { - identity_id = value.document_id(); - Value::Id(value) + submission.identity_id = value.document_id(); } (Property::Envelope, MaybePatchValue::Value(Value::Object(value))) => { - for (property, value) in &value.properties { - match (property, value) { - (Property::MailFrom, _) => match parse_envelope_address(value) { - Ok((addr, params)) => { + for (property, value) in value.0 { + match (&property, value) { + (Property::MailFrom, value) => match parse_envelope_address(value) { + Ok((addr, params, smtp_params)) => { match Rfc5321Parser::new( - &mut params + &mut smtp_params .as_ref() .map_or(&b"\n"[..], |p| p.as_bytes()) .iter(), @@ -367,6 +353,10 @@ impl EmailSubmissionSet for Server { .mail_from_parameters(addr) { Ok(addr) => { + submission.envelope.mail_from = Address { + email: addr.address.clone(), + parameters: params, + }; mail_from = addr.into(); } Err(err) => { @@ -385,9 +375,9 @@ impl EmailSubmissionSet for Server { (Property::RcptTo, Value::List(value)) => { for addr in value { match parse_envelope_address(addr) { - Ok((addr, params)) => { + Ok((addr, params, smtp_params)) => { match Rfc5321Parser::new( - &mut params + &mut smtp_params .as_ref() .map_or(&b"\n"[..], |p| p.as_bytes()) .iter(), @@ -399,6 +389,10 @@ impl EmailSubmissionSet for Server { .iter() .any(|rcpt| rcpt.address == addr.address) { + submission.envelope.rcpt_to.push(Address { + email: addr.address.clone(), + parameters: params, + }); rcpt_to.push(addr); } } @@ -426,7 +420,6 @@ impl EmailSubmissionSet for Server { } } } - Value::Object(value) } (Property::Envelope, MaybePatchValue::Value(Value::Null)) => { continue; @@ -437,13 +430,11 @@ impl EmailSubmissionSet for Server { .with_property(property) .with_description("Field could not be set."))); } - }; - - submission.append(property, value); + } } // Make sure we have all required fields. - if email_id == u32::MAX || identity_id == u32::MAX { + if submission.email_id == u32::MAX || submission.identity_id == u32::MAX { return Ok(Err(SetError::invalid_properties() .with_properties([Property::EmailId, Property::IdentityId]) .with_description( @@ -452,18 +443,16 @@ impl EmailSubmissionSet for Server { } // Fetch identity's mailFrom - let identity_mail_from = if let Some(identity_mail_from) = self - .get_property::>( + let identity_mail_from = if let Some(identity) = self + .get_property::( account_id, Collection::Identity, - identity_id, + submission.identity_id, Property::Value, ) .await? - .and_then(|mut obj| obj.properties.remove(&Property::Email)) - .and_then(|value| value.try_unwrap_string()) { - identity_mail_from + identity.email } else { return Ok(Err(SetError::invalid_properties() .with_property(Property::IdentityId) @@ -480,18 +469,10 @@ impl EmailSubmissionSet for Server { } mail_from } else { - submission - .properties - .get_mut_or_insert_with(Property::Envelope, || { - Value::Object(Object::with_capacity(2)) - }) - .as_obj_mut() - .unwrap() - .set( - Property::MailFrom, - Object::with_capacity(1) - .with_property(Property::Email, identity_mail_from.clone()), - ); + submission.envelope.mail_from = Address { + email: identity_mail_from.clone(), + parameters: None, + }; MailFrom { address: identity_mail_from, ..Default::default() @@ -503,7 +484,7 @@ impl EmailSubmissionSet for Server { .get_property::>( account_id, Collection::Email, - email_id, + submission.email_id, Property::BodyStructure, ) .await? @@ -518,7 +499,6 @@ impl EmailSubmissionSet for Server { // Add recipients to envelope if missing let mut bcc_header = None; if rcpt_to.is_empty() { - let mut envelope_values = Vec::new(); for header in &metadata.contents.parts[0].headers { if matches!( header.name, @@ -531,10 +511,10 @@ impl EmailSubmissionSet for Server { for address in addr.iter() { if let Some(address) = address.address().and_then(sanitize_email) { if !rcpt_to.iter().any(|rcpt| rcpt.address == address) { - envelope_values.push(Value::Object( - Object::with_capacity(1) - .with_property(Property::Email, address.clone()), - )); + submission.envelope.rcpt_to.push(Address { + email: address.clone(), + parameters: None, + }); rcpt_to.push(RcptTo { address, ..Default::default() @@ -546,16 +526,7 @@ impl EmailSubmissionSet for Server { } } - if !rcpt_to.is_empty() { - submission - .properties - .get_mut_or_insert_with(Property::Envelope, || { - Value::Object(Object::with_capacity(1)) - }) - .as_obj_mut() - .unwrap() - .set(Property::RcptTo, Value::List(envelope_values)); - } else { + if rcpt_to.is_empty() { return Ok(Err(SetError::new(SetErrorType::NoRecipients) .with_description("No recipients found in email."))); } @@ -567,16 +538,13 @@ impl EmailSubmissionSet for Server { } // Update sendAt - submission.append( - Property::SendAt, - UTCDate::from_timestamp(if mail_from.hold_until > 0 { - mail_from.hold_until - } else if mail_from.hold_for > 0 { - mail_from.hold_for + now() - } else { - now() - } as i64), - ); + submission.send_at = if mail_from.hold_until > 0 { + mail_from.hold_until + } else if mail_from.hold_for > 0 { + mail_from.hold_for + now() + } else { + now() + }; // Obtain raw message let mut message = @@ -636,7 +604,7 @@ impl EmailSubmissionSet for Server { session.data.message = message; let response = session.queue_message().await; if let State::Accepted(queue_id) = session.state { - submission.append(Property::MessageId, queue_id); + submission.queue_id = Some(queue_id); } else { return Ok(Err(SetError::new(SetErrorType::ForbiddenToSend) .with_description(format!( @@ -647,66 +615,73 @@ impl EmailSubmissionSet for Server { } // Set responses - submission.append( - Property::UndoStatus, - if has_success { "final" } else { "failed" }, - ); - submission.append( - Property::DeliveryStatus, - Object { - properties: responses - .into_iter() - .map(|(addr, response)| { - ( - Property::_T(addr), - Value::Object( - Object::with_capacity(3) - .with_property( - Property::Delivered, - if response.is_none() { "unknown" } else { "no" }, - ) - .with_property( - Property::SmtpReply, - response.unwrap_or_else(|| "250 2.1.5 Queued".to_string()), - ) - .with_property(Property::Displayed, "unknown"), - ), - ) - }) - .collect::>(), - }, - ); + submission.undo_status = if has_success { + UndoStatus::Final + } else { + UndoStatus::Pending + }; + submission.delivery_status = responses + .into_iter() + .map(|(addr, response)| { + ( + addr, + DeliveryStatus { + delivered: if response.is_none() { + Delivered::No + } else { + Delivered::Unknown + }, + smtp_reply: response.unwrap_or_else(|| "250 2.1.5 Queued".to_string()), + displayed: false, + }, + ) + }) + .collect(); Ok(Ok(submission)) } } -fn parse_envelope_address(envelope: &Value) -> Result<(String, Option), SetError> { - if let Value::Object(envelope) = envelope { - if let Some(Value::Text(addr)) = envelope.properties.get(&Property::Email) { - if let Some(addr) = sanitize_email(addr) { - if let Some(Value::Object(params)) = envelope.properties.get(&Property::Parameters) - { +#[allow(clippy::type_complexity)] +fn parse_envelope_address( + envelope: Value, +) -> Result< + ( + String, + Option>>, + Option, + ), + SetError, +> { + if let Value::Object(mut envelope) = envelope { + if let Some(Value::Text(addr)) = envelope.0.remove(&Property::Email) { + if let Some(addr) = sanitize_email(&addr) { + if let Some(Value::Object(params)) = envelope.0.remove(&Property::Parameters) { let mut params_text = String::new(); - for (k, v) in params.properties.iter() { - if let Property::_T(k) = &k { + let mut params_list = VecMap::with_capacity(params.0.len()); + + for (k, v) in params.0 { + if let Property::_T(k) = k { if !k.is_empty() { if !params_text.is_empty() { params_text.push(' '); } - params_text.push_str(k); + params_text.push_str(&k); if let Value::Text(v) = v { params_text.push('='); - params_text.push_str(v); + params_text.push_str(&v); + params_list.append(k, Some(v)); + } else { + params_list.append(k, None); } } } } params_text.push('\n'); - Ok((addr, Some(params_text))) + Ok((addr, Some(params_list), Some(params_text))) } else { - Ok((addr, None)) + Ok((addr, None, None)) } } else { Err(SetError::invalid_properties() diff --git a/crates/jmap/src/thread/get.rs b/crates/jmap/src/thread/get.rs index ebc1036f..40c49d6d 100644 --- a/crates/jmap/src/thread/get.rs +++ b/crates/jmap/src/thread/get.rs @@ -7,11 +7,10 @@ use common::Server; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, - object::Object, - types::{collection::Collection, id::Id, property::Property}, + types::{collection::Collection, id::Id, property::Property, value::Object}, }; use std::future::Future; -use store::query::{sort::Pagination, Comparator, ResultSet}; +use store::query::{Comparator, ResultSet, sort::Pagination}; use trc::AddContext; use crate::changes::state::StateManager; @@ -42,7 +41,7 @@ impl ThreadGet for Server { }; let add_email_ids = request .properties - .is_none_or( |p| p.unwrap().contains(&Property::EmailIds)); + .is_none_or(|p| p.unwrap().contains(&Property::EmailIds)); let mut response = GetResponse { account_id: request.account_id.into(), state: self.get_state(account_id, Collection::Thread).await?.into(), diff --git a/crates/jmap/src/vacation/get.rs b/crates/jmap/src/vacation/get.rs index de491cba..1ed860b7 100644 --- a/crates/jmap/src/vacation/get.rs +++ b/crates/jmap/src/vacation/get.rs @@ -5,16 +5,23 @@ */ use common::Server; +use email::sieve::SieveScript; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, - object::Object, request::reference::MaybeReference, - types::{any_id::AnyId, collection::Collection, id::Id, property::Property, value::Value}, + types::{ + any_id::AnyId, + collection::Collection, + date::UTCDate, + id::Id, + property::Property, + value::{Object, Value}, + }, }; use std::future::Future; use store::query::Filter; -use crate::{changes::state::StateManager, JmapMethods}; +use crate::{JmapMethods, changes::state::StateManager}; pub trait VacationResponseGet: Sync + Send { fn vacation_response_get( @@ -73,7 +80,7 @@ impl VacationResponseGet for Server { if do_get { if let Some(document_id) = self.get_vacation_sieve_script_id(account_id).await? { if let Some(mut obj) = self - .get_property::>( + .get_property::( account_id, Collection::SieveScript, document_id, @@ -88,14 +95,47 @@ impl VacationResponseGet for Server { result.append(Property::Id, Value::Id(Id::singleton())); } Property::IsEnabled => { - result.append(Property::IsEnabled, obj.remove(&Property::IsActive)); + result.append(Property::IsEnabled, obj.is_active); } - Property::FromDate - | Property::ToDate - | Property::Subject - | Property::TextBody - | Property::HtmlBody => { - result.append(property.clone(), obj.remove(property)); + Property::FromDate => { + result.append( + Property::FromDate, + obj.vacation_response.as_mut().and_then(|r| { + r.from_date.take().map(UTCDate::from).map(Value::Date) + }), + ); + } + Property::ToDate => { + result.append( + Property::ToDate, + obj.vacation_response.as_mut().and_then(|r| { + r.to_date.take().map(UTCDate::from).map(Value::Date) + }), + ); + } + Property::Subject => { + result.append( + Property::Subject, + obj.vacation_response + .as_mut() + .and_then(|r| r.subject.take().map(Value::from)), + ); + } + Property::TextBody => { + result.append( + Property::TextBody, + obj.vacation_response + .as_mut() + .and_then(|r| r.text_body.take().map(Value::from)), + ); + } + Property::HtmlBody => { + result.append( + Property::HtmlBody, + obj.vacation_response + .as_mut() + .and_then(|r| r.html_body.take().map(Value::from)), + ); } property => { result.append(property.clone(), Value::Null); diff --git a/crates/jmap/src/vacation/set.rs b/crates/jmap/src/vacation/set.rs index b0d0e2a2..b98aad10 100644 --- a/crates/jmap/src/vacation/set.rs +++ b/crates/jmap/src/vacation/set.rs @@ -6,34 +6,33 @@ use std::borrow::Cow; -use common::{auth::AccessToken, Server}; +use common::{Server, auth::AccessToken}; +use email::sieve::{SieveScript, VacationResponse}; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{RequestArguments, SetRequest, SetResponse}, - object::{index::ObjectIndexBuilder, Object}, + object::index::ObjectIndexBuilder, response::references::EvalObjectReferences, types::{ blob::BlobId, collection::Collection, + date::UTCDate, id::Id, property::Property, - value::{MaybePatchValue, Value}, + value::{MaybePatchValue, Object, Value}, }, }; use mail_builder::MessageBuilder; use mail_parser::decoders::html::html_to_text; use std::future::Future; use store::write::{ + BatchBuilder, BlobOp, F_CLEAR, F_VALUE, assert::HashedValue, log::{Changes, LogInsert}, - BatchBuilder, BlobOp, DirectoryClass, F_CLEAR, F_VALUE, }; use trc::AddContext; -use crate::{ - sieve::set::{ObjectBlobId, SieveScriptSet, SCHEMA}, - JmapMethods, -}; +use crate::{JmapMethods, sieve::set::SieveScriptSet}; use super::get::VacationResponseGet; @@ -44,7 +43,7 @@ pub trait VacationResponseSet: Sync + Send { access_token: &AccessToken, ) -> impl Future> + Send; - fn build_script(&self, obj: &mut ObjectIndexBuilder) -> trc::Result>; + fn build_script(&self, obj: &mut SieveScript) -> trc::Result>; } impl VacationResponseSet for Server { @@ -125,13 +124,13 @@ impl VacationResponseSet for Server { .with_collection(Collection::SieveScript); // Process changes - if let Some(changes_) = changes { + if let Some(changes) = changes { // Parse properties - let mut changes = Object::with_capacity(changes_.properties.len()); + let mut vacation = VacationResponse::default(); let mut is_active = false; let mut build_script = create_id.is_some(); - for (property, value) in changes_.properties { + for (property, value) in changes.0 { let value = match response.eval_object_references(value) { Ok(value) => value, Err(err) => { @@ -143,29 +142,33 @@ impl VacationResponseSet for Server { if value.len() < 512 => { build_script = true; - changes.append(property, Value::Text(value)); + vacation.subject = Some(value); } - ( - Property::HtmlBody | Property::TextBody, - MaybePatchValue::Value(Value::Text(value)), - ) if value.len() < 2048 => { + (Property::HtmlBody, MaybePatchValue::Value(Value::Text(value))) + if value.len() < 2048 => + { build_script = true; - - changes.append(property, Value::Text(value)); + vacation.html_body = Some(value); } - ( - Property::ToDate | Property::FromDate, - MaybePatchValue::Value(value @ Value::Date(_)), - ) => { + (Property::TextBody, MaybePatchValue::Value(Value::Text(value))) + if value.len() < 2048 => + { + build_script = true; + vacation.text_body = Some(value); + } + (Property::FromDate, MaybePatchValue::Value(Value::Date(date))) => { + vacation.from_date = Some(date.timestamp() as u64); + build_script = true; + } + (Property::ToDate, MaybePatchValue::Value(Value::Date(date))) => { + vacation.to_date = Some(date.timestamp() as u64); build_script = true; - changes.append(property, value); } (Property::IsEnabled, MaybePatchValue::Value(Value::Bool(value))) => { is_active = value; - changes.append(Property::IsActive, value); } (Property::IsEnabled, MaybePatchValue::Value(Value::Null)) => { - changes.append(Property::IsActive, Value::Bool(false)); + is_active = false; } ( Property::Subject @@ -177,8 +180,24 @@ impl VacationResponseSet for Server { ) => { if create_id.is_none() { build_script = true; - - changes.append(property, Value::Null); + match property { + Property::Subject => { + vacation.subject = None; + } + Property::HtmlBody => { + vacation.html_body = None; + } + Property::TextBody => { + vacation.text_body = None; + } + Property::FromDate => { + vacation.from_date = None; + } + Property::ToDate => { + vacation.to_date = None; + } + _ => unreachable!(), + } } } _ => { @@ -193,41 +212,40 @@ impl VacationResponseSet for Server { } } - // Add name and isActive - if create_id.is_some() { - changes.append(Property::Name, Value::Text("vacation".into())); - if !changes.properties.contains_key(&Property::IsActive) { - changes.append(Property::IsActive, Value::Bool(false)); - } - } - // Obtain current script let document_id = self.get_vacation_sieve_script_id(account_id).await?; let mut was_active = false; - let mut obj = ObjectIndexBuilder::new(SCHEMA) - .with_current_opt(if let Some(document_id) = document_id { - self.get_property::>>( + let mut obj = if let Some(document_id) = document_id { + let prev_sieve = self + .get_property::>( account_id, Collection::SieveScript, document_id, Property::Value, ) .await? - .inspect(|value| { - was_active = value.inner.properties.get(&Property::IsActive) - == Some(&Value::Bool(true)); - }) .ok_or_else(|| { trc::StoreEvent::NotFound .into_err() .caused_by(trc::location!()) - })? - .into() - } else { - None + })?; + was_active = prev_sieve.inner.is_active; + let mut sieve = prev_sieve.inner.clone(); + sieve.vacation_response = vacation.into(); + sieve.is_active = is_active; + + ObjectIndexBuilder::new() + .with_current(prev_sieve) + .with_changes(sieve) + } else { + ObjectIndexBuilder::new().with_changes(SieveScript { + name: "vacation".into(), + is_active, + blob_id: Default::default(), + vacation_response: vacation.into(), }) - .with_changes(changes); + }; // Update id if let Some(document_id) = document_id { @@ -243,10 +261,14 @@ impl VacationResponseSet for Server { if build_script { // Upload new blob let hash = self - .put_blob(account_id, &self.build_script(&mut obj)?, false) + .put_blob( + account_id, + &self.build_script(obj.changes_mut().unwrap())?, + false, + ) .await? .hash; - let blob_id = obj.changes_mut().unwrap().blob_id_mut().unwrap(); + let blob_id = &mut obj.changes_mut().unwrap().blob_id; blob_id.hash = hash; // Link blob @@ -257,48 +279,18 @@ impl VacationResponseSet for Server { Vec::new(), ); - let script_size = blob_id.section.as_ref().unwrap().size as i64; - + // Unlink previous blob if let Some(current) = obj.current() { - let current_blob_id = current.inner.blob_id().ok_or_else(|| { - trc::StoreEvent::NotFound - .into_err() - .caused_by(trc::location!()) - .document_id(document_id.unwrap_or(u32::MAX)) - })?; - - // Unlink previous blob batch.clear(BlobOp::Link { - hash: current_blob_id.hash.clone(), + hash: current.inner.blob_id.hash.clone(), }); + } - // Update quota - let current_script_size = current_blob_id.section.as_ref().unwrap().size as i64; - let quota = match script_size.cmp(¤t_script_size) { - std::cmp::Ordering::Greater => script_size - current_script_size, - std::cmp::Ordering::Less => -current_script_size + script_size, - std::cmp::Ordering::Equal => 0, - }; - if quota != 0 { - batch.add(DirectoryClass::UsedQuota(account_id), quota); - - // Update tenant quota - #[cfg(feature = "enterprise")] - if self.core.is_enterprise_edition() { - if let Some(tenant) = resource_token.tenant { - batch.add(DirectoryClass::UsedQuota(tenant.id), quota); - } - } - } - } else { - batch.add(DirectoryClass::UsedQuota(account_id), script_size); - - // Update tenant quota - #[cfg(feature = "enterprise")] - if self.core.is_enterprise_edition() { - if let Some(tenant) = resource_token.tenant { - batch.add(DirectoryClass::UsedQuota(tenant.id), script_size); - } + // Update tenant quota + #[cfg(feature = "enterprise")] + if self.core.is_enterprise_edition() { + if let Some(tenant) = resource_token.tenant { + obj.set_tenant_id(tenant.id); } } }; @@ -364,30 +356,34 @@ impl VacationResponseSet for Server { Ok(response) } - fn build_script(&self, obj: &mut ObjectIndexBuilder) -> trc::Result> { + fn build_script(&self, obj: &mut SieveScript) -> trc::Result> { // Build Sieve script let mut script = Vec::with_capacity(1024); script.extend_from_slice(b"require [\"vacation\", \"relational\", \"date\"];\r\n\r\n"); let mut num_blocks = 0; // Add start date - if let Value::Date(value) = obj.get(&Property::FromDate) { + if let Some(value) = obj.vacation_response.as_ref().and_then(|v| v.from_date) { script.extend_from_slice(b"if currentdate :value \"ge\" \"iso8601\" \""); - script.extend_from_slice(value.to_string().as_bytes()); + script.extend_from_slice(UTCDate::from(value).to_string().as_bytes()); script.extend_from_slice(b"\" {\r\n"); num_blocks += 1; } // Add end date - if let Value::Date(value) = obj.get(&Property::ToDate) { + if let Some(value) = obj.vacation_response.as_ref().and_then(|v| v.to_date) { script.extend_from_slice(b"if currentdate :value \"le\" \"iso8601\" \""); - script.extend_from_slice(value.to_string().as_bytes()); + script.extend_from_slice(UTCDate::from(value).to_string().as_bytes()); script.extend_from_slice(b"\" {\r\n"); num_blocks += 1; } script.extend_from_slice(b"vacation :mime "); - if let Value::Text(value) = obj.get(&Property::Subject) { + if let Some(value) = obj + .vacation_response + .as_ref() + .and_then(|v| v.subject.as_ref()) + { script.extend_from_slice(b":subject \""); for &ch in value.as_bytes().iter() { match ch { @@ -404,12 +400,20 @@ impl VacationResponseSet for Server { script.extend_from_slice(b"\" "); } - let mut text_body = if let Value::Text(value) = obj.get(&Property::TextBody) { + let mut text_body = if let Some(value) = obj + .vacation_response + .as_ref() + .and_then(|v| v.text_body.as_ref()) + { Cow::from(value.as_str()).into() } else { None }; - let html_body = if let Value::Text(value) = obj.get(&Property::HtmlBody) { + let html_body = if let Some(value) = obj + .vacation_response + .as_ref() + .and_then(|v| v.html_body.as_ref()) + { Cow::from(value.as_str()).into() } else { None @@ -454,10 +458,7 @@ impl VacationResponseSet for Server { match self.core.sieve.untrusted_compiler.compile(&script) { Ok(compiled_script) => { // Update blob length - obj.set( - Property::BlobId, - BlobId::default().with_section_size(script.len()).into(), - ); + obj.blob_id = BlobId::default().with_section_size(script.len()); // Serialize script script.extend(bincode::serialize(&compiled_script).unwrap_or_default()); diff --git a/crates/managesieve/Cargo.toml b/crates/managesieve/Cargo.toml index bc4a202e..297ac005 100644 --- a/crates/managesieve/Cargo.toml +++ b/crates/managesieve/Cargo.toml @@ -13,6 +13,7 @@ directory = { path = "../directory" } common = { path = "../common" } store = { path = "../store" } utils = { path = "../utils" } +email = { path = "../email" } trc = { path = "../trc" } mail-parser = { version = "0.10", features = ["full_encoding"] } mail-send = { version = "0.5", default-features = false, features = ["cram-md5", "ring", "tls12"] } diff --git a/crates/managesieve/src/op/getscript.rs b/crates/managesieve/src/op/getscript.rs index 52c42fc9..17eb9f58 100644 --- a/crates/managesieve/src/op/getscript.rs +++ b/crates/managesieve/src/op/getscript.rs @@ -8,12 +8,10 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; +use email::sieve::SieveScript; use imap_proto::receiver::Request; -use jmap::{blob::download::BlobDownload, sieve::set::ObjectBlobId}; -use jmap_proto::{ - object::Object, - types::{collection::Collection, property::Property, value::Value}, -}; +use jmap::blob::download::BlobDownload; +use jmap_proto::types::{collection::Collection, property::Property}; use trc::AddContext; use crate::core::{Command, ResponseCode, Session, StatusResponse}; @@ -38,7 +36,7 @@ impl Session { let document_id = self.get_script_id(account_id, &name).await?; let (blob_section, blob_hash) = self .server - .get_property::>( + .get_property::( account_id, Collection::SieveScript, document_id, @@ -46,19 +44,12 @@ impl Session { ) .await .caused_by(trc::location!())? + .and_then(|id| (id.blob_id.section?, id.blob_id.hash).into()) .ok_or_else(|| { trc::ManageSieveEvent::Error .into_err() .details("Script not found") .code(ResponseCode::NonExistent) - })? - .blob_id() - .and_then(|id| (id.section.as_ref()?.clone(), id.hash.clone()).into()) - .ok_or_else(|| { - trc::ManageSieveEvent::Error - .into_err() - .details("Failed to retrieve blobId") - .code(ResponseCode::TryLater) })?; let script = self .server diff --git a/crates/managesieve/src/op/listscripts.rs b/crates/managesieve/src/op/listscripts.rs index 6437ebf6..300e43f0 100644 --- a/crates/managesieve/src/op/listscripts.rs +++ b/crates/managesieve/src/op/listscripts.rs @@ -8,10 +8,8 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; -use jmap_proto::{ - object::Object, - types::{collection::Collection, property::Property, value::Value}, -}; +use email::sieve::SieveScript; +use jmap_proto::types::{collection::Collection, property::Property}; use trc::AddContext; use crate::core::{Session, StatusResponse}; @@ -40,7 +38,7 @@ impl Session { for document_id in document_ids { if let Some(script) = self .server - .get_property::>( + .get_property::( account_id, Collection::SieveScript, document_id, @@ -50,16 +48,13 @@ impl Session { .caused_by(trc::location!())? { response.push(b'\"'); - if let Some(name) = script.get(&Property::Name).as_string() { - for ch in name.as_bytes() { - if [b'\\', b'\"'].contains(ch) { - response.push(b'\\'); - } - response.push(*ch); + for ch in script.name.as_bytes() { + if [b'\\', b'\"'].contains(ch) { + response.push(b'\\'); } + response.push(*ch); } - - if script.get(&Property::IsActive).as_bool() == Some(true) { + if script.is_active { response.extend_from_slice(b"\" ACTIVE\r\n"); } else { response.extend_from_slice(b"\"\r\n"); diff --git a/crates/managesieve/src/op/putscript.rs b/crates/managesieve/src/op/putscript.rs index 58de4279..5f3ff7d5 100644 --- a/crates/managesieve/src/op/putscript.rs +++ b/crates/managesieve/src/op/putscript.rs @@ -8,20 +8,18 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; +use email::sieve::SieveScript; use imap_proto::receiver::Request; -use jmap::{ - sieve::set::{ObjectBlobId, SCHEMA}, - JmapMethods, -}; +use jmap::JmapMethods; use jmap_proto::{ - object::{index::ObjectIndexBuilder, Object}, - types::{blob::BlobId, collection::Collection, property::Property, value::Value}, + object::index::ObjectIndexBuilder, + types::{blob::BlobId, collection::Collection, property::Property}, }; use sieve::compiler::ErrorType; use store::{ - query::Filter, - write::{assert::HashedValue, log::LogInsert, BatchBuilder, BlobOp, DirectoryClass}, BlobClass, + query::Filter, + write::{BatchBuilder, BlobOp, DirectoryClass, assert::HashedValue, log::LogInsert}, }; use trc::AddContext; @@ -107,7 +105,7 @@ impl Session { // Obtain script values let script = self .server - .get_property::>>( + .get_property::>( account_id, Collection::SieveScript, document_id, @@ -121,12 +119,6 @@ impl Session { .details("Script not found") .code(ResponseCode::NonExistent) })?; - let prev_blob_id = script.inner.blob_id().ok_or_else(|| { - trc::ManageSieveEvent::Error - .into_err() - .details("Internal error while obtaining blobId") - .code(ResponseCode::TryLater) - })?; // Write script blob let blob_id = BlobId::new( @@ -142,50 +134,33 @@ impl Session { }, ) .with_section_size(script_size as usize); + let prev_blob_id_hash = script.inner.blob_id.hash.clone(); + let blob_id_hash = blob_id.hash.clone(); // Write record + let mut obj = ObjectIndexBuilder::new() + .with_changes(script.inner.clone().with_blob_id(blob_id)) + .with_current(script); + + // Update tenant quota + #[cfg(feature = "enterprise")] + if self.server.core.is_enterprise_edition() { + if let Some(tenant) = resource_token.tenant { + obj.set_tenant_id(tenant.id); + } + } + let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::SieveScript) .update_document(document_id) .clear(BlobOp::Link { - hash: prev_blob_id.hash.clone(), + hash: prev_blob_id_hash, }) - .set( - BlobOp::Link { - hash: blob_id.hash.clone(), - }, - Vec::new(), - ); + .set(BlobOp::Link { hash: blob_id_hash }, Vec::new()) + .custom(obj); - // Update quota - let prev_script_size = prev_blob_id.section.as_ref().unwrap().size as i64; - let update_quota = match script_size.cmp(&prev_script_size) { - std::cmp::Ordering::Greater => script_size - prev_script_size, - std::cmp::Ordering::Less => -prev_script_size + script_size, - std::cmp::Ordering::Equal => 0, - }; - if update_quota != 0 { - batch.add(DirectoryClass::UsedQuota(account_id), update_quota); - - // Update tenant quota - #[cfg(feature = "enterprise")] - if self.server.core.is_enterprise_edition() { - if let Some(tenant) = resource_token.tenant { - batch.add(DirectoryClass::UsedQuota(tenant.id), update_quota); - } - } - } - - batch.custom( - ObjectIndexBuilder::new(SCHEMA) - .with_current(script) - .with_changes( - Object::with_capacity(1) - .with_property(Property::BlobId, Value::BlobId(blob_id)), - ), - ); self.server .store() .write(batch) @@ -214,8 +189,20 @@ impl Session { }, ) .with_section_size(script_size as usize); + let blob_id_hash = blob_id.hash.clone(); // Write record + let mut obj = ObjectIndexBuilder::new() + .with_changes(SieveScript::new(name.clone(), blob_id).with_is_active(false)); + + // Update tenant quota + #[cfg(feature = "enterprise")] + if self.server.core.is_enterprise_edition() { + if let Some(tenant) = resource_token.tenant { + obj.set_tenant_id(tenant.id); + } + } + let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) @@ -223,28 +210,8 @@ impl Session { .create_document() .log(LogInsert()) .add(DirectoryClass::UsedQuota(account_id), script_size) - .set( - BlobOp::Link { - hash: blob_id.hash.clone(), - }, - Vec::new(), - ) - .custom( - ObjectIndexBuilder::new(SCHEMA).with_changes( - Object::with_capacity(3) - .with_property(Property::Name, name.clone()) - .with_property(Property::IsActive, Value::Bool(false)) - .with_property(Property::BlobId, Value::BlobId(blob_id)), - ), - ); - - // Update tenant quota - #[cfg(feature = "enterprise")] - if self.server.core.is_enterprise_edition() { - if let Some(tenant) = resource_token.tenant { - batch.add(DirectoryClass::UsedQuota(tenant.id), script_size); - } - } + .set(BlobOp::Link { hash: blob_id_hash }, Vec::new()) + .custom(obj); let assigned_ids = self .server diff --git a/crates/managesieve/src/op/renamescript.rs b/crates/managesieve/src/op/renamescript.rs index 01e7f144..243b369d 100644 --- a/crates/managesieve/src/op/renamescript.rs +++ b/crates/managesieve/src/op/renamescript.rs @@ -8,13 +8,13 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; +use email::sieve::SieveScript; use imap_proto::receiver::Request; -use jmap::sieve::set::SCHEMA; use jmap_proto::{ - object::{index::ObjectIndexBuilder, Object}, - types::{collection::Collection, property::Property, value::Value}, + object::index::ObjectIndexBuilder, + types::{collection::Collection, property::Property}, }; -use store::write::{assert::HashedValue, log::ChangeLogBuilder, BatchBuilder}; +use store::write::{BatchBuilder, assert::HashedValue, log::ChangeLogBuilder}; use trc::AddContext; use crate::core::{Command, ResponseCode, Session, StatusResponse}; @@ -63,7 +63,7 @@ impl Session { // Obtain script values let script = self .server - .get_property::>>( + .get_property::>( account_id, Collection::SieveScript, document_id, @@ -85,11 +85,9 @@ impl Session { .with_collection(Collection::SieveScript) .update_document(document_id) .custom( - ObjectIndexBuilder::new(SCHEMA) - .with_current(script) - .with_changes( - Object::with_capacity(1).with_property(Property::Name, new_name.clone()), - ), + ObjectIndexBuilder::new() + .with_changes(script.inner.clone().with_name(new_name.clone())) + .with_current(script), ); if !batch.is_empty() { self.server diff --git a/crates/pop3/src/mailbox.rs b/crates/pop3/src/mailbox.rs index d865916a..8221d8f9 100644 --- a/crates/pop3/src/mailbox.rs +++ b/crates/pop3/src/mailbox.rs @@ -7,13 +7,10 @@ use std::collections::BTreeMap; use common::listener::SessionStream; -use email::mailbox::{MailboxFnc, UidMailbox, INBOX_ID}; -use jmap_proto::{ - object::Object, - types::{collection::Collection, property::Property, value::Value}, -}; +use email::mailbox::{INBOX_ID, UidMailbox, manage::MailboxFnc}; +use jmap_proto::types::{collection::Collection, property::Property}; use store::{ - ahash::AHashMap, write::key::DeserializeBigEndian, IndexKey, IterateParams, Serialize, U32_LEN, + IndexKey, IterateParams, Serialize, U32_LEN, ahash::AHashMap, write::key::DeserializeBigEndian, }; use trc::AddContext; @@ -64,7 +61,7 @@ impl Session { .caused_by(trc::location!())?; let uid_validity = self .server - .get_property::>( + .get_property::( account_id, Collection::Mailbox, INBOX_ID, @@ -72,15 +69,14 @@ impl Session { ) .await .caused_by(trc::location!())? - .and_then(|obj| obj.get(&Property::Cid).as_uint()) .ok_or_else(|| { trc::StoreEvent::UnexpectedError .caused_by(trc::location!()) .details("Failed to obtain UID validity") .account_id(account_id) .document_id(INBOX_ID) - }) - .map(|v| v as u32)?; + })? + .uid_validity; // Obtain message sizes self.server diff --git a/crates/pop3/src/op/delete.rs b/crates/pop3/src/op/delete.rs index 89990e58..73ebf36e 100644 --- a/crates/pop3/src/op/delete.rs +++ b/crates/pop3/src/op/delete.rs @@ -8,12 +8,12 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; -use jmap::email::delete::EmailDeletion; +use email::message::delete::EmailDeletion; use jmap_proto::types::{state::StateChange, type_state::DataType}; use store::roaring::RoaringBitmap; use trc::AddContext; -use crate::{protocol::response::Response, Session, State}; +use crate::{Session, State, protocol::response::Response}; impl Session { pub async fn handle_dele(&mut self, msgs: Vec) -> trc::Result<()> { diff --git a/crates/pop3/src/op/fetch.rs b/crates/pop3/src/op/fetch.rs index d37c631b..d74d9a6c 100644 --- a/crates/pop3/src/op/fetch.rs +++ b/crates/pop3/src/op/fetch.rs @@ -8,13 +8,13 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; -use email::metadata::MessageMetadata; +use email::message::metadata::MessageMetadata; use jmap::blob::download::BlobDownload; use jmap_proto::types::{collection::Collection, property::Property}; use store::write::Bincode; use trc::AddContext; -use crate::{protocol::response::Response, Session}; +use crate::{Session, protocol::response::Response}; impl Session { pub async fn handle_fetch(&mut self, msg: u32, lines: Option) -> trc::Result<()> { diff --git a/crates/smtp/src/outbound/local.rs b/crates/smtp/src/outbound/local.rs index aa5a72f3..880cc710 100644 --- a/crates/smtp/src/outbound/local.rs +++ b/crates/smtp/src/outbound/local.rs @@ -5,14 +5,14 @@ */ use common::Server; -use email::delivery::{IngestMessage, LocalDeliveryStatus, MailDelivery}; +use email::message::delivery::{IngestMessage, LocalDeliveryStatus, MailDelivery}; use smtp_proto::Response; use trc::SieveEvent; use crate::{ queue::{ - quota::HasQueueQuota, spool::SmtpSpool, DomainPart, Error, ErrorDetails, HostResponse, - Message, MessageSource, Recipient, Status, RCPT_STATUS_CHANGED, + DomainPart, Error, ErrorDetails, HostResponse, Message, MessageSource, RCPT_STATUS_CHANGED, + Recipient, Status, quota::HasQueueQuota, spool::SmtpSpool, }, reporting::SmtpReporting, }; diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index b451e742..012e87d2 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -34,6 +34,9 @@ http-body-util = "0.1.0" form_urlencoded = "1.1.0" psl = "2" quick_cache = "0.6.9" +downcast-rs = "2.0.1" +fast-float = "0.2.0" +erased-serde = "0.4.5" [target.'cfg(unix)'.dependencies] privdrop = "0.5.3" diff --git a/crates/utils/src/json/mod.rs b/crates/utils/src/json/mod.rs new file mode 100644 index 00000000..574fe6d1 --- /dev/null +++ b/crates/utils/src/json/mod.rs @@ -0,0 +1,26 @@ +pub mod parser; +pub mod pointer; + +use downcast_rs::{Downcast, impl_downcast}; +use std::{fmt::Debug, slice::Iter}; + +pub trait JsonQueryable: Downcast + Debug + 'static { + fn eval_pointer<'x>( + &'x self, + pointer: Iter, + results: &mut Vec<&'x dyn JsonQueryable>, + ); +} + +impl_downcast!(JsonQueryable); + +#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize)] +pub struct JsonPointer(pub Vec); + +#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize)] +pub enum JsonPointerItem { + Root, + Wildcard, + String(String), + Number(u64), +} diff --git a/crates/utils/src/json/parser/base32.rs b/crates/utils/src/json/parser/base32.rs new file mode 100644 index 00000000..f99d725f --- /dev/null +++ b/crates/utils/src/json/parser/base32.rs @@ -0,0 +1,65 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::codec::{base32_custom::BASE32_INVERSE, leb128::Leb128Iterator}; + +use super::json::Parser; + +#[derive(Debug)] +pub struct JsonBase32Reader<'x, 'y> { + bytes: &'y mut Parser<'x>, + last_byte: u8, + pos: usize, +} + +impl<'x, 'y> JsonBase32Reader<'x, 'y> { + pub fn new(bytes: &'y mut Parser<'x>) -> Self { + JsonBase32Reader { + bytes, + pos: 0, + last_byte: 0, + } + } + + #[inline(always)] + fn map_byte(&mut self) -> Option { + match self.bytes.next_unescaped() { + Ok(Some(byte)) => match BASE32_INVERSE[byte as usize] { + decoded_byte if decoded_byte != u8::MAX => { + self.last_byte = decoded_byte; + Some(decoded_byte) + } + _ => None, + }, + _ => None, + } + } + + pub fn error(&mut self) -> trc::Error { + self.bytes.error_value() + } +} + +impl Iterator for JsonBase32Reader<'_, '_> { + type Item = u8; + fn next(&mut self) -> Option { + let pos = self.pos % 5; + let last_byte = self.last_byte; + let byte = self.map_byte()?; + self.pos += 1; + + match pos { + 0 => ((byte << 3) | (self.map_byte().unwrap_or(0) >> 2)).into(), + 1 => ((last_byte << 6) | (byte << 1) | (self.map_byte().unwrap_or(0) >> 4)).into(), + 2 => ((last_byte << 4) | (byte >> 1)).into(), + 3 => ((last_byte << 7) | (byte << 2) | (self.map_byte().unwrap_or(0) >> 3)).into(), + 4 => ((last_byte << 5) | byte).into(), + _ => None, + } + } +} + +impl Leb128Iterator for JsonBase32Reader<'_, '_> {} diff --git a/crates/utils/src/json/parser/impls.rs b/crates/utils/src/json/parser/impls.rs new file mode 100644 index 00000000..b250f3ad --- /dev/null +++ b/crates/utils/src/json/parser/impls.rs @@ -0,0 +1,308 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::fmt::Display; + +use crate::map::{ + bitmap::{Bitmap, BitmapItem}, + vec_map::VecMap, +}; + +use super::{Ignore, JsonObjectParser, Token, json::Parser}; + +impl JsonObjectParser for u64 { + fn parse(parser: &mut Parser<'_>) -> trc::Result + where + Self: Sized, + { + let mut hash = 0; + let mut shift = 0; + + while let Some(ch) = parser.next_unescaped()? { + if shift < 64 { + hash |= (ch as u64) << shift; + shift += 8; + } else { + hash = 0; + break; + } + } + + Ok(hash) + } +} + +impl JsonObjectParser for u128 { + fn parse(parser: &mut Parser<'_>) -> trc::Result + where + Self: Sized, + { + let mut hash = 0; + let mut shift = 0; + + while let Some(ch) = parser.next_unescaped()? { + if shift < 128 { + hash |= (ch as u128) << shift; + shift += 8; + } else { + hash = 0; + break; + } + } + + Ok(hash) + } +} + +impl JsonObjectParser for String { + fn parse(parser: &mut Parser<'_>) -> trc::Result + where + Self: Sized, + { + let start_pos = parser.pos; + + while let Some(ch) = parser.next_char() { + match ch { + b'\\' => { + let mut is_escaped = true; + let mut buf = Vec::with_capacity((parser.pos - start_pos) + 16); + buf.extend_from_slice(&parser.bytes[start_pos..parser.pos - 1]); + + while let Some(ch) = parser.next_char() { + match ch { + b'\\' if !is_escaped => { + is_escaped = true; + } + b'"' if !is_escaped => { + parser.is_eof = true; + return String::from_utf8(buf).map_err(|_| parser.error_utf8()); + } + _ => { + if !is_escaped { + buf.push(ch); + } else { + match ch { + b'"' => { + buf.push(b'"'); + } + b'\\' => { + buf.push(b'\\'); + } + b'n' => { + buf.push(b'\n'); + } + b't' => { + buf.push(b'\t'); + } + b'r' => { + buf.push(b'\r'); + } + b'b' => { + buf.push(0x08); + } + b'f' => { + buf.push(0x0c); + } + b'/' => { + buf.push(b'/'); + } + b'u' => { + let mut code = [ + *parser.iter.next().ok_or_else(|| { + parser.error("Incomplete unicode sequence") + })?, + *parser.iter.next().ok_or_else(|| { + parser.error("Incomplete unicode sequence") + })?, + *parser.iter.next().ok_or_else(|| { + parser.error("Incomplete unicode sequence") + })?, + *parser.iter.next().ok_or_else(|| { + parser.error("Incomplete unicode sequence") + })?, + ]; + parser.pos += 4; + let code_str = std::str::from_utf8(&code) + .map_err(|_| parser.error_utf8())?; + let code_str = char::from_u32( + u32::from_str_radix(code_str, 16).map_err( + |_| { + parser.error(&format!( + "Invalid unicode sequence {code_str}" + )) + }, + )?, + ) + .ok_or_else(|| { + parser.error(&format!( + "Invalid unicode sequence {code_str}" + )) + })? + .encode_utf8(&mut code); + buf.extend_from_slice(code_str.as_bytes()); + } + _ => { + buf.push(ch); + } + } + is_escaped = false; + } + } + } + } + break; + } + b'"' => { + parser.is_eof = true; + return std::str::from_utf8( + parser + .bytes + .get(start_pos..parser.pos - 1) + .unwrap_or_default(), + ) + .map(Into::into) + .map_err(|_| parser.error_utf8()); + } + _ => (), + } + } + + Err(parser.error_unterminated()) + } +} + +impl JsonObjectParser for Vec { + fn parse(parser: &mut Parser<'_>) -> trc::Result + where + Self: Sized, + { + let mut vec = Vec::new(); + + parser.next_token::()?.assert(Token::ArrayStart)?; + loop { + match parser.next_token::()? { + Token::String(item) => vec.push(item), + Token::Comma => (), + Token::ArrayEnd => break, + token => return Err(token.error("", "[ or string")), + } + } + Ok(vec) + } +} + +impl JsonObjectParser for Option> { + fn parse(parser: &mut Parser<'_>) -> trc::Result + where + Self: Sized, + { + match parser.next_token::()? { + Token::ArrayStart => { + let mut vec = Vec::new(); + loop { + match parser.next_token::()? { + Token::String(item) => vec.push(item), + Token::Comma => (), + Token::ArrayEnd => break, + token => return Err(token.error("", "string")), + } + } + Ok(Some(vec)) + } + Token::Null => Ok(None), + token => Err(token.error("", "array or null")), + } + } +} + +impl JsonObjectParser for Bitmap { + fn parse(parser: &mut Parser<'_>) -> trc::Result + where + Self: Sized, + { + let mut bm = Bitmap::new(); + match parser.next_token::()? { + Token::ArrayStart => { + loop { + match parser.next_token::()? { + Token::String(item) => bm.insert(item), + Token::Comma => (), + Token::ArrayEnd => break, + token => return Err(token.error("", "string")), + } + } + Ok(bm) + } + Token::Null => Ok(bm), + token => Err(token.error("", "array or null")), + } + } +} + +impl JsonObjectParser for VecMap { + fn parse(parser: &mut Parser<'_>) -> trc::Result + where + Self: Sized, + { + let mut map = VecMap::new(); + + parser.next_token::()?.assert(Token::DictStart)?; + while let Some(key) = parser.next_dict_key()? { + map.append(key, V::parse(parser)?); + } + + Ok(map) + } +} + +impl JsonObjectParser + for Option> +{ + fn parse(parser: &mut Parser<'_>) -> trc::Result + where + Self: Sized, + { + match parser.next_token::()? { + Token::DictStart => { + let mut map = VecMap::new(); + + while let Some(key) = parser.next_dict_key()? { + map.append(key, V::parse(parser)?); + } + + Ok(Some(map)) + } + Token::Null => Ok(None), + token => Err(token.error("", &token.to_string())), + } + } +} + +impl JsonObjectParser for bool { + fn parse(parser: &mut Parser<'_>) -> trc::Result + where + Self: Sized, + { + match parser.next_token::()? { + Token::Boolean(value) => Ok(value), + Token::Null => Ok(false), + token => Err(token.error("", &token.to_string())), + } + } +} + +impl JsonObjectParser for Ignore { + fn parse(parser: &mut Parser<'_>) -> trc::Result + where + Self: Sized, + { + if parser.skip_string() { + Ok(Ignore {}) + } else { + Err(parser.error_unterminated()) + } + } +} diff --git a/crates/utils/src/json/parser/json.rs b/crates/utils/src/json/parser/json.rs new file mode 100644 index 00000000..1a1c1ce1 --- /dev/null +++ b/crates/utils/src/json/parser/json.rs @@ -0,0 +1,386 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::{fmt::Display, iter::Peekable, slice::Iter}; + +use super::{Ignore, JsonObjectParser, Token}; + +const MAX_NESTED_LEVELS: u32 = 16; + +#[derive(Debug)] +pub struct Parser<'x> { + pub bytes: &'x [u8], + pub iter: Peekable>, + pub next_ch: Option, + pub pos: usize, + pub pos_marker: usize, + pub depth_array: u32, + pub depth_dict: u32, + pub is_eof: bool, +} + +impl<'x> Parser<'x> { + pub fn new(bytes: &'x [u8]) -> Self { + Self { + bytes, + iter: bytes.iter().peekable(), + next_ch: None, + pos: 0, + pos_marker: 0, + is_eof: false, + depth_array: 0, + depth_dict: 0, + } + } + + pub fn error(&self, message: &str) -> trc::Error { + trc::JmapEvent::NotJson + .into_err() + .details(format!("{message} at position {}.", self.pos)) + } + + pub fn error_unterminated(&self) -> trc::Error { + trc::JmapEvent::NotJson.into_err().details(format!( + "Unterminated string at position {pos}.", + pos = self.pos + )) + } + + pub fn error_utf8(&self) -> trc::Error { + trc::JmapEvent::NotJson.into_err().details(format!( + "Invalid UTF-8 sequence at position {pos}.", + pos = self.pos + )) + } + + pub fn error_value(&mut self) -> trc::Error { + if self.is_eof || self.skip_string() { + trc::JmapEvent::InvalidArguments.into_err().details(format!( + "Invalid value {:?} at position {}.", + String::from_utf8_lossy(self.bytes[self.pos_marker..self.pos - 1].as_ref()), + self.pos + )) + } else { + self.error_unterminated() + } + } + + #[inline(always)] + pub fn peek_char(&mut self) -> Option { + self.iter.peek().map(|&&ch| ch) + } + + #[inline(always)] + pub fn next_char(&mut self) -> Option { + self.pos += 1; + self.iter.next().copied() + } + + #[inline(always)] + pub fn next_unescaped(&mut self) -> trc::Result> { + match self.next_char() { + Some(b'"') => { + self.is_eof = true; + Ok(None) + } + Some(b'\\') => self + .next_char() + .ok_or_else(|| self.error_unterminated()) + .map(Some), + Some(ch) => Ok(Some(ch)), + None => { + if self.is_eof { + Ok(None) + } else { + Err(self.error_unterminated()) + } + } + } + } + + pub fn skip_string(&mut self) -> bool { + let mut last_ch = 0; + + while let Some(ch) = self.next_char() { + if ch == b'"' && last_ch != b'\\' { + self.is_eof = true; + return true; + } else { + last_ch = ch; + } + } + + false + } + + pub fn next_token(&mut self) -> trc::Result> { + let mut next_ch = self.next_ch.take().or_else(|| self.next_char()); + + while let Some(mut ch) = next_ch { + match ch { + b'"' => { + self.pos_marker = self.pos; + self.is_eof = false; + let value = T::parse(self)?; + return if self.is_eof || self.skip_string() { + Ok(Token::String(value)) + } else { + Err(self.error_unterminated()) + }; + } + b',' => { + return Ok(Token::Comma); + } + b':' => { + return Ok(Token::Colon); + } + b'[' => { + if self.depth_array + self.depth_dict < MAX_NESTED_LEVELS { + self.depth_array += 1; + return Ok(Token::ArrayStart); + } else { + return Err(self.error("Too many nested objects")); + } + } + b']' => { + return if self.depth_array != 0 { + self.depth_array -= 1; + Ok(Token::ArrayEnd) + } else { + Err(self.error("Unexpected array end")) + }; + } + b'{' => { + if self.depth_array + self.depth_dict < MAX_NESTED_LEVELS { + self.depth_dict += 1; + return Ok(Token::DictStart); + } else { + return Err(self.error("Too many nested objects")); + } + } + b'}' => { + return if self.depth_dict != 0 { + self.depth_dict -= 1; + Ok(Token::DictEnd) + } else { + Err(self.error("Unexpected dictionary end")) + }; + } + b'0'..=b'9' | b'-' | b'+' => { + let mut num: i64 = 0; + let mut is_float = false; + let mut is_negative = false; + let num_start = self.pos - 1; + + loop { + match ch { + b'-' => { + is_negative = true; + } + b'0'..=b'9' => { + if !is_float { + num = num.saturating_mul(10).saturating_add((ch - b'0') as i64); + } + } + b',' | b']' | b'}' => { + self.next_ch = ch.into(); + break; + } + b'+' => (), + b'.' | b'e' | b'E' => { + is_float = true; + } + b' ' | b'\r' | b'\t' | b'\n' => { + break; + } + _ => { + return Err(self + .error(&format!("Unexpected character {:?}", char::from(ch)))); + } + } + + ch = self.next_char().ok_or_else(|| self.error_unterminated())?; + } + + return if !is_float { + Ok(Token::Integer(if !is_negative { num } else { -num })) + } else { + fast_float::parse( + self.bytes.get(num_start..self.pos - 1).unwrap_or_default(), + ) + .map(Token::Float) + .map_err(|_| { + self.error(&format!( + "Failed to parse number {:?}", + String::from_utf8_lossy( + self.bytes.get(num_start..self.pos - 1).unwrap_or_default() + ) + )) + }) + }; + } + b't' => { + return if let (Some(b'r'), Some(b'u'), Some(b'e')) = + (self.iter.next(), self.iter.next(), self.iter.next()) + { + self.pos += 3; + Ok(Token::Boolean(true)) + } else { + Err(self.error("Invalid JSON token")) + }; + } + b'f' => { + return if let (Some(b'a'), Some(b'l'), Some(b's'), Some(b'e')) = ( + self.iter.next(), + self.iter.next(), + self.iter.next(), + self.iter.next(), + ) { + self.pos += 4; + Ok(Token::Boolean(false)) + } else { + Err(self.error("Invalid JSON token")) + }; + } + b'n' => { + return if let (Some(b'u'), Some(b'l'), Some(b'l')) = + (self.iter.next(), self.iter.next(), self.iter.next()) + { + self.pos += 3; + Ok(Token::Null) + } else { + Err(self.error("Invalid JSON token")) + }; + } + b' ' | b'\t' | b'\r' | b'\n' => (), + _ => { + return Err(self.error(&format!("Unexpected character {:?}", char::from(ch)))); + } + } + + next_ch = self.next_char(); + } + + Err(self.error("Unexpected EOF")) + } + + pub fn next_dict_key(&mut self) -> trc::Result> { + loop { + match self.next_token::()? { + Token::String(k) => { + self.next_token::()?.assert(Token::Colon)?; + return Ok(Some(k)); + } + Token::Comma => (), + Token::DictEnd => return Ok(None), + token => { + return Err(self.error(&format!("Expected object property, found {}", token))); + } + } + } + } + + pub fn skip_token(&mut self, start_depth_array: u32, start_depth_dict: u32) -> trc::Result<()> { + while { + self.next_token::()?; + start_depth_array != self.depth_array || start_depth_dict != self.depth_dict + } {} + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + + use crate::json::parser::Token; + + use super::Parser; + + #[test] + fn parse_json() { + for (input, expected_result) in [ + ( + &b"[true, false, 123, 456 , -123, 0.123, -0.456, 3.7e-5, 6.02e+23, null]"[..], + vec![ + Token::ArrayStart, + Token::Boolean(true), + Token::Comma, + Token::Boolean(false), + Token::Comma, + Token::Integer(123), + Token::Comma, + Token::Integer(456), + Token::Comma, + Token::Integer(-123), + Token::Comma, + Token::Float(0.123), + Token::Comma, + Token::Float(-0.456), + Token::Comma, + Token::Float(3.7e-5), + Token::Comma, + Token::Float(6.02e23), + Token::Comma, + Token::Null, + Token::ArrayEnd, + ], + ), + ( + &b"{\"\": true, \"\": false , \"\": {\"\": 123}, \"\": [ ]}"[..], + vec![ + Token::DictStart, + Token::String("".to_string()), + Token::Colon, + Token::Boolean(true), + Token::Comma, + Token::String("".to_string()), + Token::Colon, + Token::Boolean(false), + Token::Comma, + Token::String("".to_string()), + Token::Colon, + Token::DictStart, + Token::String("".to_string()), + Token::Colon, + Token::Integer(123), + Token::DictEnd, + Token::Comma, + Token::String("".to_string()), + Token::Colon, + Token::ArrayStart, + Token::ArrayEnd, + Token::DictEnd, + ], + ), + ] { + let mut p = Parser::new(input); + let mut result = Vec::new(); + while let Ok(token) = p.next_token() { + result.push(token); + } + + assert_eq!(result, expected_result); + } + + for (input, expected_result) in [ + ("hello\t\nworld", "hello\t\nworld"), + ("hello\t\n\\\"world\\\"\\n", "hello\t\n\"world\"\n"), + ("\\\"hello\\\tworld\\\"", "\"hello\tworld\""), + ("\\u0009\\u0020\\u263A", "\t ☺"), + ("", ""), + ] { + assert_eq!( + Parser::new(format!("\"{input}\"").as_bytes()) + .next_token::() + .unwrap() + .unwrap_string("") + .unwrap(), + expected_result + ); + } + } +} diff --git a/crates/utils/src/json/parser/mod.rs b/crates/utils/src/json/parser/mod.rs new file mode 100644 index 00000000..a4249fb1 --- /dev/null +++ b/crates/utils/src/json/parser/mod.rs @@ -0,0 +1,158 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::fmt::Display; + +use self::json::Parser; + +pub mod base32; +pub mod impls; +pub mod json; +pub mod pointer; + +#[derive(Debug, PartialEq, Clone)] +pub enum Token { + Colon, + Comma, + DictStart, + DictEnd, + ArrayStart, + ArrayEnd, + Integer(i64), + Float(f64), + Boolean(bool), + String(T), + Null, +} + +impl Eq for Token {} + +pub trait JsonObjectParser { + fn parse(parser: &mut Parser<'_>) -> trc::Result + where + Self: Sized; +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Ignore {} + +impl Token { + pub fn unwrap_string(self, property: &str) -> trc::Result { + match self { + Token::String(s) => Ok(s), + token => Err(token.error(property, "string")), + } + } + + pub fn unwrap_string_or_null(self, property: &str) -> trc::Result> { + match self { + Token::String(s) => Ok(Some(s)), + Token::Null => Ok(None), + token => Err(token.error(property, "string")), + } + } + + pub fn unwrap_bool(self, property: &str) -> trc::Result { + match self { + Token::Boolean(v) => Ok(v), + token => Err(token.error(property, "boolean")), + } + } + + pub fn unwrap_bool_or_null(self, property: &str) -> trc::Result> { + match self { + Token::Boolean(v) => Ok(Some(v)), + Token::Null => Ok(None), + token => Err(token.error(property, "boolean")), + } + } + + pub fn unwrap_usize_or_null(self, property: &str) -> trc::Result> { + match self { + Token::Integer(v) if v >= 0 => Ok(Some(v as usize)), + Token::Float(v) if v >= 0.0 => Ok(Some(v as usize)), + Token::Null => Ok(None), + token => Err(token.error(property, "unsigned integer")), + } + } + + pub fn unwrap_uint_or_null(self, property: &str) -> trc::Result> { + match self { + Token::Integer(v) if v >= 0 => Ok(Some(v as u64)), + Token::Float(v) if v >= 0.0 => Ok(Some(v as u64)), + Token::Null => Ok(None), + token => Err(token.error(property, "unsigned integer")), + } + } + + pub fn unwrap_int_or_null(self, property: &str) -> trc::Result> { + match self { + Token::Integer(v) => Ok(Some(v)), + Token::Float(v) => Ok(Some(v as i64)), + Token::Null => Ok(None), + token => Err(token.error(property, "unsigned integer")), + } + } + + pub fn unwrap_ints_or_null(self, property: &str) -> trc::Result> { + match self { + Token::Integer(v) => Ok(Some(v as i32)), + Token::Float(v) => Ok(Some(v as i32)), + Token::Null => Ok(None), + token => Err(token.error(property, "unsigned integer")), + } + } + + pub fn assert(self, token: Token) -> trc::Result<()> { + if self == token { + Ok(()) + } else { + Err(self.error("", &token.to_string())) + } + } + + pub fn assert_jmap(self, token: Token) -> trc::Result<()> { + if self == token { + Ok(()) + } else { + Err(trc::JmapEvent::NotRequest.into_err().details(format!( + "Invalid JMAP request: expected '{token}', got '{self}'." + ))) + } + } + + pub fn error(&self, property: &str, expected: &str) -> trc::Error { + trc::JmapEvent::InvalidArguments.into_err().details(if !property.is_empty() { + format!("Invalid argument for '{property:?}': expected '{expected}', got '{self}'.",) + } else { + format!("Invalid argument: expected '{expected}', got '{self}'.") + }) + } +} + +impl Display for Ignore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "string") + } +} + +impl Display for Token { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Token::Colon => write!(f, ":"), + Token::Comma => write!(f, ","), + Token::DictStart => write!(f, "{{"), + Token::DictEnd => write!(f, "}}"), + Token::ArrayStart => write!(f, "["), + Token::ArrayEnd => write!(f, "]"), + Token::Integer(i) => write!(f, "{}", i), + Token::Float(v) => write!(f, "{}", v), + Token::Boolean(b) => write!(f, "{}", b), + Token::Null => write!(f, "null"), + Token::String(_) => write!(f, "string"), + } + } +} diff --git a/crates/utils/src/json/parser/pointer.rs b/crates/utils/src/json/parser/pointer.rs new file mode 100644 index 00000000..e50542a8 --- /dev/null +++ b/crates/utils/src/json/parser/pointer.rs @@ -0,0 +1,222 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::fmt::Display; + +use crate::json::{JsonPointer, JsonPointerItem}; + +use super::{JsonObjectParser, json::Parser}; + +enum TokenType { + Unknown, + Number, + String, + Wildcard, + Escaped, +} + +impl JsonObjectParser for JsonPointer { + fn parse(parser: &mut Parser<'_>) -> trc::Result + where + Self: Sized, + { + let mut path = Vec::new(); + let mut num = 0u64; + let mut buf = Vec::new(); + let mut token = TokenType::Unknown; + let mut start_pos = parser.pos; + + while let Some(ch) = parser.next_char() { + match (ch, &token) { + (b'0'..=b'9', TokenType::Unknown | TokenType::Number) => { + num = num.saturating_mul(10).saturating_add((ch - b'0') as u64); + token = TokenType::Number; + } + (b'*', TokenType::Unknown) => { + token = TokenType::Wildcard; + } + (b'0', TokenType::Escaped) => { + buf.push(b'~'); + token = TokenType::String; + } + (b'1', TokenType::Escaped) => { + buf.push(b'/'); + token = TokenType::String; + } + (b'/' | b'"', _) => { + match token { + TokenType::String => { + path.push(JsonPointerItem::String( + String::from_utf8(buf).map_err(|_| parser.error_utf8())?, + )); + buf = Vec::new(); + } + TokenType::Number => { + path.push(JsonPointerItem::Number(num)); + num = 0; + } + TokenType::Wildcard => { + path.push(JsonPointerItem::Wildcard); + } + TokenType::Unknown if parser.pos_marker != start_pos => { + path.push(JsonPointerItem::String(String::new())); + } + _ => (), + } + + if ch == b'/' { + token = TokenType::Unknown; + start_pos = parser.pos; + } else { + parser.is_eof = true; + + if path.is_empty() { + path.push(JsonPointerItem::Root); + } + + return Ok(JsonPointer(path)); + } + } + (_, _) => { + if matches!(&token, TokenType::Number | TokenType::Wildcard) + && parser.pos - 1 > start_pos + { + buf.extend_from_slice( + parser + .bytes + .get(start_pos..parser.pos - 1) + .unwrap_or_default(), + ); + } + + token = match ch { + b'~' if !matches!(&token, TokenType::Escaped) => TokenType::Escaped, + b'\\' => { + buf.push(parser.next_char().unwrap_or(b'\\')); + TokenType::String + } + _ => { + buf.push(ch); + TokenType::String + } + }; + } + } + } + + Err(parser.error_unterminated()) + } +} + +impl Display for JsonPointer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + for (i, ptr) in self.0.iter().enumerate() { + if i > 0 { + write!(f, "/")?; + } + write!(f, "{}", ptr)?; + } + Ok(()) + } +} + +impl Display for JsonPointerItem { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + JsonPointerItem::Root => write!(f, "/"), + JsonPointerItem::Wildcard => write!(f, "*"), + JsonPointerItem::String(s) => write!(f, "{}", s), + JsonPointerItem::Number(n) => write!(f, "{}", n), + } + } +} + +#[cfg(test)] +mod tests { + + use crate::json::parser::json::Parser; + + use super::{JsonPointer, JsonPointerItem}; + + #[test] + fn json_pointer_parse() { + for (input, output) in vec![ + ("hello", vec![JsonPointerItem::String("hello".to_string())]), + ("9a", vec![JsonPointerItem::String("9a".to_string())]), + ("a9", vec![JsonPointerItem::String("a9".to_string())]), + ("*a", vec![JsonPointerItem::String("*a".to_string())]), + ( + "/hello/world", + vec![ + JsonPointerItem::String("hello".to_string()), + JsonPointerItem::String("world".to_string()), + ], + ), + ("*", vec![JsonPointerItem::Wildcard]), + ( + "/hello/*", + vec![ + JsonPointerItem::String("hello".to_string()), + JsonPointerItem::Wildcard, + ], + ), + ("1234", vec![JsonPointerItem::Number(1234)]), + ( + "/hello/1234", + vec![ + JsonPointerItem::String("hello".to_string()), + JsonPointerItem::Number(1234), + ], + ), + ("~0~1", vec![JsonPointerItem::String("~/".to_string())]), + ( + "/hello/~0~1", + vec![ + JsonPointerItem::String("hello".to_string()), + JsonPointerItem::String("~/".to_string()), + ], + ), + ( + "/hello/1~0~1/*~1~0", + vec![ + JsonPointerItem::String("hello".to_string()), + JsonPointerItem::String("1~/".to_string()), + JsonPointerItem::String("*/~".to_string()), + ], + ), + ( + "/hello/world/*/99", + vec![ + JsonPointerItem::String("hello".to_string()), + JsonPointerItem::String("world".to_string()), + JsonPointerItem::Wildcard, + JsonPointerItem::Number(99), + ], + ), + ("/", vec![JsonPointerItem::String("".to_string())]), + ( + "///", + vec![ + JsonPointerItem::String("".to_string()), + JsonPointerItem::String("".to_string()), + JsonPointerItem::String("".to_string()), + ], + ), + ("", vec![JsonPointerItem::Root]), + ] { + assert_eq!( + Parser::new(format!("\"{input}\"").as_bytes()) + .next_token::() + .unwrap() + .unwrap_string("") + .unwrap() + .0, + output, + "{input}" + ); + } + } +} diff --git a/crates/utils/src/json/pointer.rs b/crates/utils/src/json/pointer.rs new file mode 100644 index 00000000..a23ade97 --- /dev/null +++ b/crates/utils/src/json/pointer.rs @@ -0,0 +1,106 @@ +use super::{JsonPointerItem, JsonQueryable}; +use std::hash::BuildHasher; +use std::{collections::HashMap, slice::Iter}; + +impl JsonQueryable for Vec { + fn eval_pointer<'x>( + &'x self, + mut pointer: Iter, + results: &mut Vec<&'x dyn JsonQueryable>, + ) { + match pointer.next() { + Some(JsonPointerItem::Number(n)) => { + if let Some(v) = self.get(*n as usize) { + v.eval_pointer(pointer, results); + } + } + Some(JsonPointerItem::Wildcard) => { + for v in self { + v.eval_pointer(pointer.clone(), results); + } + } + Some(JsonPointerItem::Root) | None => { + results.push(self); + } + _ => {} + } + } +} + +impl JsonQueryable for HashMap { + fn eval_pointer<'x>( + &'x self, + mut pointer: Iter, + results: &mut Vec<&'x dyn JsonQueryable>, + ) { + match pointer.next() { + Some(JsonPointerItem::String(n)) => { + if let Some(v) = self.get(n) { + v.eval_pointer(pointer, results); + } + } + Some(JsonPointerItem::Number(n)) => { + let n = n.to_string(); + if let Some(v) = self.get(&n) { + v.eval_pointer(pointer, results); + } + } + Some(JsonPointerItem::Wildcard) => { + for v in self.values() { + v.eval_pointer(pointer.clone(), results); + } + } + Some(JsonPointerItem::Root) | None => { + results.push(self); + } + } + } +} + +impl JsonQueryable for serde_json::Value { + fn eval_pointer<'x>( + &'x self, + mut pointer: Iter, + results: &mut Vec<&'x dyn JsonQueryable>, + ) { + match pointer.next() { + Some(JsonPointerItem::String(n)) => { + if let serde_json::Value::Object(map) = self { + if let Some(v) = map.get(n) { + v.eval_pointer(pointer, results); + } + } + } + Some(JsonPointerItem::Number(n)) => match self { + serde_json::Value::Array(values) => { + if let Some(v) = values.get(*n as usize) { + v.eval_pointer(pointer, results); + } + } + serde_json::Value::Object(map) => { + let n = n.to_string(); + if let Some(v) = map.get(&n) { + v.eval_pointer(pointer, results); + } + } + _ => {} + }, + Some(JsonPointerItem::Wildcard) => match self { + serde_json::Value::Array(values) => { + for v in values { + v.eval_pointer(pointer.clone(), results); + } + } + serde_json::Value::Object(map) => { + for v in map.values() { + v.eval_pointer(pointer.clone(), results); + } + } + _ => {} + }, + Some(JsonPointerItem::Root) | None => { + results.push(self); + } + } + } +} diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index acec2f04..f6c6bc4f 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -10,6 +10,7 @@ pub mod cache; pub mod codec; pub mod config; pub mod glob; +pub mod json; pub mod map; pub mod snowflake; pub mod url_params; @@ -17,11 +18,14 @@ pub mod url_params; use futures::StreamExt; use reqwest::Response; use rustls::{ - client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}, ClientConfig, RootCertStore, SignatureScheme, + client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}, }; use rustls_pki_types::TrustAnchor; +pub use downcast_rs; +pub use erased_serde; + pub const BLOB_HASH_LEN: usize = 32; #[derive(Clone, Debug, Default, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] @@ -247,7 +251,7 @@ pub fn failed(message: &str) -> ! { pub async fn wait_for_shutdown() { #[cfg(not(target_env = "msvc"))] let signal = { - use tokio::signal::unix::{signal, SignalKind}; + use tokio::signal::unix::{SignalKind, signal}; let mut h_term = signal(SignalKind::terminate()).failed("start signal handler"); let mut h_int = signal(SignalKind::interrupt()).failed("start signal handler"); diff --git a/crates/utils/src/map/vec_map.rs b/crates/utils/src/map/vec_map.rs index b2b7c9b5..5d49498b 100644 --- a/crates/utils/src/map/vec_map.rs +++ b/crates/utils/src/map/vec_map.rs @@ -6,7 +6,7 @@ use std::{borrow::Borrow, cmp::Ordering, fmt, hash::Hash}; -use serde::{de::DeserializeOwned, ser::SerializeMap, Deserialize, Serialize}; +use serde::{Deserialize, Serialize, de::DeserializeOwned, ser::SerializeMap}; // A map implemented using vectors // used for small datasets of less than 20 items @@ -56,6 +56,12 @@ impl VecMap { self.inner.push(KeyValue { key, value }); } + #[inline(always)] + pub fn with_append(mut self, key: K, value: V) -> Self { + self.append(key, value); + self + } + #[inline(always)] pub fn insert(&mut self, idx: usize, key: K, value: V) { self.inner.insert(idx, KeyValue { key, value }); diff --git a/tests/src/jmap/crypto.rs b/tests/src/jmap/crypto.rs index 3e828d02..35db3c7c 100644 --- a/tests/src/jmap/crypto.rs +++ b/tests/src/jmap/crypto.rs @@ -6,15 +6,15 @@ use std::path::PathBuf; -use email::crypto::{ - try_parse_certs, Algorithm, EncryptMessage, EncryptionMethod, EncryptionParams, EncryptionType, +use email::message::crypto::{ + Algorithm, EncryptMessage, EncryptionMethod, EncryptionParams, EncryptionType, try_parse_certs, }; use jmap_proto::types::id::Id; use mail_parser::{MessageParser, MimeHeaders}; use crate::{ directory::internal::TestInternalDirectory, - jmap::{delivery::SmtpConnection, ManagementApi}, + jmap::{ManagementApi, delivery::SmtpConnection}, }; use super::JMAPTest; @@ -218,17 +218,19 @@ pub async fn import_certs_and_encrypt() { } // S/MIME and PGP should not be allowed mixed - assert!(try_parse_certs( - EncryptionMethod::PGP, - std::fs::read( - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("resources") - .join("crypto") - .join("cert_mixed.pem"), + assert!( + try_parse_certs( + EncryptionMethod::PGP, + std::fs::read( + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("resources") + .join("crypto") + .join("cert_mixed.pem"), + ) + .unwrap(), ) - .unwrap(), - ) - .is_err()); + .is_err() + ); } #[test] diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index f44f0781..484498c2 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -23,10 +23,11 @@ use common::{ config::{ConfigManager, Patterns}, }, }; +use email::message::delete::EmailDeletion; use enterprise::{EnterpriseCore, insert_test_metrics}; use hyper::{Method, header::AUTHORIZATION}; use imap::core::ImapSessionManager; -use jmap::{SpawnServices, api::JmapSessionManager, email::delete::EmailDeletion}; +use jmap::{SpawnServices, api::JmapSessionManager}; use jmap_client::client::{Client, Credentials}; use jmap_proto::{error::request::RequestError, types::id::Id}; use managesieve::core::ManageSieveSessionManager; diff --git a/tests/src/jmap/permissions.rs b/tests/src/jmap/permissions.rs index a4b8015c..bf68b0de 100644 --- a/tests/src/jmap/permissions.rs +++ b/tests/src/jmap/permissions.rs @@ -9,15 +9,15 @@ use std::sync::Arc; use ahash::AHashSet; use common::auth::{AccessToken, TenantInfo}; use directory::{ - backend::internal::{PrincipalField, PrincipalUpdate, PrincipalValue}, Permission, Principal, Type, + backend::internal::{PrincipalField, PrincipalUpdate, PrincipalValue}, }; -use email::delivery::{IngestMessage, LocalDeliveryStatus, MailDelivery}; +use email::message::delivery::{IngestMessage, LocalDeliveryStatus, MailDelivery}; use utils::BlobHash; use crate::jmap::assert_is_empty; -use super::{enterprise::List, JMAPTest, ManagementApi}; +use super::{JMAPTest, ManagementApi, enterprise::List}; pub async fn test(params: &JMAPTest) { println!("Running permissions tests..."); diff --git a/tests/src/jmap/purge.rs b/tests/src/jmap/purge.rs index ad628c13..7db8a817 100644 --- a/tests/src/jmap/purge.rs +++ b/tests/src/jmap/purge.rs @@ -6,14 +6,16 @@ use ahash::AHashSet; use common::Server; -use directory::{backend::internal::manage::ManageDirectory, QueryBy}; -use email::mailbox::{INBOX_ID, JUNK_ID, TRASH_ID}; +use directory::{QueryBy, backend::internal::manage::ManageDirectory}; +use email::{ + mailbox::{INBOX_ID, JUNK_ID, TRASH_ID}, + message::delete::EmailDeletion, +}; use imap_proto::ResponseType; -use jmap::email::delete::EmailDeletion; use jmap_proto::types::{collection::Collection, id::Id, property::Property}; use store::{ - write::{key::DeserializeBigEndian, TagValue}, IterateParams, LogKey, U32_LEN, U64_LEN, + write::{TagValue, key::DeserializeBigEndian}, }; use crate::{ diff --git a/tests/src/jmap/thread_merge.rs b/tests/src/jmap/thread_merge.rs index d7c50383..82feb250 100644 --- a/tests/src/jmap/thread_merge.rs +++ b/tests/src/jmap/thread_merge.rs @@ -12,10 +12,10 @@ use crate::{ }; use common::auth::AccessToken; -use ::email::ingest::{EmailIngest, IngestEmail, IngestSource}; +use ::email::message::ingest::{EmailIngest, IngestEmail, IngestSource}; use jmap_client::{email, mailbox::Role}; use jmap_proto::types::{collection::Collection, id::Id}; -use mail_parser::{mailbox::mbox::MessageIterator, MessageParser}; +use mail_parser::{MessageParser, mailbox::mbox::MessageIterator}; use store::{ ahash::{AHashMap, AHashSet}, rand::{self, Rng},