From e69b5ec2b7c99e6795c6eee4472fb6734c60063c Mon Sep 17 00:00:00 2001 From: mdecimus Date: Tue, 6 May 2025 16:00:16 +0200 Subject: [PATCH] Message caching improvements --- Cargo.lock | 445 +++++++------- crates/common/src/config/inner.rs | 15 +- crates/common/src/config/jmap/settings.rs | 6 +- crates/common/src/core.rs | 37 +- crates/common/src/lib.rs | 31 +- crates/common/src/storage/index.rs | 73 ++- crates/dav/src/common/propfind.rs | 7 +- crates/dav/src/principal/propfind.rs | 1 + crates/email/src/cache/email.rs | 321 ++++++++++ crates/email/src/cache/mailbox.rs | 296 +++++++++ crates/email/src/cache/mod.rs | 181 ++++++ crates/email/src/identity/index.rs | 13 +- crates/email/src/lib.rs | 1 + crates/email/src/mailbox/cache.rs | 371 ------------ crates/email/src/mailbox/destroy.rs | 31 +- crates/email/src/mailbox/index.rs | 10 +- crates/email/src/mailbox/manage.rs | 13 +- crates/email/src/mailbox/mod.rs | 1 - crates/email/src/message/cache.rs | 431 ------------- crates/email/src/message/copy.rs | 11 +- crates/email/src/message/delete.rs | 18 +- crates/email/src/message/index.rs | 37 +- crates/email/src/message/ingest.rs | 51 +- crates/email/src/message/mod.rs | 1 - crates/email/src/sieve/index.rs | 12 +- crates/email/src/sieve/ingest.rs | 39 +- crates/email/src/submission/index.rs | 12 +- crates/groupware/src/calendar/index.rs | 20 +- crates/groupware/src/contact/index.rs | 20 +- crates/groupware/src/file/index.rs | 10 +- crates/groupware/src/hierarchy.rs | 2 + crates/imap/src/core/mailbox.rs | 48 +- crates/imap/src/core/message.rs | 22 +- crates/imap/src/core/mod.rs | 8 +- crates/imap/src/op/create.rs | 10 +- crates/imap/src/op/expunge.rs | 20 +- crates/imap/src/op/fetch.rs | 29 +- crates/imap/src/op/idle.rs | 29 +- crates/imap/src/op/search.rs | 32 +- crates/imap/src/op/status.rs | 11 +- crates/imap/src/op/store.rs | 29 +- crates/imap/src/op/thread.rs | 7 +- crates/jmap-proto/src/types/collection.rs | 41 +- crates/jmap-proto/src/types/type_state.rs | 34 +- crates/jmap/src/blob/download.rs | 27 +- crates/jmap/src/changes/get.rs | 166 +++-- crates/jmap/src/changes/state.rs | 43 +- crates/jmap/src/email/copy.rs | 42 +- crates/jmap/src/email/get.rs | 48 +- crates/jmap/src/email/import.rs | 25 +- crates/jmap/src/email/query.rs | 30 +- crates/jmap/src/email/set.rs | 44 +- crates/jmap/src/email/snippet.rs | 25 +- crates/jmap/src/identity/get.rs | 4 +- crates/jmap/src/identity/set.rs | 4 +- crates/jmap/src/lib.rs | 27 +- crates/jmap/src/mailbox/get.rs | 27 +- crates/jmap/src/mailbox/query.rs | 31 +- crates/jmap/src/mailbox/set.rs | 24 +- crates/jmap/src/principal/query.rs | 6 +- crates/jmap/src/sieve/get.rs | 4 +- crates/jmap/src/sieve/query.rs | 16 +- crates/jmap/src/sieve/set.rs | 14 +- crates/jmap/src/submission/get.rs | 4 +- crates/jmap/src/submission/query.rs | 16 +- crates/jmap/src/thread/get.rs | 18 +- crates/jmap/src/vacation/get.rs | 4 +- crates/jmap/src/vacation/set.rs | 18 +- crates/pop3/src/mailbox.rs | 34 +- crates/store/src/query/log.rs | 210 ++++++- crates/store/src/write/batch.rs | 142 +++-- crates/store/src/write/log.rs | 145 +++-- crates/store/src/write/mod.rs | 9 +- tests/resources/jmap/email_set/headers.eml | 8 +- tests/resources/jmap/email_set/headers.jmap | 4 +- tests/resources/jmap/email_set/mixed.eml | 4 +- tests/resources/jmap/email_set/mixed.jmap | 2 +- tests/src/imap/basic.rs | 2 +- tests/src/imap/mod.rs | 633 ++++++++++---------- tests/src/jmap/delivery.rs | 17 +- tests/src/jmap/email_changes.rs | 26 +- tests/src/jmap/email_query.rs | 8 +- tests/src/jmap/email_query_changes.rs | 8 +- tests/src/jmap/mod.rs | 571 +++++++++--------- tests/src/jmap/purge.rs | 28 +- tests/src/jmap/stress_test.rs | 9 +- tests/src/store/import_export.rs | 7 +- 87 files changed, 2835 insertions(+), 2536 deletions(-) create mode 100644 crates/email/src/cache/email.rs create mode 100644 crates/email/src/cache/mailbox.rs create mode 100644 crates/email/src/cache/mod.rs delete mode 100644 crates/email/src/mailbox/cache.rs delete mode 100644 crates/email/src/message/cache.rs diff --git a/Cargo.lock b/Cargo.lock index 1fb64e18..d169c820 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -86,7 +86,7 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", "once_cell", "version_check", ] @@ -98,7 +98,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", - "getrandom 0.2.15", + "getrandom 0.2.16", "once_cell", "serde", "version_check", @@ -291,8 +291,8 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", - "synstructure 0.13.1", + "syn 2.0.101", + "synstructure 0.13.2", ] [[package]] @@ -314,7 +314,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -330,9 +330,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.22" +version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59a194f9d963d8099596278594b3107448656ba73831c9d8c783e613ce86da64" +checksum = "b37fc50485c4f3f736a4fb14199f6d5f5ba008d7f28fe710306c92780f004c07" dependencies = [ "flate2", "futures-core", @@ -360,7 +360,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -382,7 +382,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -393,7 +393,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -410,11 +410,11 @@ checksum = "07a9b245ba0739fc90935094c29adbaee3f977218b5fb95e822e261cda7f56a3" dependencies = [ "http 1.3.1", "log", - "rustls 0.23.26", + "rustls 0.23.27", "serde", "serde_json", "url", - "webpki-roots 0.26.8", + "webpki-roots 0.26.10", ] [[package]] @@ -507,7 +507,7 @@ dependencies = [ "bytes", "dyn-clone", "futures", - "getrandom 0.2.15", + "getrandom 0.2.16", "hmac 0.12.1", "http-types", "once_cell", @@ -519,7 +519,7 @@ dependencies = [ "rustc_version 0.4.1", "serde", "serde_json", - "sha2 0.10.8", + "sha2 0.10.9", "time", "tracing", "url", @@ -678,7 +678,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -698,7 +698,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -716,7 +716,7 @@ dependencies = [ "regex", "rustc-hash 2.1.1", "shlex", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -819,9 +819,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.1" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389a099b34312839e16420d499a9cad9650541715937ffbdd40d36f49e77eeb3" +checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" dependencies = [ "arrayref", "arrayvec", @@ -898,7 +898,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -967,14 +967,14 @@ checksum = "efb7846e0cb180355c2dec69e721edafa36919850f1a9f52ffba4ebc0393cb71" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] name = "bytemuck" -version = "1.22.0" +version = "1.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6b1fc10dbac614ebc03540c9dbd60e83887fda27794998c6528f1782047d540" +checksum = "9134a6ef01ce4b366b50689c94f82c14bc72bc5d0386829828a2e2752ef7958c" [[package]] name = "byteorder" @@ -1060,9 +1060,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.19" +version = "1.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362" +checksum = "8691782945451c1c383942c4874dbe63814f61cb57ef773cda2972682b7bb3c0" dependencies = [ "jobserver", "libc", @@ -1127,9 +1127,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.40" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1194,9 +1194,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.36" +version = "4.5.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2df961d8c8a0d08aa9945718ccf584145eee3f3aa06cddbeac12933781102e04" +checksum = "eccb054f56cbd38340b380d4a8e69ef1f02f1af43db2f0cc817a4774d80ae071" dependencies = [ "clap_builder", "clap_derive", @@ -1204,9 +1204,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.36" +version = "4.5.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "132dbda40fb6753878316a489d5a1242a8ef2f0d9e47ba01c951ea8aa7d013a5" +checksum = "efd9466fac8543255d3b1fcad4762c5e116ffe808c8a3043d4263cd4fd4862a2" dependencies = [ "anstream", "anstyle", @@ -1223,7 +1223,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -1325,13 +1325,13 @@ dependencies = [ "ring 0.17.14", "rkyv 0.8.10", "rsa", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", "serde_json", "sha1", - "sha2 0.10.8", + "sha2 0.10.9", "sieve-rs", "smtp-proto", "store", @@ -1406,7 +1406,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", "once_cell", "tiny-keccak", ] @@ -1646,7 +1646,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -1694,7 +1694,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -1716,7 +1716,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -1762,7 +1762,7 @@ dependencies = [ "hashify", "hyper 1.6.0", "mail-parser", - "quick-xml 0.37.4", + "quick-xml 0.37.5", "rkyv 0.8.10", "serde", "serde_json", @@ -1808,7 +1808,7 @@ checksum = "3d697d376cbfa018c23eb4caab1fd1883dd9c906a8c034e8d9a3cb06a7e0bef9" dependencies = [ "async-trait", "deadpool 0.12.2", - "getrandom 0.2.15", + "getrandom 0.2.16", "tokio", "tokio-postgres", "tracing", @@ -1842,9 +1842,9 @@ checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b" [[package]] name = "der" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ "const-oid", "pem-rfc7468", @@ -1897,7 +1897,7 @@ checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -1955,13 +1955,13 @@ dependencies = [ "regex", "reqwest 0.12.15", "rkyv 0.8.10", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pki-types", "scrypt", "serde", "serde_json", "sha1", - "sha2 0.10.8", + "sha2 0.10.9", "smtp-proto", "store", "tokio", @@ -2000,7 +2000,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2049,7 +2049,7 @@ dependencies = [ "num-traits", "pkcs8", "rfc6979", - "sha2 0.10.8", + "sha2 0.10.9", "signature", "zeroize", ] @@ -2110,7 +2110,7 @@ dependencies = [ "once_cell", "openssl", "serde", - "sha2 0.10.8", + "sha2 0.10.9", "thiserror 1.0.69", ] @@ -2134,7 +2134,7 @@ dependencies = [ "ed25519", "rand_core 0.6.4", "serde", - "sha2 0.10.8", + "sha2 0.10.9", "subtle", "zeroize", ] @@ -2260,7 +2260,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2322,7 +2322,7 @@ version = "0.11.8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2499,7 +2499,7 @@ checksum = "9be610412e5a92d89855fb15b099a57792b7dbdcf8ac74c5a0e24d9b7b1b6f7f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", "try_map", ] @@ -2552,7 +2552,7 @@ checksum = "e99b8b3c28ae0e84b604c75f721c21dc77afb3706076af5e8216d15fd1deaae3" dependencies = [ "frunk_proc_macro_helpers", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2564,7 +2564,7 @@ dependencies = [ "frunk_core", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2576,7 +2576,7 @@ dependencies = [ "frunk_core", "frunk_proc_macro_helpers", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2656,7 +2656,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2730,11 +2730,11 @@ dependencies = [ [[package]] name = "gethostname" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed7131e57abbde63513e0e6636f76668a1ca9798dcae2df4e283cae9ee83859e" +checksum = "fc257fdb4038301ce4b9cd1b3b51704509692bb3ff716a410cbd07925d9dae55" dependencies = [ - "rustix 1.0.5", + "rustix 1.0.7", "windows-targets 0.52.6", ] @@ -2751,9 +2751,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "js-sys", @@ -2859,9 +2859,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75249d144030531f8dee69fe9cea04d3edf809a017ae445e2abdff6629e86633" +checksum = "a9421a676d1b147b16b82c9225157dc629087ef8ec4d5e2960f9437a90dac0a5" dependencies = [ "atomic-waker", "bytes", @@ -2897,9 +2897,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.2" +version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3" dependencies = [ "allocator-api2", "equivalent", @@ -2914,7 +2914,7 @@ checksum = "f208758247e68e239acaa059e72e4ce1f30f2a4b6523f19c1b923d25b7e9cceb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2923,7 +2923,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" dependencies = [ - "hashbrown 0.15.2", + "hashbrown 0.15.3", ] [[package]] @@ -2946,9 +2946,9 @@ checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "hermit-abi" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbd780fe5cc30f81464441920d82ac8740e2e46b29a6fad543ddd075229ce37e" +checksum = "f154ce46856750ed433c8649605bf7ed2de3bc35fd9d2a9f30cddd873c80cb08" [[package]] name = "hex" @@ -3115,7 +3115,7 @@ dependencies = [ "mail-send", "mime", "pkcs8", - "quick-xml 0.37.4", + "quick-xml 0.37.5", "rev_lines", "rkyv 0.8.10", "rsa", @@ -3123,7 +3123,7 @@ dependencies = [ "serde_json", "services", "sha1", - "sha2 0.10.8", + "sha2 0.10.9", "smtp", "smtp-proto", "spam-filter", @@ -3266,7 +3266,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.9", + "h2 0.4.10", "http 1.3.1", "http-body 1.0.1", "httparse", @@ -3302,12 +3302,12 @@ dependencies = [ "http 1.3.1", "hyper 1.6.0", "hyper-util", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pki-types", "tokio", "tokio-rustls 0.26.2", "tower-service", - "webpki-roots 0.26.8", + "webpki-roots 0.26.10", ] [[package]] @@ -3482,7 +3482,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -3545,7 +3545,7 @@ dependencies = [ "nlp", "parking_lot", "rand 0.9.1", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pemfile 2.2.0", "store", "tokio", @@ -3589,7 +3589,7 @@ dependencies = [ "libflate", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -3609,7 +3609,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", - "hashbrown 0.15.2", + "hashbrown 0.15.3", ] [[package]] @@ -3683,7 +3683,7 @@ version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" dependencies = [ - "hermit-abi 0.5.0", + "hermit-abi 0.5.1", "libc", "windows-sys 0.59.0", ] @@ -3826,7 +3826,7 @@ dependencies = [ "serde_json", "services", "sha1", - "sha2 0.10.8", + "sha2 0.10.9", "sieve-rs", "smtp", "smtp-proto", @@ -4052,9 +4052,9 @@ dependencies = [ [[package]] name = "libm" -version = "0.2.11" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" +checksum = "a25169bd5913a4b437588a7e3d127cd6e90127b60e0ffbd834a38f1599e016b8" [[package]] name = "libredox" @@ -4137,12 +4137,6 @@ dependencies = [ "scopeguard", ] -[[package]] -name = "lockfree-object-pool" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" - [[package]] name = "log" version = "0.4.27" @@ -4155,7 +4149,7 @@ version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown 0.15.2", + "hashbrown 0.15.3", ] [[package]] @@ -4215,7 +4209,7 @@ dependencies = [ "hickory-resolver", "mail-builder", "mail-parser", - "quick-xml 0.37.4", + "quick-xml 0.37.5", "quick_cache", "rand 0.8.5", "ring 0.17.14", @@ -4228,11 +4222,11 @@ dependencies = [ [[package]] name = "mail-builder" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5666db60ccc742381c715c8daf303e60d266f1d1c8b123ab3fe78b590edaf564" +checksum = "0926cff74776d4af100a95c90a6649486659526ce638bee6648ecc9c41051810" dependencies = [ - "gethostname 1.0.1", + "gethostname 1.0.2", ] [[package]] @@ -4255,12 +4249,12 @@ dependencies = [ "base64 0.22.1", "gethostname 0.5.0", "md5", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pki-types", "smtp-proto", "tokio", "tokio-rustls 0.26.2", - "webpki-roots 0.26.8", + "webpki-roots 0.26.10", ] [[package]] @@ -4305,7 +4299,7 @@ dependencies = [ "mail-send", "md5", "parking_lot", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pemfile 2.2.0", "sieve-rs", "store", @@ -4335,7 +4329,7 @@ checksum = "5cf92c10c7e361d6b99666ec1c6f9805b0bea2c3bd8c78dc6fe98ac5bd78db11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -4432,22 +4426,22 @@ dependencies = [ [[package]] name = "munge" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0091202c98cf06da46c279fdf50cccb6b1c43b4521abdf6a27b4c7e71d5d9d7" +checksum = "9e22e7961c873e8b305b176d2a4e1d41ce7ba31bc1c52d2a107a89568ec74c55" dependencies = [ "munge_macro", ] [[package]] name = "munge_macro" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "734799cf91479720b2f970c61a22850940dd91e27d4f02b1c6fc792778df2459" +checksum = "0ac7d860b767c6398e88fe93db73ce53eb496057aa6895ffa4d60cb02e1d1c6b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -4463,7 +4457,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", "termcolor", "thiserror 1.0.69", ] @@ -4502,7 +4496,7 @@ dependencies = [ "twox-hash", "url", "webpki", - "webpki-roots 0.26.8", + "webpki-roots 0.26.10", ] [[package]] @@ -4535,7 +4529,7 @@ dependencies = [ "serde", "serde_json", "sha1", - "sha2 0.10.8", + "sha2 0.10.9", "smallvec", "subprocess", "thiserror 1.0.69", @@ -4752,7 +4746,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -4763,9 +4757,9 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-sys" -version = "0.9.107" +version = "0.9.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8288979acd84749c744a9014b4382d42b8f7b2592847b5afb2ed29e5d16ede07" +checksum = "e145e1651e858e820e4860f7b9c5e169bc1d8ce1c86043be79fa7b7634821847" dependencies = [ "cc", "libc", @@ -4876,7 +4870,7 @@ dependencies = [ "ecdsa", "elliptic-curve", "primeorder", - "sha2 0.10.8", + "sha2 0.10.9", ] [[package]] @@ -4888,7 +4882,7 @@ dependencies = [ "ecdsa", "elliptic-curve", "primeorder", - "sha2 0.10.8", + "sha2 0.10.9", ] [[package]] @@ -4902,7 +4896,7 @@ dependencies = [ "elliptic-curve", "primeorder", "rand_core 0.6.4", - "sha2 0.10.8", + "sha2 0.10.9", ] [[package]] @@ -4969,7 +4963,7 @@ dependencies = [ "digest 0.10.7", "hmac 0.12.1", "password-hash", - "sha2 0.10.8", + "sha2 0.10.9", ] [[package]] @@ -5062,7 +5056,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -5127,7 +5121,7 @@ dependencies = [ "jmap_proto", "mail-parser", "mail-send", - "rustls 0.23.26", + "rustls 0.23.27", "store", "tokio", "tokio-rustls 0.26.2", @@ -5155,7 +5149,7 @@ dependencies = [ "md-5 0.10.6", "memchr", "rand 0.9.1", - "sha2 0.10.8", + "sha2 0.10.9", "stringprep", ] @@ -5182,7 +5176,7 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy 0.8.24", + "zerocopy 0.8.25", ] [[package]] @@ -5198,7 +5192,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" dependencies = [ "proc-macro2", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -5262,7 +5256,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -5280,7 +5274,7 @@ version = "0.11.8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -5317,7 +5311,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -5332,9 +5326,9 @@ dependencies = [ [[package]] name = "psl" -version = "2.1.101" +version = "2.1.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34355eb8747f514f2e0a607c8806226de08cfac767c3e1adc777021e07991e44" +checksum = "ed067c32eda3664a59207dde92e8f895016fb375564d91626591036c741a3a89" dependencies = [ "psl-types", ] @@ -5382,7 +5376,7 @@ checksum = "ca414edb151b4c8d125c12566ab0d74dc9cdba36fb80eb7b848c15f495fd32d1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -5422,9 +5416,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.37.4" +version = "0.37.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4ce8c88de324ff838700f36fb6ab86c96df0e3c4ab6ef3a9b2044465cce1369" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" dependencies = [ "memchr", ] @@ -5437,7 +5431,7 @@ checksum = "287e56aac5a2b4fb25a6fb050961d157635924c8696305a5c937a76f29841a0f" dependencies = [ "ahash 0.8.11", "equivalent", - "hashbrown 0.15.2", + "hashbrown 0.15.3", "parking_lot", ] @@ -5453,7 +5447,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.1", - "rustls 0.23.26", + "rustls 0.23.27", "socket2", "thiserror 2.0.12", "tokio", @@ -5463,16 +5457,16 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.10" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b820744eb4dc9b57a3398183639c511b5a26d2ed702cedd3febaa1393caa22cc" +checksum = "bcbafbbdbb0f638fe3f35f3c56739f77a8a1d070cb25603226c83339b391472b" dependencies = [ "bytes", "getrandom 0.3.2", "rand 0.9.1", "ring 0.17.14", "rustc-hash 2.1.1", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pki-types", "slab", "thiserror 2.0.12", @@ -5483,9 +5477,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.11" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "541d0f57c6ec747a90738a52741d3221f7960e8ac2f0ff4b1a63680e033b4ab5" +checksum = "ee4e529991f949c5e25755532370b8af5d114acae52326361d68d47af64aa842" dependencies = [ "cfg_aliases 0.2.1", "libc", @@ -5625,7 +5619,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", ] [[package]] @@ -5765,7 +5759,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rand 0.9.1", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-native-certs 0.8.1", "ryu", "sha1_smol", @@ -5774,14 +5768,14 @@ dependencies = [ "tokio-rustls 0.26.2", "tokio-util", "url", - "webpki-roots 0.26.8", + "webpki-roots 0.26.10", ] [[package]] name = "redox_syscall" -version = "0.5.11" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3" +checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af" dependencies = [ "bitflags 2.9.0", ] @@ -5792,7 +5786,7 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", "libredox", "thiserror 1.0.69", ] @@ -5899,7 +5893,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.4.9", + "h2 0.4.10", "http 1.3.1", "http-body 1.0.1", "http-body-util", @@ -5915,7 +5909,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", @@ -5932,18 +5926,15 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 0.26.8", + "webpki-roots 0.26.10", "windows-registry", ] [[package]] name = "resolv-conf" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48375394603e3dd4b2d64371f7148fd8c7baa2680e28741f2cb8d23b59e3d4c4" -dependencies = [ - "hostname", -] +checksum = "fc7c8f7f733062b66dc1c63f9db168ac0b97a9210e247fa90fdc9ad08f51b302" [[package]] name = "rev_lines" @@ -5987,7 +5978,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.15", + "getrandom 0.2.16", "libc", "untrusted 0.9.0", "windows-sys 0.52.0", @@ -6028,7 +6019,7 @@ checksum = "1e147371c75553e1e2fcdb483944a8540b8438c31426279553b9a8182a9b7b65" dependencies = [ "bytecheck 0.8.1", "bytes", - "hashbrown 0.15.2", + "hashbrown 0.15.3", "indexmap 2.9.0", "munge", "ptr_meta 0.3.0", @@ -6058,7 +6049,7 @@ checksum = "246b40ac189af6c675d124b802e8ef6d5246c53e17367ce9501f8f66a81abb7a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -6089,13 +6080,13 @@ dependencies = [ [[package]] name = "rpassword" -version = "7.3.1" +version = "7.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80472be3c897911d0137b2d2b9055faf6eeac5b14e324073d83bc17b191d7e3f" +checksum = "66d4c8b64f049c6721ec8ccec37ddfc3d641c4a7fca57e8f2a89de509c73df39" dependencies = [ "libc", "rtoolbox", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -6120,12 +6111,12 @@ dependencies = [ [[package]] name = "rtoolbox" -version = "0.0.2" +version = "0.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c247d24e63230cdb56463ae328478bd5eac8b8faa8c69461a77e8e323afac90e" +checksum = "a7cc970b249fbe527d6e02e0a227762c9108b2f49d81094fe357ffc6d14d7f6f" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -6187,7 +6178,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "sha2 0.10.8", + "sha2 0.10.9", "thiserror 1.0.69", "time", "tokio", @@ -6282,9 +6273,9 @@ dependencies = [ [[package]] name = "rustix" -version = "1.0.5" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" +checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" dependencies = [ "bitflags 2.9.0", "errno", @@ -6321,14 +6312,14 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.26" +version = "0.23.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df51b5869f3a441595eac5e8ff14d486ff285f7b8c0df8770e49c3b56351f0f0" +checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321" dependencies = [ "once_cell", "ring 0.17.14", "rustls-pki-types", - "rustls-webpki 0.103.1", + "rustls-webpki 0.103.2", "subtle", "zeroize", ] @@ -6407,9 +6398,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.1" +version = "0.103.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03" +checksum = "7149975849f1abb3832b246010ef62ccc80d3a76169517ada7188252b9cfb437" dependencies = [ "ring 0.17.14", "rustls-pki-types", @@ -6454,9 +6445,9 @@ checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71" [[package]] name = "scc" -version = "2.3.3" +version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea091f6cac2595aa38993f04f4ee692ed43757035c36e67c180b6828356385b1" +checksum = "22b2d775fb28f245817589471dd49c5edf64237f4a19d10ce9a92ff4651a27f4" dependencies = [ "sdd", ] @@ -6494,7 +6485,7 @@ dependencies = [ "password-hash", "pbkdf2", "salsa20", - "sha2 0.10.8", + "sha2 0.10.9", ] [[package]] @@ -6617,7 +6608,7 @@ dependencies = [ "ecdsa", "ed25519", "ed25519-dalek", - "getrandom 0.2.15", + "getrandom 0.2.16", "idea", "idna", "lalrpop", @@ -6638,7 +6629,7 @@ dependencies = [ "ripemd", "rsa", "sha1collisiondetection", - "sha2 0.10.8", + "sha2 0.10.9", "thiserror 1.0.69", "twofish", "typenum", @@ -6672,7 +6663,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -6755,7 +6746,7 @@ checksum = "5d69265a08751de7844521fd15003ae0a888e035773ba05695c5c759a6f89eef" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -6778,7 +6769,7 @@ dependencies = [ "rsa", "serde", "serde_json", - "sha2 0.10.8", + "sha2 0.10.9", "smtp", "store", "tokio", @@ -6842,9 +6833,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.8" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures", @@ -6874,9 +6865,9 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.2" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" dependencies = [ "libc", ] @@ -6954,13 +6945,13 @@ dependencies = [ "regex", "reqwest 0.12.15", "rkyv 0.8.10", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", "serde_json", "sha1", - "sha2 0.10.8", + "sha2 0.10.9", "sieve-rs", "smtp-proto", "spam-filter", @@ -6969,7 +6960,7 @@ dependencies = [ "tokio-rustls 0.26.2", "trc", "utils", - "webpki-roots 0.26.8", + "webpki-roots 0.26.10", "x509-parser 0.16.0", ] @@ -7033,7 +7024,7 @@ dependencies = [ "psl", "reqwest 0.12.15", "sha1", - "sha2 0.10.8", + "sha2 0.10.9", "smtp-proto", "store", "tokio", @@ -7143,7 +7134,7 @@ dependencies = [ "rocksdb", "rusqlite", "rust-s3", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pki-types", "serde", "serde_json", @@ -7219,9 +7210,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.100" +version = "2.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" dependencies = [ "proc-macro2", "quote", @@ -7257,13 +7248,13 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -7354,12 +7345,12 @@ dependencies = [ "nlp", "num_cpus", "pop3", - "quick-xml 0.37.4", + "quick-xml 0.37.5", "rayon", "reqwest 0.12.15", "ring 0.17.14", "rkyv 0.8.10", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", @@ -7403,7 +7394,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -7414,7 +7405,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -7510,7 +7501,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -7566,7 +7557,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.23.26", + "rustls 0.23.27", "tokio", ] @@ -7594,7 +7585,7 @@ dependencies = [ "tokio", "tokio-rustls 0.25.0", "tungstenite 0.21.0", - "webpki-roots 0.26.8", + "webpki-roots 0.26.10", ] [[package]] @@ -7611,9 +7602,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" dependencies = [ "bytes", "futures-core", @@ -7624,15 +7615,15 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.8" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +checksum = "3da5db5a963e24bc68be8b17b6fa82814bb22ee8660f192bb182771d498f09a3" [[package]] name = "toml_edit" -version = "0.22.24" +version = "0.22.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" +checksum = "310068873db2c5b3e7659d2cc35d21855dbafa50d1ce336397c666e3cb08137e" dependencies = [ "indexmap 2.9.0", "toml_datetime", @@ -7650,7 +7641,7 @@ dependencies = [ "axum", "base64 0.22.1", "bytes", - "h2 0.4.9", + "h2 0.4.10", "http 1.3.1", "http-body 1.0.1", "http-body-util", @@ -7679,7 +7670,7 @@ dependencies = [ "constant_time_eq", "hmac 0.12.1", "sha1", - "sha2 0.10.8", + "sha2 0.10.9", "url", "urlencoding", ] @@ -7750,7 +7741,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -8044,7 +8035,7 @@ dependencies = [ "reqwest 0.12.15", "ring 0.17.14", "rkyv 0.8.10", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", @@ -8053,7 +8044,7 @@ dependencies = [ "tokio", "tokio-rustls 0.26.2", "trc", - "webpki-roots 0.26.8", + "webpki-roots 0.26.10", "x509-parser 0.16.0", ] @@ -8159,7 +8150,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", "wasm-bindgen-shared", ] @@ -8194,7 +8185,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -8259,9 +8250,9 @@ checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "webpki-roots" -version = "0.26.8" +version = "0.26.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2210b291f7ea53617fbafcc4939f10914214ec15aace5ba62293a668f322c5c9" +checksum = "37493cadf42a2a939ed404698ded7fb378bf301b5011f973361779a3a74f8c93" dependencies = [ "rustls-pki-types", ] @@ -8345,7 +8336,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -8356,7 +8347,7 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -8617,9 +8608,9 @@ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" [[package]] name = "winnow" -version = "0.7.6" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63d3fcd9bba44b03821e7d699eeee959f3126dcc4aa8e4ae18ec617c2a5cea10" +checksum = "d9fb597c990f03753e08d3c29efbfcf2019a003b4bf4ba19225c158e1549f0f3" dependencies = [ "memchr", ] @@ -8759,8 +8750,8 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", - "synstructure 0.13.1", + "syn 2.0.101", + "synstructure 0.13.2", ] [[package]] @@ -8774,11 +8765,11 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.24" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb" dependencies = [ - "zerocopy-derive 0.8.24", + "zerocopy-derive 0.8.25", ] [[package]] @@ -8789,18 +8780,18 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] name = "zerocopy-derive" -version = "0.8.24" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -8820,8 +8811,8 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", - "synstructure 0.13.1", + "syn 2.0.101", + "synstructure 0.13.2", ] [[package]] @@ -8841,7 +8832,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -8863,7 +8854,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -8896,15 +8887,13 @@ dependencies = [ [[package]] name = "zopfli" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" +checksum = "edfc5ee405f504cd4984ecc6f14d02d55cfda60fa4b689434ef4102aae150cd7" dependencies = [ "bumpalo", "crc32fast", - "lockfree-object-pool", "log", - "once_cell", "simd-adler32", ] diff --git a/crates/common/src/config/inner.rs b/crates/common/src/config/inner.rs index 82f5a505..76a6e96e 100644 --- a/crates/common/src/config/inner.rs +++ b/crates/common/src/config/inner.rs @@ -23,8 +23,8 @@ use utils::{ }; use crate::{ - CacheSwap, Caches, Data, DavResource, DavResources, MailboxCache, MailboxStoreCache, - MessageStoreCache, MessageUidCache, TlsConnectors, + CacheSwap, Caches, Data, DavResource, DavResources, MailboxCache, MessageStoreCache, + MessageUidCache, TlsConnectors, auth::{AccessToken, roles::RolePermissions}, config::smtp::resolver::{Policy, Tlsa}, listener::blocked::BlockedIps, @@ -107,21 +107,14 @@ impl Caches { MB_5, std::mem::size_of::() as u64, ), - mailboxes: Cache::from_config( - config, - "mailbox", - MB_10, - (std::mem::size_of::() - + std::mem::size_of::>() - + (15 * (std::mem::size_of::() + 60))) as u64, - ), messages: Cache::from_config( config, "message", MB_10, (std::mem::size_of::() + std::mem::size_of::>() - + (1024 * std::mem::size_of::())) as u64, + + (1024 * std::mem::size_of::()) + + (15 * (std::mem::size_of::() + 60))) as u64, ), dav: Cache::from_config( config, diff --git a/crates/common/src/config/jmap/settings.rs b/crates/common/src/config/jmap/settings.rs index 96a5698e..894dc8af 100644 --- a/crates/common/src/config/jmap/settings.rs +++ b/crates/common/src/config/jmap/settings.rs @@ -16,7 +16,7 @@ pub struct JmapConfig { pub query_max_results: usize, pub snippet_max_results: usize, - pub changes_max_results: usize, + pub changes_max_results: Option, pub changes_max_history: Option, pub request_max_size: usize, @@ -242,8 +242,8 @@ impl JmapConfig { .property("jmap.protocol.query.max-results") .unwrap_or(5000), changes_max_results: config - .property("jmap.protocol.changes.max-results") - .unwrap_or(5000), + .property_or_default::>("jmap.protocol.changes.max-results", "5000") + .unwrap_or_default(), changes_max_history: config .property_or_default::>("jmap.protocol.changes.max-history", "30d") .unwrap_or_default(), diff --git a/crates/common/src/core.rs b/crates/common/src/core.rs index 11e1fc63..6d86bd88 100644 --- a/crates/common/src/core.rs +++ b/crates/common/src/core.rs @@ -8,7 +8,10 @@ use std::{sync::Arc, time::Duration}; use directory::{Directory, QueryBy, Type, backend::internal::manage::ManageDirectory}; use jmap_proto::types::{ - blob::BlobId, collection::Collection, property::Property, state::StateChange, + blob::BlobId, + collection::{Collection, SyncCollection}, + property::Property, + state::StateChange, type_state::DataType, }; use sieve::Sieve; @@ -506,10 +509,16 @@ impl Server { } if let Some(changes) = builder.changes() { - for (account_id, (change_id, changed_collections)) in changes { + for (account_id, changed_collections) in changes { let mut state_change = StateChange::new(account_id); - for changed_collection in changed_collections { - if let Ok(data_type) = DataType::try_from(changed_collection) { + let change_id = changed_collections.change_id; + for changed_collection in changed_collections.changed_containers { + if let Some(data_type) = DataType::try_from_id(changed_collection, true) { + state_change.set_change(data_type, change_id); + } + } + for changed_collection in changed_collections.changed_items { + if let Some(data_type) = DataType::try_from_id(changed_collection, false) { state_change.set_change(data_type, change_id); } } @@ -531,18 +540,14 @@ impl Server { })?; for collection in [ - Collection::Email.into(), - Collection::Mailbox.into(), - Collection::Mailbox.as_child_update(), - Collection::Thread.into(), - Collection::Identity.into(), - Collection::EmailSubmission.into(), - Collection::SieveScript.into(), - Collection::FileNode.into(), - Collection::AddressBook.into(), - Collection::ContactCard.into(), - Collection::Calendar.into(), - Collection::CalendarEvent.into(), + SyncCollection::Email.into(), + SyncCollection::Thread.into(), + SyncCollection::Identity.into(), + SyncCollection::EmailSubmission.into(), + SyncCollection::SieveScript.into(), + SyncCollection::FileNode.into(), + SyncCollection::AddressBook.into(), + SyncCollection::Calendar.into(), ] { self.core .storage diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 7a27b8ff..f98fabbf 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -149,7 +149,6 @@ pub struct Caches { pub permissions: Cache>, pub messages: Cache>, - pub mailboxes: Cache>, pub dav: Cache>, pub bayes: CacheWithTtl, @@ -168,21 +167,28 @@ pub struct Caches { pub struct CacheSwap(pub Arc>); #[derive(Debug, Clone)] -pub struct MailboxStoreCache { - pub change_id: u64, - pub index: AHashMap, - pub items: Vec, +pub struct MessageStoreCache { + pub emails: Arc, + pub mailboxes: Arc, pub update_lock: Arc, + pub last_change_id: u64, pub size: u64, } #[derive(Debug, Clone)] -pub struct MessageStoreCache { +pub struct MailboxesCache { + pub change_id: u64, + pub index: AHashMap, + pub items: Vec, + pub size: u64, +} + +#[derive(Debug, Clone)] +pub struct MessagesCache { pub change_id: u64, pub items: Vec, pub index: AHashMap, pub keywords: Vec, - pub update_lock: Arc, pub size: u64, } @@ -311,12 +317,6 @@ impl CacheItemWeight for MessageStoreCache { } } -impl CacheItemWeight for MailboxStoreCache { - fn weight(&self) -> u64 { - self.size - } -} - impl CacheItemWeight for HttpAuthCache { fn weight(&self) -> u64 { std::mem::size_of::() as u64 @@ -427,7 +427,6 @@ impl Default for Caches { access_tokens: Cache::new(1024, 10 * 1024 * 1024), http_auth: Cache::new(1024, 10 * 1024 * 1024), permissions: Cache::new(1024, 10 * 1024 * 1024), - mailboxes: Cache::new(1024, 10 * 1024 * 1024), messages: Cache::new(1024, 25 * 1024 * 1024), dav: Cache::new(1024, 10 * 1024 * 1024), bayes: CacheWithTtl::new(1024, 10 * 1024 * 1024), @@ -616,14 +615,14 @@ impl MessageStoreCache { bytes.extend_from_slice(message_id); let mut hash = store::gxhash::gxhash32(&bytes, 791120); - if self.items.is_empty() { + if self.emails.items.is_empty() { return hash; } // Naive pass, assume hash is unique let mut threads_ids = RoaringBitmap::new(); let mut is_unique_hash = true; - for item in self.items.iter() { + for item in self.emails.items.iter() { if is_unique_hash && item.thread_id != hash { is_unique_hash = false; } diff --git a/crates/common/src/storage/index.rs b/crates/common/src/storage/index.rs index cf84b7c4..9bae072e 100644 --- a/crates/common/src/storage/index.rs +++ b/crates/common/src/storage/index.rs @@ -40,13 +40,17 @@ pub enum IndexValue<'x> { Quota { used: u32, }, - LogChild { - prefix: Option, + LogContainer { + sync_collection: u8, }, - LogParent { - collection: u8, + LogContainerProperty { + sync_collection: u8, ids: Vec, }, + LogItem { + sync_collection: u8, + prefix: Option, + }, Acl { value: Cow<'x, [AclGrant]>, }, @@ -299,8 +303,19 @@ impl IntoOperations for (current, change) in current.inner.index_values().zip(changes.index_values()) { if current != change { merge_index(batch, current, change, self.tenant_id)?; - } else if let IndexValue::LogChild { prefix } = current { - batch.log_update(prefix); + } else { + match current { + IndexValue::LogContainer { sync_collection } => { + batch.log_container_update(sync_collection); + } + IndexValue::LogItem { + sync_collection, + prefix, + } => { + batch.log_item_update(sync_collection, prefix); + } + _ => (), + } } } batch.set(Property::Value, Archiver::new(changes).serialize()?); @@ -377,16 +392,29 @@ fn build_index(batch: &mut BatchBuilder, item: IndexValue<'_>, tenant_id: Option batch.add(DirectoryClass::UsedQuota(tenant_id), value); } } - IndexValue::LogChild { prefix } => { + IndexValue::LogItem { + sync_collection, + prefix, + } => { if set { - batch.log_insert(prefix); + batch.log_item_insert(sync_collection, prefix); } else { - batch.log_delete(prefix); + batch.log_item_delete(sync_collection, prefix); } } - IndexValue::LogParent { collection, ids } => { + IndexValue::LogContainer { sync_collection } => { + if set { + batch.log_container_insert(sync_collection); + } else { + batch.log_container_delete(sync_collection); + } + } + IndexValue::LogContainerProperty { + sync_collection, + ids, + } => { for parent_id in ids { - batch.log_parent_update(collection, parent_id); + batch.log_container_property_change(sync_collection, parent_id); } } } @@ -517,27 +545,32 @@ fn merge_index( } } ( - IndexValue::LogChild { prefix: old_prefix }, - IndexValue::LogChild { prefix: new_prefix }, + IndexValue::LogItem { + sync_collection, + prefix: old_prefix, + }, + IndexValue::LogItem { + prefix: new_prefix, .. + }, ) => { - batch.log_delete(old_prefix); - batch.log_insert(new_prefix); + batch.log_item_delete(sync_collection, old_prefix); + batch.log_item_insert(sync_collection, new_prefix); } ( - IndexValue::LogParent { - collection, + IndexValue::LogContainerProperty { + sync_collection, ids: old_ids, }, - IndexValue::LogParent { ids: new_ids, .. }, + IndexValue::LogContainerProperty { ids: new_ids, .. }, ) => { for parent_id in &old_ids { if !new_ids.contains(parent_id) { - batch.log_parent_update(collection, *parent_id); + batch.log_container_property_change(sync_collection, *parent_id); } } for parent_id in new_ids { if !old_ids.contains(&parent_id) { - batch.log_parent_update(collection, parent_id); + batch.log_container_property_change(sync_collection, parent_id); } } } diff --git a/crates/dav/src/common/propfind.rs b/crates/dav/src/common/propfind.rs index e8a17492..bf451d6a 100644 --- a/crates/dav/src/common/propfind.rs +++ b/crates/dav/src/common/propfind.rs @@ -411,6 +411,7 @@ impl PropFindRequestHandler for Server { // Filter by changelog match query.sync_type { SyncType::From { id, seq } => { + let todo = "fix"; let container_changes = self .store() .changes(account_id, collection_container, Query::Since(id)) @@ -439,7 +440,9 @@ impl PropFindRequestHandler for Server { { let changes = RoaringBitmap::from_iter( changes.changes.iter().filter_map(|change| match change { - Change::Insert(id) | Change::Update(id) => Some(*id as u32), + Change::InsertItem(id) | Change::UpdateItem(id) => { + Some(*id as u32) + } _ => None, }), ); @@ -872,6 +875,7 @@ impl PropFindRequestHandler for Server { let ctag = if let Some(ctag) = ctag { ctag } else { + let todo = "fix"; self.store() .get_last_change_id(account_id, collection) .await? @@ -1484,6 +1488,7 @@ impl PropFindData { let data = self.accounts.entry(account_id).or_default(); if data.sync_token.is_none() { + let todo = "fix"; let id = server .store() .get_last_change_id(account_id, collection_children) diff --git a/crates/dav/src/principal/propfind.rs b/crates/dav/src/principal/propfind.rs index 5bf3a23c..08a92ff7 100644 --- a/crates/dav/src/principal/propfind.rs +++ b/crates/dav/src/principal/propfind.rs @@ -184,6 +184,7 @@ impl PrincipalPropFind for Server { fields.push(DavPropertyValue::new(property.clone(), quota.used)); } WebDavProperty::SyncToken if !is_principal => { + let todo = "fix"; let id = self .store() .get_last_change_id(account_id, collection) diff --git a/crates/email/src/cache/email.rs b/crates/email/src/cache/email.rs new file mode 100644 index 00000000..e06ebb8f --- /dev/null +++ b/crates/email/src/cache/email.rs @@ -0,0 +1,321 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::message::metadata::{ArchivedMessageData, MessageData}; +use common::{ + MessageCache, MessageStoreCache, MessageUidCache, MessagesCache, Server, auth::AccessToken, + sharing::EffectiveAcl, +}; +use jmap_proto::types::{ + acl::Acl, + collection::Collection, + keyword::{Keyword, OTHER}, +}; +use store::{ahash::AHashMap, roaring::RoaringBitmap}; +use trc::AddContext; +use utils::map::bitmap::Bitmap; + +pub(crate) async fn update_email_cache( + server: &Server, + account_id: u32, + changed_ids: &AHashMap, + store_cache: &MessageStoreCache, +) -> trc::Result { + let mut new_cache = MessagesCache { + index: AHashMap::with_capacity(store_cache.emails.items.len()), + items: Vec::with_capacity(store_cache.emails.items.len()), + size: 0, + change_id: 0, + keywords: store_cache.emails.keywords.clone(), + }; + + for (document_id, is_update) in changed_ids { + if *is_update { + if let Some(archive) = server + .get_archive(account_id, Collection::Email, *document_id) + .await + .caused_by(trc::location!())? + { + insert_item( + &mut new_cache, + *document_id, + archive.unarchive::()?, + ); + } + } + } + + for item in &store_cache.emails.items { + if !changed_ids.contains_key(&item.document_id) { + email_insert(&mut new_cache, item.clone()); + } + } + + if store_cache.emails.items.len() > new_cache.items.len() { + new_cache.items.shrink_to_fit(); + new_cache.index.shrink_to_fit(); + } + if store_cache.emails.keywords.len() > new_cache.keywords.len() { + new_cache.keywords.shrink_to_fit(); + } + + Ok(new_cache) +} + +pub(crate) async fn full_email_cache_build( + server: &Server, + account_id: u32, +) -> trc::Result { + // Build cache + let mut cache = MessagesCache { + items: Vec::with_capacity(16), + index: AHashMap::with_capacity(16), + keywords: Vec::new(), + size: 0, + change_id: 0, + }; + + server + .get_archives( + account_id, + Collection::Email, + &(), + |document_id, archive| { + insert_item(&mut cache, document_id, archive.unarchive::()?); + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + + cache.items.shrink_to_fit(); + cache.index.shrink_to_fit(); + + Ok(cache) +} + +fn insert_item(cache: &mut MessagesCache, document_id: u32, message: &ArchivedMessageData) { + let mut item = MessageCache { + mailboxes: message + .mailboxes + .iter() + .map(|m| MessageUidCache { + mailbox_id: m.mailbox_id.to_native(), + uid: m.uid.to_native(), + }) + .collect(), + keywords: 0, + thread_id: message.thread_id.to_native(), + change_id: message.change_id.to_native(), + document_id, + }; + for keyword in message.keywords.iter() { + match keyword.id() { + Ok(id) => { + item.keywords |= 1 << id; + } + Err(custom) => { + if let Some(idx) = cache.keywords.iter().position(|k| k == custom) { + item.keywords |= 1 << (OTHER + idx); + } else if cache.keywords.len() < (128 - OTHER) { + cache.keywords.push(String::from(custom)); + item.keywords |= 1 << (OTHER + cache.keywords.len() - 1); + } + } + } + } + + email_insert(cache, item); +} + +pub trait MessageCacheAccess { + fn email_by_id(&self, id: &u32) -> Option<&MessageCache>; + + fn has_email_id(&self, id: &u32) -> bool; + + fn in_mailbox(&self, mailbox_id: u32) -> impl Iterator; + + fn in_thread(&self, thread_id: u32) -> impl Iterator; + + fn with_keyword(&self, keyword: &Keyword) -> impl Iterator; + + fn without_keyword(&self, keyword: &Keyword) -> impl Iterator; + + fn in_mailbox_with_keyword( + &self, + mailbox_id: u32, + keyword: &Keyword, + ) -> impl Iterator; + + fn in_mailbox_without_keyword( + &self, + mailbox_id: u32, + keyword: &Keyword, + ) -> impl Iterator; + + fn email_document_ids(&self) -> RoaringBitmap; + + fn shared_messages( + &self, + access_token: &AccessToken, + check_acls: impl Into> + Sync + Send, + ) -> RoaringBitmap; + + fn expand_keywords(&self, message: &MessageCache) -> impl Iterator; + + fn has_keyword(&self, message: &MessageCache, keyword: &Keyword) -> bool; +} + +impl MessageCacheAccess for MessageStoreCache { + fn in_mailbox(&self, mailbox_id: u32) -> impl Iterator { + self.emails + .items + .iter() + .filter(move |m| m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id)) + } + + fn in_thread(&self, thread_id: u32) -> impl Iterator { + self.emails + .items + .iter() + .filter(move |m| m.thread_id == thread_id) + } + + fn with_keyword(&self, keyword: &Keyword) -> impl Iterator { + let keyword_id = keyword_to_id(self, keyword); + self.emails + .items + .iter() + .filter(move |m| keyword_id.is_some_and(|id| m.keywords & (1 << id) != 0)) + } + + fn without_keyword(&self, keyword: &Keyword) -> impl Iterator { + let keyword_id = keyword_to_id(self, keyword); + self.emails + .items + .iter() + .filter(move |m| keyword_id.is_none_or(|id| m.keywords & (1 << id) == 0)) + } + + fn in_mailbox_with_keyword( + &self, + mailbox_id: u32, + keyword: &Keyword, + ) -> impl Iterator { + let keyword_id = keyword_to_id(self, keyword); + self.emails.items.iter().filter(move |m| { + m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id) + && keyword_id.is_some_and(|id| m.keywords & (1 << id) != 0) + }) + } + + fn in_mailbox_without_keyword( + &self, + mailbox_id: u32, + keyword: &Keyword, + ) -> impl Iterator { + let keyword_id = keyword_to_id(self, keyword); + self.emails.items.iter().filter(move |m| { + m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id) + && keyword_id.is_none_or(|id| m.keywords & (1 << id) == 0) + }) + } + + fn shared_messages( + &self, + access_token: &AccessToken, + check_acls: impl Into> + Sync + Send, + ) -> RoaringBitmap { + let check_acls = check_acls.into(); + let mut shared_messages = RoaringBitmap::new(); + for mailbox in &self.mailboxes.items { + if mailbox + .acls + .as_slice() + .effective_acl(access_token) + .contains_all(check_acls) + { + shared_messages.extend( + self.in_mailbox(mailbox.document_id) + .map(|item| item.document_id), + ); + } + } + shared_messages + } + + fn email_document_ids(&self) -> RoaringBitmap { + RoaringBitmap::from_iter(self.emails.index.keys()) + } + + fn email_by_id(&self, id: &u32) -> Option<&MessageCache> { + self.emails + .index + .get(id) + .and_then(|idx| self.emails.items.get(*idx as usize)) + } + + fn has_email_id(&self, id: &u32) -> bool { + self.emails.index.contains_key(id) + } + + fn expand_keywords(&self, message: &MessageCache) -> impl Iterator { + KeywordsIter(message.keywords).map(move |id| match Keyword::try_from_id(id) { + Ok(keyword) => keyword, + Err(id) => Keyword::Other(self.emails.keywords[id - OTHER].clone()), + }) + } + + fn has_keyword(&self, message: &MessageCache, keyword: &Keyword) -> bool { + keyword_to_id(self, keyword).is_some_and(|id| message.keywords & (1 << id) != 0) + } +} + +fn email_insert(cache: &mut MessagesCache, item: MessageCache) { + let id = item.document_id; + if let Some(idx) = cache.index.get(&id) { + cache.items[*idx as usize] = item; + } else { + cache.size += (std::mem::size_of::() + + (std::mem::size_of::() * 2) + + (item.mailboxes.len() * std::mem::size_of::())) + as u64; + + let idx = cache.items.len() as u32; + cache.items.push(item); + cache.index.insert(id, idx); + } +} + +#[inline] +fn keyword_to_id(cache: &MessageStoreCache, keyword: &Keyword) -> Option { + match keyword.id() { + Ok(id) => Some(id), + Err(name) => cache + .emails + .keywords + .iter() + .position(|k| k == name) + .map(|idx| (OTHER + idx) as u32), + } +} + +#[derive(Clone, Copy, Debug)] +struct KeywordsIter(u128); + +impl Iterator for KeywordsIter { + type Item = usize; + + fn next(&mut self) -> Option { + if self.0 != 0 { + let item = 127 - self.0.leading_zeros(); + self.0 ^= 1 << item; + Some(item as usize) + } else { + None + } + } +} diff --git a/crates/email/src/cache/mailbox.rs b/crates/email/src/cache/mailbox.rs new file mode 100644 index 00000000..007b45d0 --- /dev/null +++ b/crates/email/src/cache/mailbox.rs @@ -0,0 +1,296 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::mailbox::{ArchivedMailbox, Mailbox, manage::MailboxFnc}; +use common::{ + MailboxCache, MailboxesCache, MessageStoreCache, Server, auth::AccessToken, + config::jmap::settings::SpecialUse, sharing::EffectiveAcl, +}; +use jmap_proto::types::{acl::Acl, collection::Collection, value::AclGrant}; +use store::{ahash::AHashMap, roaring::RoaringBitmap}; +use trc::AddContext; +use utils::{map::bitmap::Bitmap, topological::TopologicalSort}; + +pub(crate) async fn update_mailbox_cache( + server: &Server, + account_id: u32, + changed_ids: &AHashMap, + store_cache: &MessageStoreCache, +) -> trc::Result { + let mut new_cache = MailboxesCache { + items: Vec::with_capacity(store_cache.mailboxes.items.len()), + index: AHashMap::with_capacity(store_cache.mailboxes.items.len()), + size: 0, + change_id: 0, + }; + + for (document_id, is_update) in changed_ids { + if *is_update { + if let Some(archive) = server + .get_archive(account_id, Collection::Mailbox, *document_id) + .await + .caused_by(trc::location!())? + { + insert_item( + &mut new_cache, + *document_id, + archive.unarchive::()?, + ); + } + } + } + + for item in store_cache.mailboxes.items.iter() { + if !changed_ids.contains_key(&item.document_id) { + mailbox_insert(&mut new_cache, item.clone()); + } + } + + build_tree(&mut new_cache); + + if store_cache.mailboxes.items.len() > new_cache.items.len() { + new_cache.items.shrink_to_fit(); + new_cache.index.shrink_to_fit(); + } + + Ok(new_cache) +} + +pub(crate) async fn full_mailbox_cache_build( + server: &Server, + account_id: u32, +) -> trc::Result { + // Build cache + let mut cache = MailboxesCache { + items: Default::default(), + index: Default::default(), + size: 0, + change_id: 0, + }; + + server + .get_archives( + account_id, + Collection::Mailbox, + &(), + |document_id, archive| { + insert_item(&mut cache, document_id, archive.unarchive::()?); + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + + if cache.items.is_empty() { + server + .create_system_folders(account_id) + .await + .caused_by(trc::location!())?; + server + .get_archives( + account_id, + Collection::Mailbox, + &(), + |document_id, archive| { + insert_item(&mut cache, document_id, archive.unarchive::()?); + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + } + + build_tree(&mut cache); + + Ok(cache) +} + +fn insert_item(cache: &mut MailboxesCache, document_id: u32, mailbox: &ArchivedMailbox) { + let parent_id = mailbox.parent_id.to_native(); + let item = MailboxCache { + document_id, + name: mailbox.name.as_str().into(), + path: "".into(), + role: (&mailbox.role).into(), + parent_id: if parent_id > 0 { + parent_id - 1 + } else { + u32::MAX + }, + sort_order: mailbox + .sort_order + .as_ref() + .map(|s| s.to_native()) + .unwrap_or(u32::MAX), + subscribers: mailbox.subscribers.iter().map(|s| s.to_native()).collect(), + uid_validity: mailbox.uid_validity.to_native(), + acls: mailbox + .acls + .iter() + .map(|acl| AclGrant { + account_id: acl.account_id.to_native(), + grants: Bitmap::from(&acl.grants), + }) + .collect(), + }; + + mailbox_insert(cache, item); +} + +fn build_tree(cache: &mut MailboxesCache) { + cache.size = 0; + let mut topological_sort = TopologicalSort::with_capacity(cache.items.len()); + + for (idx, mailbox) in cache.items.iter_mut().enumerate() { + topological_sort.insert( + if mailbox.parent_id == u32::MAX { + 0 + } else { + mailbox.parent_id + 1 + }, + mailbox.document_id + 1, + ); + mailbox.path = if matches!(mailbox.role, SpecialUse::Inbox) { + "INBOX".into() + } else if mailbox.is_root() && mailbox.name.as_str().eq_ignore_ascii_case("inbox") { + format!("INBOX {}", idx + 1) + } else { + mailbox.name.clone() + }; + + cache.size += item_size(mailbox); + } + + for folder_id in topological_sort.into_iterator() { + if folder_id != 0 { + let folder_id = folder_id - 1; + if let Some((path, parent_path)) = by_id(cache, &folder_id) + .and_then(|folder| { + folder + .parent_id() + .map(|parent_id| (&folder.path, parent_id)) + }) + .and_then(|(path, parent_id)| { + by_id(cache, &parent_id).map(|folder| (path, &folder.path)) + }) + { + let mut new_path = String::with_capacity(parent_path.len() + path.len() + 1); + new_path.push_str(parent_path.as_str()); + new_path.push('/'); + new_path.push_str(path.as_str()); + let folder = by_id_mut(cache, &folder_id).unwrap(); + folder.path = new_path; + } + } + } +} + +pub trait MailboxCacheAccess { + fn mailbox_by_id(&self, id: &u32) -> Option<&MailboxCache>; + fn mailbox_by_name(&self, name: &str) -> Option<&MailboxCache>; + fn mailbox_by_path(&self, name: &str) -> Option<&MailboxCache>; + fn mailbox_by_role(&self, role: &SpecialUse) -> Option<&MailboxCache>; + fn shared_mailboxes( + &self, + access_token: &AccessToken, + check_acls: impl Into> + Sync + Send, + ) -> RoaringBitmap; + fn has_mailbox_id(&self, id: &u32) -> bool; +} + +impl MailboxCacheAccess for MessageStoreCache { + fn mailbox_by_name(&self, name: &str) -> Option<&MailboxCache> { + self.mailboxes + .items + .iter() + .find(|m| m.name.eq_ignore_ascii_case(name)) + } + + fn mailbox_by_path(&self, path: &str) -> Option<&MailboxCache> { + self.mailboxes + .items + .iter() + .find(|m| m.path.eq_ignore_ascii_case(path)) + } + + fn mailbox_by_role(&self, role: &SpecialUse) -> Option<&MailboxCache> { + self.mailboxes.items.iter().find(|m| &m.role == role) + } + + fn shared_mailboxes( + &self, + access_token: &AccessToken, + check_acls: impl Into> + Sync + Send, + ) -> RoaringBitmap { + let check_acls = check_acls.into(); + + RoaringBitmap::from_iter( + self.mailboxes + .items + .iter() + .filter(|m| { + m.acls + .as_slice() + .effective_acl(access_token) + .contains_all(check_acls) + }) + .map(|m| m.document_id), + ) + } + + fn mailbox_by_id(&self, id: &u32) -> Option<&MailboxCache> { + self.mailboxes + .index + .get(id) + .and_then(|idx| self.mailboxes.items.get(*idx as usize)) + } + + fn has_mailbox_id(&self, id: &u32) -> bool { + self.mailboxes.index.contains_key(id) + } +} + +#[inline(always)] +fn by_id<'x>(cache: &'x MailboxesCache, id: &u32) -> Option<&'x MailboxCache> { + cache + .index + .get(id) + .and_then(|idx| cache.items.get(*idx as usize)) +} + +#[inline(always)] +fn by_id_mut<'x>(cache: &'x mut MailboxesCache, id: &u32) -> Option<&'x mut MailboxCache> { + cache + .index + .get(id) + .and_then(|idx| cache.items.get_mut(*idx as usize)) +} + +fn mailbox_insert(cache: &mut MailboxesCache, item: MailboxCache) { + let id = item.document_id; + if let Some(idx) = cache.index.get(&id) { + cache.items[*idx as usize] = item; + } else { + let idx = cache.items.len() as u32; + cache.items.push(item); + cache.index.insert(id, idx); + } +} + +#[inline(always)] +fn item_size(item: &MailboxCache) -> u64 { + (std::mem::size_of::() + + (if item.name.len() > std::mem::size_of::() { + item.name.len() + } else { + 0 + }) + + (if item.path.len() > std::mem::size_of::() { + item.path.len() + } else { + 0 + })) as u64 +} diff --git a/crates/email/src/cache/mod.rs b/crates/email/src/cache/mod.rs new file mode 100644 index 00000000..6e79b386 --- /dev/null +++ b/crates/email/src/cache/mod.rs @@ -0,0 +1,181 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::{collections::hash_map::Entry, sync::Arc}; + +use common::{CacheSwap, MessageStoreCache, Server}; +use email::{full_email_cache_build, update_email_cache}; +use jmap_proto::types::collection::SyncCollection; +use mailbox::{full_mailbox_cache_build, update_mailbox_cache}; +use store::{ + ahash::AHashMap, + query::log::{Change, Query}, +}; +use tokio::sync::Semaphore; +use trc::AddContext; + +pub mod email; +pub mod mailbox; + +pub trait MessageCacheFetch: Sync + Send { + fn get_cached_messages( + &self, + account_id: u32, + ) -> impl Future>> + Send; +} + +impl MessageCacheFetch for Server { + async fn get_cached_messages(&self, account_id: u32) -> trc::Result> { + let cache_ = match self + .inner + .cache + .messages + .get_value_or_guard_async(&account_id) + .await + { + Ok(cache) => cache, + Err(guard) => { + let cache = full_cache_build(self, account_id, Arc::new(Semaphore::new(1))).await?; + if guard.insert(CacheSwap::new(cache.clone())).is_err() { + self.inner + .cache + .messages + .insert(account_id, CacheSwap::new(cache.clone())); + } + return Ok(cache); + } + }; + + // Perform full refresh on stale ids + let cache = cache_.load_full(); + if cache.last_change_id > 0 + && self + .core + .jmap + .changes_max_history + .and_then(|history| self.inner.data.jmap_id_gen.past_id(history)) + .is_some_and(|last_change_id| cache.last_change_id < last_change_id) + { + let cache = full_cache_build(self, account_id, cache.update_lock.clone()).await?; + cache_.update(cache.clone()); + return Ok(cache); + } + + // Obtain current state + let changes = self + .core + .storage + .data + .changes( + account_id, + SyncCollection::Email, + Query::Since(cache.last_change_id), + ) + .await + .caused_by(trc::location!())?; + + // Verify changes + if changes.changes.is_empty() { + return Ok(cache); + } + + // Lock for updates + let _permit = cache.update_lock.acquire().await; + let cache = cache_.0.load(); + let mut cache = if cache.last_change_id >= changes.to_change_id { + return Ok(cache.clone()); + } else { + cache.as_ref().clone() + }; + + let mut changed_items: AHashMap = AHashMap::with_capacity(changes.changes.len()); + let mut changed_containers: AHashMap = + AHashMap::with_capacity(changes.changes.len()); + + for change in changes.changes { + match change { + Change::InsertItem(id) => match changed_items.entry(id as u32) { + Entry::Occupied(mut entry) => { + *entry.get_mut() = true; + } + Entry::Vacant(entry) => { + entry.insert(true); + } + }, + Change::UpdateItem(id) => { + changed_items.insert(id as u32, true); + } + Change::DeleteItem(id) => { + match changed_items.entry(id as u32) { + Entry::Occupied(mut entry) => { + // Thread reassignment + *entry.get_mut() = true; + } + Entry::Vacant(entry) => { + entry.insert(false); + } + } + } + Change::InsertContainer(id) | Change::UpdateContainer(id) => { + changed_containers.insert(id as u32, true); + } + Change::DeleteContainer(id) => { + changed_containers.insert(id as u32, false); + } + Change::UpdateContainerProperty(_) => (), + } + } + + if !changed_items.is_empty() { + let mut email_cache = + update_email_cache(self, account_id, &changed_items, &cache).await?; + email_cache.change_id = changes.item_change_id.unwrap_or(changes.to_change_id); + cache.emails = Arc::new(email_cache); + } + + if !changed_containers.is_empty() { + let mut mailbox_cache = + update_mailbox_cache(self, account_id, &changed_containers, &cache).await?; + mailbox_cache.change_id = changes.container_change_id.unwrap_or(changes.to_change_id); + cache.mailboxes = Arc::new(mailbox_cache); + } + cache.size = cache.emails.size + cache.mailboxes.size; + cache.last_change_id = changes.to_change_id; + + let cache = Arc::new(cache); + cache_.update(cache.clone()); + + Ok(cache) + } +} + +async fn full_cache_build( + server: &Server, + account_id: u32, + update_lock: Arc, +) -> trc::Result> { + let last_change_id = server + .core + .storage + .data + .get_last_change_id(account_id, SyncCollection::Email) + .await + .caused_by(trc::location!())? + .unwrap_or_default(); + let mut emails = full_email_cache_build(server, account_id).await?; + let mut mailboxes = full_mailbox_cache_build(server, account_id).await?; + let size = emails.size + mailboxes.size; + emails.change_id = last_change_id; + mailboxes.change_id = last_change_id; + + Ok(Arc::new(MessageStoreCache { + update_lock, + emails: Arc::new(emails), + mailboxes: Arc::new(mailboxes), + last_change_id, + size, + })) +} diff --git a/crates/email/src/identity/index.rs b/crates/email/src/identity/index.rs index 98289701..000b3cf0 100644 --- a/crates/email/src/identity/index.rs +++ b/crates/email/src/identity/index.rs @@ -5,18 +5,27 @@ */ use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject}; +use jmap_proto::types::collection::SyncCollection; use super::{ArchivedIdentity, Identity}; impl IndexableObject for Identity { fn index_values(&self) -> impl Iterator> { - [IndexValue::LogChild { prefix: None }].into_iter() + [IndexValue::LogItem { + sync_collection: SyncCollection::Identity.into(), + prefix: None, + }] + .into_iter() } } impl IndexableObject for &ArchivedIdentity { fn index_values(&self) -> impl Iterator> { - [IndexValue::LogChild { prefix: None }].into_iter() + [IndexValue::LogItem { + sync_collection: SyncCollection::Identity.into(), + prefix: None, + }] + .into_iter() } } diff --git a/crates/email/src/lib.rs b/crates/email/src/lib.rs index 3dfdb23c..48a38f16 100644 --- a/crates/email/src/lib.rs +++ b/crates/email/src/lib.rs @@ -4,6 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +pub mod cache; pub mod identity; pub mod mailbox; pub mod message; diff --git a/crates/email/src/mailbox/cache.rs b/crates/email/src/mailbox/cache.rs deleted file mode 100644 index 619e5af2..00000000 --- a/crates/email/src/mailbox/cache.rs +++ /dev/null @@ -1,371 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use std::sync::Arc; - -use common::{ - CacheSwap, MailboxCache, MailboxStoreCache, Server, auth::AccessToken, - config::jmap::settings::SpecialUse, sharing::EffectiveAcl, -}; - -use jmap_proto::types::{acl::Acl, collection::Collection, value::AclGrant}; -use std::future::Future; -use store::{ - ahash::{AHashMap, AHashSet}, - query::log::{Change, Query}, - roaring::RoaringBitmap, -}; -use tokio::sync::Semaphore; -use trc::AddContext; -use utils::{map::bitmap::Bitmap, topological::TopologicalSort}; - -use super::{ArchivedMailbox, Mailbox, manage::MailboxFnc}; - -pub trait MessageMailboxCache: Sync + Send { - fn get_cached_mailboxes( - &self, - account_id: u32, - ) -> impl Future>> + Send; -} - -impl MessageMailboxCache for Server { - async fn get_cached_mailboxes(&self, account_id: u32) -> trc::Result> { - let cache_ = match self - .inner - .cache - .mailboxes - .get_value_or_guard_async(&account_id) - .await - { - Ok(cache) => cache, - Err(guard) => { - let cache = full_cache_build(self, account_id, Arc::new(Semaphore::new(1))).await?; - - if guard.insert(CacheSwap::new(cache.clone())).is_err() { - self.inner - .cache - .mailboxes - .insert(account_id, CacheSwap::new(cache.clone())); - } - - return Ok(cache); - } - }; - - // Perform full refresh on stale ids - let cache = cache_.load_full(); - if cache.change_id > 0 - && self - .core - .jmap - .changes_max_history - .and_then(|history| self.inner.data.jmap_id_gen.past_id(history)) - .is_some_and(|last_change_id| cache.change_id < last_change_id) - { - let cache = full_cache_build(self, account_id, cache.update_lock.clone()).await?; - cache_.update(cache.clone()); - return Ok(cache); - } - - // Obtain current state - let changes = self - .core - .storage - .data - .changes( - account_id, - Collection::Mailbox, - Query::Since(cache.change_id), - ) - .await - .caused_by(trc::location!())?; - - // Verify changes - if changes.changes.is_empty() { - return Ok(cache); - } - - // Lock for updates - let _permit = cache.update_lock.acquire().await; - let cache = cache_.load_full(); - if cache.change_id >= changes.to_change_id { - return Ok(cache); - } - - let mut changed_ids = AHashSet::with_capacity(changes.changes.len()); - let mut new_cache = MailboxStoreCache { - items: Vec::with_capacity(cache.items.len()), - index: AHashMap::with_capacity(cache.items.len()), - size: 0, - change_id: changes.to_change_id, - update_lock: cache.update_lock.clone(), - }; - - for change in changes.changes { - match change { - Change::Insert(id) | Change::Update(id) => { - let document_id = id as u32; - if let Some(archive) = self - .get_archive(account_id, Collection::Mailbox, document_id) - .await - .caused_by(trc::location!())? - { - insert_item(&mut new_cache, document_id, archive.unarchive::()?); - changed_ids.insert(document_id); - } - } - Change::Delete(id) => { - changed_ids.insert(id as u32); - } - } - } - - for item in cache.items.iter() { - if !changed_ids.contains(&item.document_id) { - new_cache.insert(item.clone()); - } - } - - build_tree(&mut new_cache); - - if cache.items.len() > new_cache.items.len() { - new_cache.items.shrink_to_fit(); - new_cache.index.shrink_to_fit(); - } - - let cache = Arc::new(new_cache); - cache_.update(cache.clone()); - - Ok(cache) - } -} - -async fn full_cache_build( - server: &Server, - account_id: u32, - update_lock: Arc, -) -> trc::Result> { - // Build cache - let mut cache = MailboxStoreCache { - items: Default::default(), - index: Default::default(), - size: 0, - change_id: 0, - update_lock, - }; - - server - .get_archives( - account_id, - Collection::Mailbox, - &(), - |document_id, archive| { - insert_item(&mut cache, document_id, archive.unarchive::()?); - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - - if cache.items.is_empty() { - server - .create_system_folders(account_id) - .await - .caused_by(trc::location!())?; - server - .get_archives( - account_id, - Collection::Mailbox, - &(), - |document_id, archive| { - insert_item(&mut cache, document_id, archive.unarchive::()?); - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - } - - build_tree(&mut cache); - - Ok(Arc::new(cache)) -} - -fn insert_item(cache: &mut MailboxStoreCache, document_id: u32, mailbox: &ArchivedMailbox) { - let parent_id = mailbox.parent_id.to_native(); - let item = MailboxCache { - document_id, - name: mailbox.name.as_str().into(), - path: "".into(), - role: (&mailbox.role).into(), - parent_id: if parent_id > 0 { - parent_id - 1 - } else { - u32::MAX - }, - sort_order: mailbox - .sort_order - .as_ref() - .map(|s| s.to_native()) - .unwrap_or(u32::MAX), - subscribers: mailbox.subscribers.iter().map(|s| s.to_native()).collect(), - uid_validity: mailbox.uid_validity.to_native(), - acls: mailbox - .acls - .iter() - .map(|acl| AclGrant { - account_id: acl.account_id.to_native(), - grants: Bitmap::from(&acl.grants), - }) - .collect(), - }; - - cache.insert(item); -} - -fn build_tree(cache: &mut MailboxStoreCache) { - cache.size = 0; - let mut topological_sort = TopologicalSort::with_capacity(cache.items.len()); - - for (idx, mailbox) in cache.items.iter_mut().enumerate() { - topological_sort.insert( - if mailbox.parent_id == u32::MAX { - 0 - } else { - mailbox.parent_id + 1 - }, - mailbox.document_id + 1, - ); - mailbox.path = if matches!(mailbox.role, SpecialUse::Inbox) { - "INBOX".into() - } else if mailbox.is_root() && mailbox.name.as_str().eq_ignore_ascii_case("inbox") { - format!("INBOX {}", idx + 1) - } else { - mailbox.name.clone() - }; - - cache.size += item_size(mailbox); - } - - for folder_id in topological_sort.into_iterator() { - if folder_id != 0 { - let folder_id = folder_id - 1; - if let Some((path, parent_path)) = cache - .by_id(&folder_id) - .and_then(|folder| { - folder - .parent_id() - .map(|parent_id| (&folder.path, parent_id)) - }) - .and_then(|(path, parent_id)| { - cache.by_id(&parent_id).map(|folder| (path, &folder.path)) - }) - { - let mut new_path = String::with_capacity(parent_path.len() + path.len() + 1); - new_path.push_str(parent_path.as_str()); - new_path.push('/'); - new_path.push_str(path.as_str()); - let folder = cache.by_id_mut(&folder_id).unwrap(); - folder.path = new_path; - } - } - } -} - -pub trait MailboxCacheAccess { - fn by_id(&self, id: &u32) -> Option<&MailboxCache>; - fn by_id_mut(&mut self, id: &u32) -> Option<&mut MailboxCache>; - fn insert(&mut self, item: MailboxCache); - fn by_name(&self, name: &str) -> Option<&MailboxCache>; - fn by_path(&self, name: &str) -> Option<&MailboxCache>; - fn by_role(&self, role: &SpecialUse) -> Option<&MailboxCache>; - fn shared_mailboxes( - &self, - access_token: &AccessToken, - check_acls: impl Into> + Sync + Send, - ) -> RoaringBitmap; - fn has_id(&self, id: &u32) -> bool; -} - -impl MailboxCacheAccess for MailboxStoreCache { - fn by_name(&self, name: &str) -> Option<&MailboxCache> { - self.items - .iter() - .find(|m| m.name.eq_ignore_ascii_case(name)) - } - - fn by_path(&self, path: &str) -> Option<&MailboxCache> { - self.items - .iter() - .find(|m| m.path.eq_ignore_ascii_case(path)) - } - - fn by_role(&self, role: &SpecialUse) -> Option<&MailboxCache> { - self.items.iter().find(|m| &m.role == role) - } - - fn shared_mailboxes( - &self, - access_token: &AccessToken, - check_acls: impl Into> + Sync + Send, - ) -> RoaringBitmap { - let check_acls = check_acls.into(); - - RoaringBitmap::from_iter( - self.items - .iter() - .filter(|m| { - m.acls - .as_slice() - .effective_acl(access_token) - .contains_all(check_acls) - }) - .map(|m| m.document_id), - ) - } - - fn by_id(&self, id: &u32) -> Option<&MailboxCache> { - self.index - .get(id) - .and_then(|idx| self.items.get(*idx as usize)) - } - - fn by_id_mut(&mut self, id: &u32) -> Option<&mut MailboxCache> { - self.index - .get(id) - .and_then(|idx| self.items.get_mut(*idx as usize)) - } - - fn insert(&mut self, item: MailboxCache) { - let id = item.document_id; - if let Some(idx) = self.index.get(&id) { - self.items[*idx as usize] = item; - } else { - let idx = self.items.len() as u32; - self.items.push(item); - self.index.insert(id, idx); - } - } - - fn has_id(&self, id: &u32) -> bool { - self.index.contains_key(id) - } -} - -#[inline(always)] -fn item_size(item: &MailboxCache) -> u64 { - (std::mem::size_of::() - + (if item.name.len() > std::mem::size_of::() { - item.name.len() - } else { - 0 - }) - + (if item.path.len() > std::mem::size_of::() { - item.path.len() - } else { - 0 - })) as u64 -} diff --git a/crates/email/src/mailbox/destroy.rs b/crates/email/src/mailbox/destroy.rs index 3201b00e..ab61f68e 100644 --- a/crates/email/src/mailbox/destroy.rs +++ b/crates/email/src/mailbox/destroy.rs @@ -4,6 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::*; +use crate::{ + cache::{MessageCacheFetch, email::MessageCacheAccess}, + message::{delete::EmailDeletion, metadata::MessageData}, +}; use common::{ Server, auth::AccessToken, sharing::EffectiveAcl, storage::index::ObjectIndexBuilder, }; @@ -15,14 +20,6 @@ use jmap_proto::{ use store::{roaring::RoaringBitmap, write::BatchBuilder}; use trc::AddContext; -use crate::message::{ - cache::{MessageCacheAccess, MessageCacheFetch}, - delete::EmailDeletion, - metadata::MessageData, -}; - -use super::{cache::MessageMailboxCache, *}; - pub trait MailboxDestroy: Sync + Send { fn mailbox_destroy( &self, @@ -61,9 +58,12 @@ impl MailboxDestroy for Server { } // Verify that this mailbox does not have sub-mailboxes - if self - .get_cached_mailboxes(account_id) - .await? + let cache = self + .get_cached_messages(account_id) + .await + .caused_by(trc::location!())?; + if cache + .mailboxes .items .iter() .any(|item| item.parent_id == document_id) @@ -77,13 +77,8 @@ impl MailboxDestroy for Server { batch.with_account_id(account_id); - let message_ids = RoaringBitmap::from_iter( - self.get_cached_messages(account_id) - .await - .caused_by(trc::location!())? - .in_mailbox(document_id) - .map(|m| m.document_id), - ); + let message_ids = + RoaringBitmap::from_iter(cache.in_mailbox(document_id).map(|m| m.document_id)); if !message_ids.is_empty() { if remove_emails { diff --git a/crates/email/src/mailbox/index.rs b/crates/email/src/mailbox/index.rs index 7291b6cf..ce70d6d0 100644 --- a/crates/email/src/mailbox/index.rs +++ b/crates/email/src/mailbox/index.rs @@ -8,14 +8,16 @@ use common::storage::{ folder::FolderHierarchy, index::{IndexValue, IndexableAndSerializableObject, IndexableObject}, }; -use jmap_proto::types::value::AclGrant; +use jmap_proto::types::{collection::SyncCollection, value::AclGrant}; use super::{ArchivedMailbox, Mailbox}; impl IndexableObject for Mailbox { fn index_values(&self) -> impl Iterator> { [ - IndexValue::LogChild { prefix: None }, + IndexValue::LogContainer { + sync_collection: SyncCollection::Email.into(), + }, IndexValue::Acl { value: (&self.acls).into(), }, @@ -27,7 +29,9 @@ impl IndexableObject for Mailbox { impl IndexableObject for &ArchivedMailbox { fn index_values(&self) -> impl Iterator> { [ - IndexValue::LogChild { prefix: None }, + IndexValue::LogContainer { + sync_collection: SyncCollection::Email.into(), + }, IndexValue::Acl { value: self .acls diff --git a/crates/email/src/mailbox/manage.rs b/crates/email/src/mailbox/manage.rs index 22623dcd..899ef0ec 100644 --- a/crates/email/src/mailbox/manage.rs +++ b/crates/email/src/mailbox/manage.rs @@ -4,11 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::future::Future; - -use super::{cache::MessageMailboxCache, *}; +use super::*; +use crate::cache::MessageCacheFetch; use common::{Server, config::jmap::settings::SpecialUse, storage::index::ObjectIndexBuilder}; use jmap_proto::types::collection::Collection; +use std::future::Future; use store::write::BatchBuilder; use trc::AddContext; @@ -79,8 +79,8 @@ impl MailboxFnc for Server { } async fn mailbox_create_path(&self, account_id: u32, path: &str) -> trc::Result> { - let folders = self - .get_cached_mailboxes(account_id) + let cache = self + .get_cached_messages(account_id) .await .caused_by(trc::location!())?; @@ -101,7 +101,8 @@ impl MailboxFnc for Server { } } - if let Some(item) = folders + if let Some(item) = cache + .mailboxes .items .iter() .find(|item| item.path.to_lowercase() == found_path) diff --git a/crates/email/src/mailbox/mod.rs b/crates/email/src/mailbox/mod.rs index 8f09b0bc..10f0dcba 100644 --- a/crates/email/src/mailbox/mod.rs +++ b/crates/email/src/mailbox/mod.rs @@ -9,7 +9,6 @@ use common::config::jmap::settings::SpecialUse; use jmap_proto::types::value::AclGrant; use store::{SERIALIZE_MAILBOX_V1, SerializedVersion}; -pub mod cache; pub mod destroy; pub mod index; pub mod manage; diff --git a/crates/email/src/message/cache.rs b/crates/email/src/message/cache.rs deleted file mode 100644 index 114d5495..00000000 --- a/crates/email/src/message/cache.rs +++ /dev/null @@ -1,431 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use super::metadata::{ArchivedMessageData, MessageData}; -use common::{ - CacheSwap, MailboxStoreCache, MessageCache, MessageStoreCache, MessageUidCache, Server, - auth::AccessToken, sharing::EffectiveAcl, -}; - -use jmap_proto::types::{ - acl::Acl, - collection::Collection, - keyword::{Keyword, OTHER}, -}; -use std::sync::Arc; -use std::{collections::hash_map::Entry, future::Future}; -use store::{ - ahash::AHashMap, - query::log::{Change, Query}, - roaring::RoaringBitmap, -}; -use tokio::sync::Semaphore; -use trc::AddContext; -use utils::map::bitmap::Bitmap; - -pub trait MessageCacheFetch: Sync + Send { - fn get_cached_messages( - &self, - account_id: u32, - ) -> impl Future>> + Send; -} - -impl MessageCacheFetch for Server { - async fn get_cached_messages(&self, account_id: u32) -> trc::Result> { - let cache_ = match self - .inner - .cache - .messages - .get_value_or_guard_async(&account_id) - .await - { - Ok(cache) => cache, - Err(guard) => { - let cache = full_cache_build(self, account_id, Arc::new(Semaphore::new(1))).await?; - if guard.insert(CacheSwap::new(cache.clone())).is_err() { - self.inner - .cache - .messages - .insert(account_id, CacheSwap::new(cache.clone())); - } - return Ok(cache); - } - }; - - // Perform full refresh on stale ids - let cache = cache_.load_full(); - if cache.change_id > 0 - && self - .core - .jmap - .changes_max_history - .and_then(|history| self.inner.data.jmap_id_gen.past_id(history)) - .is_some_and(|last_change_id| cache.change_id < last_change_id) - { - let cache = full_cache_build(self, account_id, cache.update_lock.clone()).await?; - cache_.update(cache.clone()); - return Ok(cache); - } - - // Obtain current state - let changes = self - .core - .storage - .data - .changes(account_id, Collection::Email, Query::Since(cache.change_id)) - .await - .caused_by(trc::location!())?; - - // Verify changes - if changes.changes.is_empty() { - return Ok(cache); - } - - // Lock for updates - let _permit = cache.update_lock.acquire().await; - let cache = cache_.load_full(); - if cache.change_id >= changes.to_change_id { - return Ok(cache); - } - - let mut new_cache = MessageStoreCache { - index: AHashMap::with_capacity(cache.items.len()), - items: Vec::with_capacity(cache.items.len()), - size: 0, - change_id: changes.to_change_id, - update_lock: cache.update_lock.clone(), - keywords: cache.keywords.clone(), - }; - let mut changed_ids: AHashMap = AHashMap::with_capacity(changes.changes.len()); - - for change in changes.changes { - match change { - Change::Insert(id) => match changed_ids.entry(id as u32) { - Entry::Occupied(mut entry) => { - *entry.get_mut() = true; - } - Entry::Vacant(entry) => { - entry.insert(true); - } - }, - Change::Update(id) => { - changed_ids.insert(id as u32, true); - } - Change::Delete(id) => { - match changed_ids.entry(id as u32) { - Entry::Occupied(mut entry) => { - // Thread reassignment - *entry.get_mut() = true; - } - Entry::Vacant(entry) => { - entry.insert(false); - } - } - } - } - } - - for (document_id, is_update) in &changed_ids { - if *is_update { - if let Some(archive) = self - .get_archive(account_id, Collection::Email, *document_id) - .await - .caused_by(trc::location!())? - { - insert_item( - &mut new_cache, - *document_id, - archive.unarchive::()?, - ); - } - } - } - - for item in &cache.items { - if !changed_ids.contains_key(&item.document_id) { - new_cache.insert(item.clone()); - } - } - - if cache.items.len() > new_cache.items.len() { - new_cache.items.shrink_to_fit(); - new_cache.index.shrink_to_fit(); - } - if cache.keywords.len() > new_cache.keywords.len() { - new_cache.keywords.shrink_to_fit(); - } - - let cache = Arc::new(new_cache); - cache_.update(cache.clone()); - - Ok(cache) - } -} - -async fn full_cache_build( - server: &Server, - account_id: u32, - update_lock: Arc, -) -> trc::Result> { - // Build cache - let mut cache = MessageStoreCache { - items: Vec::with_capacity(16), - index: AHashMap::with_capacity(16), - keywords: Vec::new(), - size: 0, - change_id: 0, - update_lock, - }; - - server - .get_archives( - account_id, - Collection::Email, - &(), - |document_id, archive| { - let message = archive.unarchive::()?; - cache.change_id = std::cmp::max(cache.change_id, message.change_id.to_native()); - - insert_item(&mut cache, document_id, message); - - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - - cache.items.shrink_to_fit(); - cache.index.shrink_to_fit(); - - Ok(Arc::new(cache)) -} - -fn insert_item(cache: &mut MessageStoreCache, document_id: u32, message: &ArchivedMessageData) { - let mut item = MessageCache { - mailboxes: message - .mailboxes - .iter() - .map(|m| MessageUidCache { - mailbox_id: m.mailbox_id.to_native(), - uid: m.uid.to_native(), - }) - .collect(), - keywords: 0, - thread_id: message.thread_id.to_native(), - change_id: message.change_id.to_native(), - document_id, - }; - for keyword in message.keywords.iter() { - match keyword.id() { - Ok(id) => { - item.keywords |= 1 << id; - } - Err(custom) => { - if let Some(idx) = cache.keywords.iter().position(|k| k == custom) { - item.keywords |= 1 << (OTHER + idx); - } else if cache.keywords.len() < (128 - OTHER) { - cache.keywords.push(String::from(custom)); - item.keywords |= 1 << (OTHER + cache.keywords.len() - 1); - } - } - } - } - - cache.insert(item); -} - -pub trait MessageCacheAccess { - fn by_id(&self, id: &u32) -> Option<&MessageCache>; - - fn has_id(&self, id: &u32) -> bool; - - fn by_id_mut(&mut self, id: &u32) -> Option<&mut MessageCache>; - - fn insert(&mut self, item: MessageCache); - - fn in_mailbox(&self, mailbox_id: u32) -> impl Iterator; - - fn in_thread(&self, thread_id: u32) -> impl Iterator; - - fn with_keyword(&self, keyword: &Keyword) -> impl Iterator; - - fn without_keyword(&self, keyword: &Keyword) -> impl Iterator; - - fn in_mailbox_with_keyword( - &self, - mailbox_id: u32, - keyword: &Keyword, - ) -> impl Iterator; - - fn in_mailbox_without_keyword( - &self, - mailbox_id: u32, - keyword: &Keyword, - ) -> impl Iterator; - - fn document_ids(&self) -> RoaringBitmap; - - fn shared_messages( - &self, - access_token: &AccessToken, - mailboxes: &MailboxStoreCache, - check_acls: impl Into> + Sync + Send, - ) -> RoaringBitmap; - - fn expand_keywords(&self, message: &MessageCache) -> impl Iterator; - - fn has_keyword(&self, message: &MessageCache, keyword: &Keyword) -> bool; -} - -impl MessageCacheAccess for MessageStoreCache { - fn in_mailbox(&self, mailbox_id: u32) -> impl Iterator { - self.items - .iter() - .filter(move |m| m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id)) - } - - fn in_thread(&self, thread_id: u32) -> impl Iterator { - self.items.iter().filter(move |m| m.thread_id == thread_id) - } - - fn with_keyword(&self, keyword: &Keyword) -> impl Iterator { - let keyword_id = keyword_to_id(self, keyword); - self.items - .iter() - .filter(move |m| keyword_id.is_some_and(|id| m.keywords & (1 << id) != 0)) - } - - fn without_keyword(&self, keyword: &Keyword) -> impl Iterator { - let keyword_id = keyword_to_id(self, keyword); - self.items - .iter() - .filter(move |m| keyword_id.is_none_or(|id| m.keywords & (1 << id) == 0)) - } - - fn in_mailbox_with_keyword( - &self, - mailbox_id: u32, - keyword: &Keyword, - ) -> impl Iterator { - let keyword_id = keyword_to_id(self, keyword); - self.items.iter().filter(move |m| { - m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id) - && keyword_id.is_some_and(|id| m.keywords & (1 << id) != 0) - }) - } - - fn in_mailbox_without_keyword( - &self, - mailbox_id: u32, - keyword: &Keyword, - ) -> impl Iterator { - let keyword_id = keyword_to_id(self, keyword); - self.items.iter().filter(move |m| { - m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id) - && keyword_id.is_none_or(|id| m.keywords & (1 << id) == 0) - }) - } - - fn shared_messages( - &self, - access_token: &AccessToken, - mailboxes: &MailboxStoreCache, - check_acls: impl Into> + Sync + Send, - ) -> RoaringBitmap { - let check_acls = check_acls.into(); - let mut shared_messages = RoaringBitmap::new(); - for mailbox in &mailboxes.items { - if mailbox - .acls - .as_slice() - .effective_acl(access_token) - .contains_all(check_acls) - { - shared_messages.extend( - self.in_mailbox(mailbox.document_id) - .map(|item| item.document_id), - ); - } - } - shared_messages - } - - fn document_ids(&self) -> RoaringBitmap { - RoaringBitmap::from_iter(self.index.keys()) - } - - fn by_id(&self, id: &u32) -> Option<&MessageCache> { - self.index - .get(id) - .and_then(|idx| self.items.get(*idx as usize)) - } - - fn by_id_mut(&mut self, id: &u32) -> Option<&mut MessageCache> { - self.index - .get(id) - .and_then(|idx| self.items.get_mut(*idx as usize)) - } - - fn insert(&mut self, item: MessageCache) { - let id = item.document_id; - if let Some(idx) = self.index.get(&id) { - self.items[*idx as usize] = item; - } else { - self.size += (std::mem::size_of::() - + (std::mem::size_of::() * 2) - + (item.mailboxes.len() * std::mem::size_of::())) - as u64; - - let idx = self.items.len() as u32; - self.items.push(item); - self.index.insert(id, idx); - } - } - - fn has_id(&self, id: &u32) -> bool { - self.index.contains_key(id) - } - - fn expand_keywords(&self, message: &MessageCache) -> impl Iterator { - KeywordsIter(message.keywords).map(move |id| match Keyword::try_from_id(id) { - Ok(keyword) => keyword, - Err(id) => Keyword::Other(self.keywords[id - OTHER].clone()), - }) - } - - fn has_keyword(&self, message: &MessageCache, keyword: &Keyword) -> bool { - keyword_to_id(self, keyword).is_some_and(|id| message.keywords & (1 << id) != 0) - } -} - -#[inline] -fn keyword_to_id(cache: &MessageStoreCache, keyword: &Keyword) -> Option { - match keyword.id() { - Ok(id) => Some(id), - Err(name) => cache - .keywords - .iter() - .position(|k| k == name) - .map(|idx| (OTHER + idx) as u32), - } -} - -#[derive(Clone, Copy, Debug)] -struct KeywordsIter(u128); - -impl Iterator for KeywordsIter { - type Item = usize; - - fn next(&mut self) -> Option { - if self.0 != 0 { - let item = 127 - self.0.leading_zeros(); - self.0 ^= 1 << item; - Some(item as usize) - } else { - None - } - } -} diff --git a/crates/email/src/message/copy.rs b/crates/email/src/message/copy.rs index 8a6abe56..b80b860b 100644 --- a/crates/email/src/message/copy.rs +++ b/crates/email/src/message/copy.rs @@ -8,7 +8,11 @@ use common::{Server, auth::ResourceToken, storage::index::ObjectIndexBuilder}; use jmap_proto::{ error::set::SetError, types::{ - blob::BlobId, collection::Collection, date::UTCDate, id::Id, keyword::Keyword, + blob::BlobId, + collection::{Collection, SyncCollection}, + date::UTCDate, + id::Id, + keyword::Keyword, property::Property, }, }; @@ -19,10 +23,9 @@ use store::{ }; use trc::AddContext; -use crate::mailbox::UidMailbox; +use crate::{cache::MessageCacheFetch, mailbox::UidMailbox}; use super::{ - cache::MessageCacheFetch, index::{MAX_ID_LENGTH, MAX_SORT_FIELD_LENGTH, TrimTextValue}, ingest::{EmailIngest, IngestedEmail, ThreadResult}, metadata::{HeaderName, HeaderValue, MessageData, MessageMetadata}, @@ -178,7 +181,7 @@ impl EmailCopy for Server { batch .with_collection(Collection::Thread) .update_document(thread_id) - .log_insert(None); + .log_container_insert(SyncCollection::Thread); } let document_id = self diff --git a/crates/email/src/message/delete.rs b/crates/email/src/message/delete.rs index 0c8c167d..caff82b9 100644 --- a/crates/email/src/message/delete.rs +++ b/crates/email/src/message/delete.rs @@ -4,10 +4,13 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::time::Duration; - +use super::metadata::MessageData; +use crate::{cache::MessageCacheFetch, mailbox::*, message::metadata::MessageMetadata}; use common::{KV_LOCK_PURGE_ACCOUNT, Server, storage::index::ObjectIndexBuilder}; use jmap_proto::types::{collection::Collection, property::Property}; +use std::future::Future; +use std::time::Duration; +use store::rand::prelude::SliceRandom; use store::{ BitmapKey, ValueKey, roaring::RoaringBitmap, @@ -16,16 +19,6 @@ use store::{ use trc::AddContext; use utils::BlobHash; -use std::future::Future; -use store::rand::prelude::SliceRandom; - -use crate::{ - mailbox::*, - message::{cache::MessageCacheFetch, metadata::MessageMetadata}, -}; - -use super::metadata::MessageData; - pub trait EmailDeletion: Sync + Send { fn emails_tombstone( &self, @@ -184,6 +177,7 @@ impl EmailDeletion for Server { self.get_cached_messages(account_id) .await .caused_by(trc::location!())? + .emails .items .iter() .filter(|item| { diff --git a/crates/email/src/message/index.rs b/crates/email/src/message/index.rs index 3050d7bb..0305a5df 100644 --- a/crates/email/src/message/index.rs +++ b/crates/email/src/message/index.rs @@ -4,8 +4,14 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::metadata::{ + Addr, Address, ArchivedAddress, ArchivedGetHeader, ArchivedHeaderName, ArchivedHeaderValue, + ArchivedMessageData, ArchivedMessageMetadata, ArchivedMessageMetadataContents, + ArchivedMessageMetadataPart, ArchivedMetadataPartType, DecodedPartContent, Group, HeaderName, + HeaderValue, MessageData, MessageMetadata, MessageMetadataPart, +}; use common::storage::index::{IndexValue, IndexableObject, ObjectIndexBuilder}; -use jmap_proto::types::{collection::Collection, property::Property}; +use jmap_proto::types::{collection::SyncCollection, property::Property}; use mail_parser::{ decoders::html::html_to_text, parsers::{fields::thread::thread_name, preview::preview_text}, @@ -21,13 +27,6 @@ use store::{ use trc::AddContext; use utils::BlobHash; -use super::metadata::{ - Addr, Address, ArchivedAddress, ArchivedGetHeader, ArchivedHeaderName, ArchivedHeaderValue, - ArchivedMessageData, ArchivedMessageMetadata, ArchivedMessageMetadataContents, - ArchivedMessageMetadataPart, ArchivedMetadataPartType, DecodedPartContent, Group, HeaderName, - HeaderValue, MessageData, MessageMetadata, MessageMetadataPart, -}; - pub const MAX_MESSAGE_PARTS: usize = 1000; pub const MAX_ID_LENGTH: usize = 100; pub const MAX_SORT_FIELD_LENGTH: usize = 255; @@ -577,15 +576,16 @@ impl IndexMessage for BatchBuilder { impl IndexableObject for MessageData { fn index_values(&self) -> impl Iterator> { [ - IndexValue::LogChild { + IndexValue::LogItem { + sync_collection: SyncCollection::Email.into(), prefix: self.thread_id.into(), }, - IndexValue::LogParent { - collection: Collection::Thread.into(), + IndexValue::LogContainerProperty { + sync_collection: SyncCollection::Thread.into(), ids: vec![self.thread_id], }, - IndexValue::LogParent { - collection: Collection::Mailbox.as_child_update(), + IndexValue::LogContainerProperty { + sync_collection: SyncCollection::Email.into(), ids: self.mailboxes.iter().map(|m| m.mailbox_id).collect(), }, ] @@ -596,15 +596,16 @@ impl IndexableObject for MessageData { impl IndexableObject for &ArchivedMessageData { fn index_values(&self) -> impl Iterator> { [ - IndexValue::LogChild { + IndexValue::LogItem { + sync_collection: SyncCollection::Email.into(), prefix: self.thread_id.to_native().into(), }, - IndexValue::LogParent { - collection: Collection::Thread.into(), + IndexValue::LogContainerProperty { + sync_collection: SyncCollection::Thread.into(), ids: vec![self.thread_id.to_native()], }, - IndexValue::LogParent { - collection: Collection::Mailbox.as_child_update(), + IndexValue::LogContainerProperty { + sync_collection: SyncCollection::Email.into(), ids: self .mailboxes .iter() diff --git a/crates/email/src/message/ingest.rs b/crates/email/src/message/ingest.rs index d60ef16e..8958ad35 100644 --- a/crates/email/src/message/ingest.rs +++ b/crates/email/src/message/ingest.rs @@ -4,12 +4,19 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{ - borrow::Cow, - fmt::Write, - time::{Duration, Instant}, +use super::{ + crypto::{EncryptMessage, EncryptMessageError}, + index::{MAX_SORT_FIELD_LENGTH, TrimTextValue}, +}; +use crate::{ + cache::{MessageCacheFetch, email::MessageCacheAccess}, + mailbox::{INBOX_ID, JUNK_ID, UidMailbox}, + message::{ + crypto::EncryptionParams, + index::{IndexMessage, MAX_ID_LENGTH, VisitValues}, + metadata::MessageData, + }, }; - use common::{ Server, auth::{AccessToken, ResourceToken}, @@ -18,7 +25,7 @@ use common::{ use directory::Permission; use jmap_proto::types::{ blob::BlobId, - collection::Collection, + collection::{Collection, SyncCollection}, id::Id, keyword::Keyword, property::Property, @@ -28,11 +35,15 @@ use mail_parser::{ Header, HeaderName, HeaderValue, Message, MessageParser, PartType, parsers::fields::thread::thread_name, }; - use spam_filter::{ SpamFilterInput, analysis::init::SpamFilterInit, modules::bayes::BayesClassifier, }; use std::future::Future; +use std::{ + borrow::Cow, + fmt::Write, + time::{Duration, Instant}, +}; use store::{ BlobClass, IndexKey, IndexKeyPrefix, IterateParams, U32_LEN, ahash::AHashMap, @@ -42,22 +53,6 @@ use store::{ use store::{SerializeInfallible, rand::Rng}; use trc::{AddContext, MessageIngestEvent}; -use crate::{ - mailbox::{INBOX_ID, JUNK_ID, UidMailbox}, - message::{ - cache::MessageCacheFetch, - crypto::EncryptionParams, - index::{IndexMessage, MAX_ID_LENGTH, VisitValues}, - metadata::MessageData, - }, -}; - -use super::{ - cache::MessageCacheAccess, - crypto::{EncryptMessage, EncryptMessageError}, - index::{MAX_SORT_FIELD_LENGTH, TrimTextValue}, -}; - #[derive(Default)] pub struct IngestedEmail { pub id: Id, @@ -504,7 +499,7 @@ impl EmailIngest for Server { batch .with_collection(Collection::Thread) .update_document(thread_id) - .log_insert(None); + .log_container_insert(SyncCollection::Thread); } let document_id = self @@ -742,7 +737,7 @@ impl EmailIngest for Server { let mut thread_counts = AHashMap::::with_capacity(16); let mut thread_id = u32::MAX; let mut thread_count = 0; - for item in &cache.items { + for item in &cache.emails.items { if results.contains(item.document_id) { let tc = thread_counts.entry(item.thread_id).or_default(); *tc += 1; @@ -766,14 +761,16 @@ impl EmailIngest for Server { .with_collection(Collection::Thread); for &delete_thread_id in thread_counts.keys() { if delete_thread_id != thread_id { - batch.update_document(delete_thread_id).log_delete(None); + batch + .update_document(delete_thread_id) + .log_container_delete(SyncCollection::Thread); } } // Move messages to the new threadId batch.with_collection(Collection::Email); - for item in &cache.items { + for item in &cache.emails.items { if thread_id == item.thread_id || !thread_counts.contains_key(&item.thread_id) { continue; } diff --git a/crates/email/src/message/mod.rs b/crates/email/src/message/mod.rs index 14b634ca..e0ee70c5 100644 --- a/crates/email/src/message/mod.rs +++ b/crates/email/src/message/mod.rs @@ -5,7 +5,6 @@ */ pub mod bayes; -pub mod cache; pub mod copy; pub mod crypto; pub mod delete; diff --git a/crates/email/src/sieve/index.rs b/crates/email/src/sieve/index.rs index da46efbc..495293d9 100644 --- a/crates/email/src/sieve/index.rs +++ b/crates/email/src/sieve/index.rs @@ -5,7 +5,7 @@ */ use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject}; -use jmap_proto::types::property::Property; +use jmap_proto::types::{collection::SyncCollection, property::Property}; use super::{ArchivedSieveScript, SieveScript}; @@ -25,7 +25,10 @@ impl IndexableObject for SieveScript { IndexValue::Blob { value: self.blob_hash.clone(), }, - IndexValue::LogChild { prefix: None }, + IndexValue::LogItem { + sync_collection: SyncCollection::SieveScript.into(), + prefix: None, + }, IndexValue::Quota { used: self.size }, ] .into_iter() @@ -50,7 +53,10 @@ impl IndexableObject for &ArchivedSieveScript { IndexValue::Blob { value: (&self.blob_hash).into(), }, - IndexValue::LogChild { prefix: None }, + IndexValue::LogItem { + sync_collection: SyncCollection::SieveScript.into(), + prefix: None, + }, IndexValue::Quota { used: u32::from(self.size), }, diff --git a/crates/email/src/sieve/ingest.rs b/crates/email/src/sieve/ingest.rs index 98d0ae9f..3bc59652 100644 --- a/crates/email/src/sieve/ingest.rs +++ b/crates/email/src/sieve/ingest.rs @@ -4,14 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{borrow::Cow, sync::Arc}; - +use super::{ActiveScript, SeenIdHash, SieveScript}; use crate::{ - mailbox::{ - INBOX_ID, TRASH_ID, - cache::{MailboxCacheAccess, MessageMailboxCache}, - manage::MailboxFnc, - }, + cache::{MessageCacheFetch, mailbox::MailboxCacheAccess}, + mailbox::{INBOX_ID, TRASH_ID, manage::MailboxFnc}, message::{ delivery::AutogeneratedMessage, ingest::{EmailIngest, IngestEmail, IngestSource, IngestedEmail}, @@ -20,11 +16,12 @@ use crate::{ use common::{ Server, auth::AccessToken, config::jmap::settings::SpecialUse, scripts::plugins::PluginContext, }; - use directory::{Permission, QueryBy}; use jmap_proto::types::{collection::Collection, id::Id, keyword::Keyword, property::Property}; use mail_parser::MessageParser; use sieve::{Envelope, Event, Input, Mailbox, Recipient, Sieve}; +use std::future::Future; +use std::{borrow::Cow, sync::Arc}; use store::{ Deserialize, Serialize, SerializeInfallible, ahash::AHashMap, @@ -35,10 +32,6 @@ use store::{ use trc::{AddContext, SieveEvent}; use utils::config::utils::ParseValue; -use std::future::Future; - -use super::{ActiveScript, SeenIdHash, SieveScript}; - struct SieveMessage<'x> { pub raw_message: Cow<'x, [u8]>, pub file_into: Vec, @@ -101,8 +94,8 @@ impl SieveScriptIngest for Server { // Obtain mailboxIds let account_id = access_token.primary_id; - let mut mailbox_cache = self - .get_cached_mailboxes(account_id) + let mut cache = self + .get_cached_messages(account_id) .await .caused_by(trc::location!())?; @@ -190,7 +183,7 @@ impl SieveScriptIngest for Server { } else { let mut mailbox_id = u32::MAX; if let Ok(role) = SpecialUse::parse_value(&role) { - if let Some(m) = mailbox_cache.by_role(&role) { + if let Some(m) = cache.mailbox_by_role(&role) { mailbox_id = m.document_id; } } @@ -204,7 +197,7 @@ impl SieveScriptIngest for Server { match mailbox { Mailbox::Name(name) => { if !matches!( - mailbox_cache.by_path(&name), + cache.mailbox_by_path(&name), Some(item) if special_use_ids.is_empty() || special_use_ids.contains(&item.document_id) ) { @@ -214,7 +207,7 @@ impl SieveScriptIngest for Server { } Mailbox::Id(id) => { if !matches!(Id::from_bytes(id.as_bytes()), Some(id) if - mailbox_cache.has_id(&id.document_id()) && + cache.has_mailbox_id(&id.document_id()) && (special_use_ids.is_empty() || special_use_ids.contains(&id.document_id()))) { @@ -234,7 +227,7 @@ impl SieveScriptIngest for Server { { let role = SpecialUse::parse_value(&role); if role.is_err() - || mailbox_cache.by_role(&role.unwrap()).is_none() + || cache.mailbox_by_role(&role.unwrap()).is_none() { result = false; break; @@ -310,7 +303,7 @@ impl SieveScriptIngest for Server { mailbox_id.and_then(|m| Id::from_bytes(m.as_bytes())) { let mailbox_id = mailbox_id.document_id(); - if mailbox_cache.has_id(&mailbox_id) { + if cache.has_mailbox_id(&mailbox_id) { target_id = mailbox_id; } } @@ -323,7 +316,7 @@ impl SieveScriptIngest for Server { } else if special_use.eq_ignore_ascii_case("trash") { target_id = TRASH_ID; } else if let Ok(role) = SpecialUse::parse_value(&special_use) { - if let Some(item) = mailbox_cache.by_role(&role) { + if let Some(item) = cache.mailbox_by_role(&role) { target_id = item.document_id; } } @@ -333,7 +326,7 @@ impl SieveScriptIngest for Server { // Find mailbox by name if target_id == u32::MAX { if !create { - if let Some(m) = mailbox_cache.by_path(&folder) { + if let Some(m) = cache.mailbox_by_path(&folder) { target_id = m.document_id; } } else if let Some(document_id) = self @@ -341,8 +334,8 @@ impl SieveScriptIngest for Server { .await .caused_by(trc::location!())? { - mailbox_cache = self - .get_cached_mailboxes(account_id) + cache = self + .get_cached_messages(account_id) .await .caused_by(trc::location!())?; target_id = document_id; diff --git a/crates/email/src/submission/index.rs b/crates/email/src/submission/index.rs index dd5c65b6..355b0759 100644 --- a/crates/email/src/submission/index.rs +++ b/crates/email/src/submission/index.rs @@ -5,7 +5,7 @@ */ use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject}; -use jmap_proto::types::property::Property; +use jmap_proto::types::{collection::SyncCollection, property::Property}; use super::{ArchivedEmailSubmission, EmailSubmission}; @@ -32,7 +32,10 @@ impl IndexableObject for EmailSubmission { field: Property::SendAt.into(), value: self.send_at.into(), }, - IndexValue::LogChild { prefix: None }, + IndexValue::LogItem { + sync_collection: SyncCollection::EmailSubmission.into(), + prefix: None, + }, ] .into_iter() } @@ -61,7 +64,10 @@ impl IndexableObject for &ArchivedEmailSubmission { field: Property::SendAt.into(), value: self.send_at.into(), }, - IndexValue::LogChild { prefix: None }, + IndexValue::LogItem { + sync_collection: SyncCollection::EmailSubmission.into(), + prefix: None, + }, ] .into_iter() } diff --git a/crates/groupware/src/calendar/index.rs b/crates/groupware/src/calendar/index.rs index 5e551e96..f2e11d11 100644 --- a/crates/groupware/src/calendar/index.rs +++ b/crates/groupware/src/calendar/index.rs @@ -7,7 +7,7 @@ use common::storage::index::{ IndexItem, IndexValue, IndexableAndSerializableObject, IndexableObject, }; -use jmap_proto::types::value::AclGrant; +use jmap_proto::types::{collection::SyncCollection, value::AclGrant}; use store::{SerializeInfallible, write::key::KeySerializer}; use crate::{IDX_NAME, IDX_TIME, IDX_UID}; @@ -44,7 +44,9 @@ impl IndexableObject for Calendar { + self.default_alerts.iter().map(|a| a.size()).sum::() as u32 + self.name.len() as u32, }, - IndexValue::LogChild { prefix: None }, + IndexValue::LogContainer { + sync_collection: SyncCollection::Calendar.into(), + }, ] .into_iter() } @@ -80,7 +82,9 @@ impl IndexableObject for &ArchivedCalendar { + self.default_alerts.iter().map(|a| a.size()).sum::() as u32 + self.name.len() as u32, }, - IndexValue::LogChild { prefix: None }, + IndexValue::LogContainer { + sync_collection: SyncCollection::Calendar.into(), + }, ] .into_iter() } @@ -122,7 +126,10 @@ impl IndexableObject for CalendarEvent { + self.names.iter().map(|n| n.name.len() as u32).sum::() + self.size, }, - IndexValue::LogChild { prefix: None }, + IndexValue::LogItem { + sync_collection: SyncCollection::Calendar.into(), + prefix: None, + }, ] .into_iter() } @@ -162,7 +169,10 @@ impl IndexableObject for &ArchivedCalendarEvent { + self.names.iter().map(|n| n.name.len() as u32).sum::() + self.size, }, - IndexValue::LogChild { prefix: None }, + IndexValue::LogItem { + sync_collection: SyncCollection::Calendar.into(), + prefix: None, + }, ] .into_iter() } diff --git a/crates/groupware/src/contact/index.rs b/crates/groupware/src/contact/index.rs index b4115564..1de1e962 100644 --- a/crates/groupware/src/contact/index.rs +++ b/crates/groupware/src/contact/index.rs @@ -7,7 +7,7 @@ use common::storage::index::{ IndexItem, IndexValue, IndexableAndSerializableObject, IndexableObject, }; -use jmap_proto::types::value::AclGrant; +use jmap_proto::types::{collection::SyncCollection, value::AclGrant}; use store::SerializeInfallible; use crate::{IDX_NAME, IDX_UID}; @@ -32,7 +32,9 @@ impl IndexableObject for AddressBook { + self.description.as_ref().map_or(0, |n| n.len() as u32) + self.name.len() as u32, }, - IndexValue::LogChild { prefix: None }, + IndexValue::LogContainer { + sync_collection: SyncCollection::AddressBook.into(), + }, ] .into_iter() } @@ -59,7 +61,9 @@ impl IndexableObject for &ArchivedAddressBook { + self.description.as_ref().map_or(0, |n| n.len() as u32) + self.name.len() as u32, }, - IndexValue::LogChild { prefix: None }, + IndexValue::LogContainer { + sync_collection: SyncCollection::AddressBook.into(), + }, ] .into_iter() } @@ -88,7 +92,10 @@ impl IndexableObject for ContactCard { + self.names.iter().map(|n| n.name.len() as u32).sum::() + self.size, }, - IndexValue::LogChild { prefix: None }, + IndexValue::LogItem { + sync_collection: SyncCollection::AddressBook.into(), + prefix: None, + }, ] .into_iter() } @@ -115,7 +122,10 @@ impl IndexableObject for &ArchivedContactCard { + self.names.iter().map(|n| n.name.len() as u32).sum::() + self.size, }, - IndexValue::LogChild { prefix: None }, + IndexValue::LogItem { + sync_collection: SyncCollection::AddressBook.into(), + prefix: None, + }, ] .into_iter() } diff --git a/crates/groupware/src/file/index.rs b/crates/groupware/src/file/index.rs index 6f9e8687..59d2c831 100644 --- a/crates/groupware/src/file/index.rs +++ b/crates/groupware/src/file/index.rs @@ -8,7 +8,7 @@ use common::storage::{ folder::FolderHierarchy, index::{IndexValue, IndexableAndSerializableObject, IndexableObject}, }; -use jmap_proto::types::{property::Property, value::AclGrant}; +use jmap_proto::types::{collection::SyncCollection, property::Property, value::AclGrant}; use super::{ArchivedFileNode, FileNode}; @@ -36,7 +36,9 @@ impl IndexableObject for FileNode { IndexValue::Acl { value: (&self.acls).into(), }, - IndexValue::LogChild { prefix: None }, + IndexValue::LogContainer { + sync_collection: SyncCollection::FileNode.into(), + }, ]); if let Some(file) = &self.file { @@ -80,7 +82,9 @@ impl IndexableObject for &ArchivedFileNode { .collect::>() .into(), }, - IndexValue::LogChild { prefix: None }, + IndexValue::LogContainer { + sync_collection: SyncCollection::FileNode.into(), + }, ]); let size = self.size(); diff --git a/crates/groupware/src/hierarchy.rs b/crates/groupware/src/hierarchy.rs index dfe9c2ed..b514a7cc 100644 --- a/crates/groupware/src/hierarchy.rs +++ b/crates/groupware/src/hierarchy.rs @@ -60,6 +60,8 @@ impl DavHierarchy for Server { account_id: u32, collection: Collection, ) -> trc::Result> { + let todo = "fix"; + let is_files = collection == Collection::FileNode; let mut change_id = self .store() diff --git a/crates/imap/src/core/mailbox.rs b/crates/imap/src/core/mailbox.rs index e1ca972d..d16a27e7 100644 --- a/crates/imap/src/core/mailbox.rs +++ b/crates/imap/src/core/mailbox.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{Account, AccountState, MailboxId, MailboxSync, Session, SessionData}; +use super::{Account, MailboxId, MailboxSync, Session, SessionData}; use crate::core::Mailbox; use ahash::AHashMap; use common::{ @@ -16,11 +16,8 @@ use common::{ use directory::backend::internal::manage::ManageDirectory; use email::{ - mailbox::{ - INBOX_ID, - cache::{MailboxCacheAccess, MessageMailboxCache}, - }, - message::cache::{MessageCacheAccess, MessageCacheFetch}, + cache::{MessageCacheFetch, email::MessageCacheAccess, mailbox::MailboxCacheAccess}, + mailbox::INBOX_ID, }; use imap_proto::protocol::list::Attribute; use jmap_proto::types::{acl::Acl, collection::Collection, id::Id, keyword::Keyword}; @@ -90,21 +87,14 @@ impl SessionData { account_id: u32, mailbox_prefix: Option, access_token: &AccessToken, - current_state: Option, + current_state: Option, ) -> trc::Result> { - let cached_mailboxes = self - .server - .get_cached_mailboxes(account_id) - .await - .caused_by(trc::location!())?; - let cached_messages = self + let cache = self .server .get_cached_messages(account_id) .await .caused_by(trc::location!())?; - if current_state.is_some_and(|state| { - state.email == cached_messages.change_id && state.mailbox == cached_mailboxes.change_id - }) { + if current_state.is_some_and(|state| state == cache.last_change_id) { return Ok(None); } @@ -113,14 +103,12 @@ impl SessionData { { None } else { - cached_mailboxes - .shared_mailboxes(access_token, Acl::Read) - .into() + cache.shared_mailboxes(access_token, Acl::Read).into() }; // Build special uses let mut special_uses = AHashMap::new(); - for mailbox in &cached_mailboxes.items { + for mailbox in &cache.mailboxes.items { if shared_mailbox_ids .as_ref() .is_none_or(|ids| ids.contains(mailbox.document_id)) @@ -135,14 +123,11 @@ impl SessionData { account_id, prefix: mailbox_prefix, mailbox_names: BTreeMap::new(), - mailbox_state: AHashMap::with_capacity(cached_mailboxes.items.len()), - state: AccountState { - email: cached_messages.change_id, - mailbox: cached_mailboxes.change_id, - }, + mailbox_state: AHashMap::with_capacity(cache.mailboxes.items.len()), + last_change_id: cache.last_change_id, }; - for mailbox in &cached_mailboxes.items { + for mailbox in &cache.mailboxes.items { if shared_mailbox_ids .as_ref() .is_some_and(|ids| !ids.contains(mailbox.document_id)) @@ -176,7 +161,8 @@ impl SessionData { account.mailbox_state.insert( mailbox.document_id, Mailbox { - has_children: cached_mailboxes + has_children: cache + .mailboxes .items .iter() .any(|child| child.parent_id == mailbox.document_id), @@ -190,11 +176,11 @@ impl SessionData { SpecialUse::Important => Some(Attribute::Important), _ => None, }, - total_messages: cached_messages.in_mailbox(mailbox.document_id).count() as u64, - total_unseen: cached_messages + total_messages: cache.in_mailbox(mailbox.document_id).count() as u64, + total_unseen: cache .in_mailbox_without_keyword(mailbox.document_id, &Keyword::Seen) .count() as u64, - total_deleted: cached_messages + total_deleted: cache .in_mailbox_with_keyword(mailbox.document_id, &Keyword::Deleted) .count() as u64, uid_validity: mailbox.uid_validity as u64, @@ -301,7 +287,7 @@ impl SessionData { .mailboxes .lock() .iter() - .map(|m| (m.account_id, m.prefix.clone(), m.state)) + .map(|m| (m.account_id, m.prefix.clone(), m.last_change_id)) .collect::>(); for (account_id, prefix, last_state) in account_states { if let Some(changed_account) = self diff --git a/crates/imap/src/core/message.rs b/crates/imap/src/core/message.rs index 27eaf26d..b69f2cb7 100644 --- a/crates/imap/src/core/message.rs +++ b/crates/imap/src/core/message.rs @@ -6,7 +6,7 @@ use ahash::AHashMap; use common::listener::SessionStream; -use email::message::cache::MessageCacheFetch; +use email::cache::MessageCacheFetch; use imap_proto::protocol::{Sequence, expunge, select::Exists}; use jmap_proto::types::{collection::Collection, property::Property}; use std::collections::BTreeMap; @@ -31,12 +31,13 @@ impl SessionData { .await .caused_by(trc::location!())?; - if current_state.is_some_and(|state| state == cached_messages.change_id) { + if current_state.is_some_and(|state| state == cached_messages.emails.change_id) { return Ok(None); } // Obtain UID next and assign UIDs let uid_map = cached_messages + .emails .items .iter() .filter_map(|item| { @@ -72,7 +73,7 @@ impl SessionData { id_to_imap, uid_to_id, uid_max, - modseq: cached_messages.change_id, + modseq: cached_messages.emails.change_id, next_state: None, })) } @@ -161,21 +162,6 @@ impl SessionData { Ok(modseq) } - pub async fn get_modseq(&self, account_id: u32) -> trc::Result> { - // Obtain current modseq - self.server - .core - .storage - .data - .get_last_change_id(account_id, Collection::Email) - .await - .add_context(|e| { - e.caused_by(trc::location!()) - .account_id(account_id) - .collection(Collection::Email) - }) - } - pub async fn get_uid_next(&self, mailbox: &MailboxId) -> trc::Result { self.server .core diff --git a/crates/imap/src/core/mod.rs b/crates/imap/src/core/mod.rs index cc816119..ba3cf136 100644 --- a/crates/imap/src/core/mod.rs +++ b/crates/imap/src/core/mod.rs @@ -97,13 +97,7 @@ pub struct Account { pub prefix: Option, pub mailbox_names: BTreeMap, pub mailbox_state: AHashMap, - pub state: AccountState, -} - -#[derive(Debug, Clone, Copy, Default)] -pub struct AccountState { - pub email: u64, - pub mailbox: u64, + pub last_change_id: u64, } #[derive(Debug, Default, Clone)] diff --git a/crates/imap/src/op/create.rs b/crates/imap/src/op/create.rs index bb96017b..ab4f59c9 100644 --- a/crates/imap/src/op/create.rs +++ b/crates/imap/src/op/create.rs @@ -4,8 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::time::Instant; - use crate::{ core::{Session, SessionData}, op::ImapContext, @@ -14,15 +12,15 @@ use crate::{ use common::{ config::jmap::settings::SpecialUse, listener::SessionStream, storage::index::ObjectIndexBuilder, }; - use directory::Permission; -use email::mailbox::cache::{MailboxCacheAccess, MessageMailboxCache}; +use email::cache::{MessageCacheFetch, mailbox::MailboxCacheAccess}; use imap_proto::{ Command, ResponseCode, StatusResponse, protocol::{create::Arguments, list::Attribute}, receiver::Request, }; use jmap_proto::types::{acl::Acl, collection::Collection, id::Id}; +use std::time::Instant; use store::write::BatchBuilder; use trc::AddContext; @@ -280,10 +278,10 @@ impl SessionData { let special_use = attr_to_role(mailbox_role); if self .server - .get_cached_mailboxes(account_id) + .get_cached_messages(account_id) .await .caused_by(trc::location!())? - .by_role(&special_use) + .mailbox_by_role(&special_use) .is_some() { return Err(trc::ImapEvent::Error diff --git a/crates/imap/src/op/expunge.rs b/crates/imap/src/op/expunge.rs index 6ccba07b..b38d59fc 100644 --- a/crates/imap/src/op/expunge.rs +++ b/crates/imap/src/op/expunge.rs @@ -4,28 +4,24 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{sync::Arc, time::Instant}; - +use super::{ImapContext, ToModSeq}; +use crate::core::{ImapId, SavedSearch, SelectedMailbox, Session, SessionData}; use ahash::AHashMap; +use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; -use email::message::{ - cache::{MessageCacheFetch, MessageCacheAccess}, - delete::EmailDeletion, - metadata::MessageData, +use email::{ + cache::{MessageCacheFetch, email::MessageCacheAccess}, + message::{delete::EmailDeletion, metadata::MessageData}, }; use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, parser::parse_sequence_set, receiver::{Request, Token}, }; -use trc::AddContext; - -use crate::core::{ImapId, SavedSearch, SelectedMailbox, Session, SessionData}; -use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use jmap_proto::types::{acl::Acl, collection::Collection, keyword::Keyword}; +use std::{sync::Arc, time::Instant}; use store::{roaring::RoaringBitmap, write::BatchBuilder}; - -use super::{ImapContext, ToModSeq}; +use trc::AddContext; impl Session { pub async fn handle_expunge( diff --git a/crates/imap/src/op/fetch.rs b/crates/imap/src/op/fetch.rs index 9bd1ed15..dc4097d5 100644 --- a/crates/imap/src/op/fetch.rs +++ b/crates/imap/src/op/fetch.rs @@ -4,8 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{borrow::Cow, sync::Arc, time::Instant}; - +use super::{FromModSeq, ImapContext}; use crate::{ core::{SelectedMailbox, Session, SessionData}, spawn_op, @@ -13,9 +12,9 @@ use crate::{ use ahash::AHashMap; use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; -use email::message::{ - cache::{MessageCacheAccess, MessageCacheFetch}, - metadata::{ +use email::{ + cache::{MessageCacheFetch, email::MessageCacheAccess}, + message::metadata::{ ArchivedAddress, ArchivedGetHeader, ArchivedHeaderName, ArchivedHeaderValue, ArchivedMessageMetadata, ArchivedMessageMetadataContents, ArchivedMetadataPartType, DecodedParts, MessageData, MessageMetadata, @@ -35,16 +34,19 @@ use imap_proto::{ receiver::Request, }; use jmap_proto::types::{ - acl::Acl, collection::Collection, id::Id, keyword::Keyword, property::Property, + acl::Acl, + collection::{Collection, SyncCollection}, + id::Id, + keyword::Keyword, + property::Property, }; +use std::{borrow::Cow, sync::Arc, time::Instant}; use store::{ query::log::{Change, Query}, rkyv::rend::u16_le, write::BatchBuilder, }; -use super::{FromModSeq, ImapContext}; - impl Session { pub async fn handle_fetch(&mut self, requests: Vec>) -> trc::Result<()> { // Validate access @@ -155,7 +157,7 @@ impl SessionData { .store() .changes( account_id, - Collection::Email, + SyncCollection::Email, Query::from_modseq(changed_since), ) .await @@ -167,18 +169,19 @@ impl SessionData { for change in changelog.changes { match change { - Change::Insert(id) | Change::Update(id) => { + Change::InsertItem(id) | Change::UpdateItem(id) => { let id = (id & u32::MAX as u64) as u32; if let Some(uid) = ids.get(&id) { changed_ids.insert(id, *uid); } if !has_vanished { - has_vanished = matches!(change, Change::Update(_)); + has_vanished = matches!(change, Change::UpdateItem(_)); } } - Change::Delete(_) => { + Change::DeleteItem(_) => { has_vanished = true; } + _ => (), } } @@ -314,7 +317,7 @@ impl SessionData { ) .await .imap_ctx(&arguments.tag, trc::location!())?, - message_cache.by_id(&id), + message_cache.email_by_id(&id), ) { (email, data) } else { diff --git a/crates/imap/src/op/idle.rs b/crates/imap/src/op/idle.rs index cf35d80e..55785095 100644 --- a/crates/imap/src/op/idle.rs +++ b/crates/imap/src/op/idle.rs @@ -4,9 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{sync::Arc, time::Instant}; - +use crate::{ + core::{SelectedMailbox, Session, SessionData, State}, + op::ImapContext, +}; use ahash::AHashSet; +use common::listener::SessionStream; use directory::Permission; use imap_proto::{ Command, StatusResponse, @@ -17,19 +20,13 @@ use imap_proto::{ }, receiver::Request, }; - -use common::listener::SessionStream; -use jmap_proto::types::{collection::Collection, type_state::DataType}; +use jmap_proto::types::{collection::SyncCollection, type_state::DataType}; +use std::{sync::Arc, time::Instant}; use store::query::log::Query; use tokio::io::AsyncReadExt; use trc::AddContext; use utils::map::bitmap::Bitmap; -use crate::{ - core::{SelectedMailbox, Session, SessionData, State}, - op::ImapContext, -}; - impl Session { pub async fn handle_idle(&mut self, request: Request) -> trc::Result<()> { // Validate access @@ -205,7 +202,7 @@ impl SessionData { .store() .changes( mailbox.id.account_id, - Collection::Email, + SyncCollection::Email, Query::Since(modseq), ) .await @@ -216,10 +213,12 @@ impl SessionData { .changes .into_iter() .filter_map(|change| { - state - .id_to_imap - .get(&((change.unwrap_id() & u32::MAX as u64) as u32)) - .map(|id| id.uid) + change.try_unwrap_item_id().and_then(|item_id| { + state + .id_to_imap + .get(&((item_id & u32::MAX as u64) as u32)) + .map(|id| id.uid) + }) }) .collect::>() }; diff --git a/crates/imap/src/op/search.rs b/crates/imap/src/op/search.rs index 62703e69..e388892e 100644 --- a/crates/imap/src/op/search.rs +++ b/crates/imap/src/op/search.rs @@ -4,11 +4,14 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{sync::Arc, time::Instant}; - +use super::{FromModSeq, ToModSeq}; +use crate::{ + core::{ImapId, SavedSearch, SelectedMailbox, Session, SessionData}, + spawn_op, +}; use common::listener::SessionStream; use directory::Permission; -use email::message::cache::{MessageCacheAccess, MessageCacheFetch}; +use email::cache::{MessageCacheFetch, email::MessageCacheAccess}; use imap_proto::{ Command, StatusResponse, protocol::{ @@ -17,9 +20,15 @@ use imap_proto::{ }, receiver::Request, }; -use jmap_proto::types::{collection::Collection, id::Id, keyword::Keyword, property::Property}; +use jmap_proto::types::{ + collection::{Collection, SyncCollection}, + id::Id, + keyword::Keyword, + property::Property, +}; use mail_parser::HeaderName; use nlp::language::Language; +use std::{sync::Arc, time::Instant}; use store::{ SerializeInfallible, fts::{Field, FilterGroup, FtsFilter, IntoFilterGroup}, @@ -30,13 +39,6 @@ use store::{ use tokio::sync::watch; use trc::AddContext; -use crate::{ - core::{ImapId, SavedSearch, SelectedMailbox, Session, SessionData}, - spawn_op, -}; - -use super::{FromModSeq, ToModSeq}; - impl Session { pub async fn handle_search( &mut self, @@ -626,18 +628,20 @@ impl SessionData { } search::Filter::ModSeq((modseq, _)) => { let mut set = RoaringBitmap::new(); - for change in self + for id in self .server .store() .changes( mailbox.id.account_id, - Collection::Email, + SyncCollection::Email, Query::from_modseq(modseq), ) .await? .changes + .into_iter() + .filter_map(|change| change.try_unwrap_item_id()) { - let id = (change.unwrap_id() & u32::MAX as u64) as u32; + let id = (id & u32::MAX as u64) as u32; if message_ids.contains(id) { set.insert(id); } diff --git a/crates/imap/src/op/status.rs b/crates/imap/src/op/status.rs index f5969a10..b574497f 100644 --- a/crates/imap/src/op/status.rs +++ b/crates/imap/src/op/status.rs @@ -4,17 +4,15 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::time::Instant; - +use super::ToModSeq; use crate::{ core::{Mailbox, Session, SessionData}, op::ImapContext, spawn_op, }; use common::listener::SessionStream; - use directory::Permission; -use email::message::cache::{MessageCacheAccess, MessageCacheFetch}; +use email::cache::{MessageCacheFetch, email::MessageCacheAccess}; use imap_proto::{ Command, ResponseCode, StatusResponse, parser::PushUnique, @@ -22,14 +20,13 @@ use imap_proto::{ receiver::Request, }; use jmap_proto::types::{collection::Collection, id::Id, keyword::Keyword, property::Property}; +use std::time::Instant; use store::{Deserialize, U32_LEN}; use store::{ IndexKeyPrefix, IterateParams, roaring::RoaringBitmap, write::key::DeserializeBigEndian, }; use trc::AddContext; -use super::ToModSeq; - impl Session { pub async fn handle_status(&mut self, requests: Vec>) -> trc::Result<()> { // Validate access @@ -188,7 +185,7 @@ impl SessionData { Status::HighestModSeq => { items_response.push(( *item, - StatusItemType::Number(account.state.email.to_modseq()), + StatusItemType::Number(account.last_change_id.to_modseq()), )); } Status::MailboxId => { diff --git a/crates/imap/src/op/store.rs b/crates/imap/src/op/store.rs index 58561f54..bfde6b3a 100644 --- a/crates/imap/src/op/store.rs +++ b/crates/imap/src/op/store.rs @@ -4,8 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{sync::Arc, time::Instant}; - +use super::{FromModSeq, ImapContext}; use crate::{ core::{SelectedMailbox, Session, SessionData}, spawn_op, @@ -23,15 +22,18 @@ use imap_proto::{ }, receiver::Request, }; -use jmap_proto::types::{acl::Acl, collection::Collection, keyword::Keyword}; +use jmap_proto::types::{ + acl::Acl, + collection::{Collection, SyncCollection}, + keyword::Keyword, +}; +use std::{sync::Arc, time::Instant}; use store::{ query::log::{Change, Query}, write::{BatchBuilder, ValueClass}, }; use trc::AddContext; -use super::{FromModSeq, ImapContext}; - impl Session { pub async fn handle_store( &mut self, @@ -112,7 +114,7 @@ impl SessionData { .store() .changes( account_id, - Collection::Email, + SyncCollection::Email, Query::from_modseq(unchanged_since), ) .await @@ -123,15 +125,20 @@ impl SessionData { .await; // Add all IDs that changed in this mailbox - for change in changelog.changes { - let (Change::Insert(id) | Change::Update(id) | Change::Delete(id)) = change; - let id = (id & u32::MAX as u64) as u32; + for (id, is_delete) in changelog.changes.into_iter().filter_map(|change| { + change.item_id().map(|id| { + ( + (id & u32::MAX as u64) as u32, + matches!(change, Change::DeleteItem(_)), + ) + }) + }) { if let Some(imap_id) = ids.remove(&id) { if is_uid { modified.push(imap_id.uid); } else { modified.push(imap_id.seqnum); - if matches!(change, Change::Delete(_)) { + if is_delete { unchanged_failed = true; } } @@ -345,7 +352,7 @@ impl SessionData { // Log mailbox changes if !changed_mailboxes.is_empty() { for parent_id in changed_mailboxes { - batch.log_parent_update(Collection::Mailbox.as_child_update(), parent_id); + batch.log_container_property_change(SyncCollection::Email, parent_id); } } diff --git a/crates/imap/src/op/thread.rs b/crates/imap/src/op/thread.rs index f4ddf723..3b28365b 100644 --- a/crates/imap/src/op/thread.rs +++ b/crates/imap/src/op/thread.rs @@ -4,8 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{sync::Arc, time::Instant}; - use crate::{ core::{SelectedMailbox, Session, SessionData}, spawn_op, @@ -13,7 +11,7 @@ use crate::{ use ahash::AHashMap; use common::listener::SessionStream; use directory::Permission; -use email::message::cache::MessageCacheFetch; +use email::cache::MessageCacheFetch; use imap_proto::{ Command, StatusResponse, protocol::{ @@ -22,6 +20,7 @@ use imap_proto::{ }, receiver::Request, }; +use std::{sync::Arc, time::Instant}; use trc::AddContext; impl Session { @@ -89,7 +88,7 @@ impl SessionData { // Group messages by thread let mut threads: AHashMap> = AHashMap::new(); let state = mailbox.state.lock(); - for item in &cache.items { + for item in &cache.emails.items { if result_set.results.contains(item.document_id) { if let Some((imap_id, _)) = state.map_result_id(item.document_id, is_uid) { threads.entry(item.thread_id).or_default().push(imap_id); diff --git a/crates/jmap-proto/src/types/collection.rs b/crates/jmap-proto/src/types/collection.rs index 5dc2e422..d796b05e 100644 --- a/crates/jmap-proto/src/types/collection.rs +++ b/crates/jmap-proto/src/types/collection.rs @@ -34,6 +34,21 @@ pub enum Collection { None = 13, } +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Default)] +#[repr(u8)] +pub enum SyncCollection { + Email = 0, + Thread = 1, + Calendar = 2, + AddressBook = 3, + FileNode = 4, + Identity = 5, + EmailSubmission = 6, + SieveScript = 7, + #[default] + None = 8, +} + impl Collection { pub fn main_collection(&self) -> Collection { match self { @@ -96,6 +111,22 @@ impl From for Collection { } } +impl From for SyncCollection { + fn from(v: u8) -> Self { + match v { + 0 => SyncCollection::Email, + 1 => SyncCollection::Thread, + 2 => SyncCollection::Calendar, + 3 => SyncCollection::AddressBook, + 4 => SyncCollection::FileNode, + 5 => SyncCollection::Identity, + 6 => SyncCollection::EmailSubmission, + 7 => SyncCollection::SieveScript, + _ => SyncCollection::None, + } + } +} + impl From for Collection { fn from(v: u64) -> Self { match v { @@ -123,6 +154,12 @@ impl From for u8 { } } +impl From for u8 { + fn from(v: SyncCollection) -> Self { + v as u8 + } +} + impl From for u64 { fn from(collection: Collection) -> u64 { collection as u64 @@ -171,10 +208,6 @@ impl Collection { Collection::None => "", } } - - pub fn as_child_update(&self) -> u8 { - u8::MAX - u8::from(*self) - } } impl FromStr for Collection { diff --git a/crates/jmap-proto/src/types/type_state.rs b/crates/jmap-proto/src/types/type_state.rs index 98473132..480e5613 100644 --- a/crates/jmap-proto/src/types/type_state.rs +++ b/crates/jmap-proto/src/types/type_state.rs @@ -172,25 +172,21 @@ impl TryFrom<&str> for DataType { } } -impl TryFrom for DataType { - type Error = (); - - fn try_from(value: ShortId) -> Result { - const MAILBOX_CHANGE: u8 = u8::MAX - 1; - match value.0 { - 0 => Ok(DataType::Email), - 1 | MAILBOX_CHANGE => Ok(DataType::Mailbox), - 2 => Ok(DataType::Thread), - 3 => Ok(DataType::Identity), - 4 => Ok(DataType::EmailSubmission), - 5 => Ok(DataType::SieveScript), - 6 => Ok(DataType::PushSubscription), - 8 => Ok(DataType::Calendar), - 9 => Ok(DataType::CalendarEvent), - 10 => Ok(DataType::AddressBook), - 11 => Ok(DataType::ContactCard), - 12 => Ok(DataType::FileNode), - _ => Err(()), +impl DataType { + pub fn try_from_id(value: ShortId, is_container: bool) -> Option { + match (value.0, is_container) { + (0, false) => DataType::Email.into(), + (0, true) => DataType::Mailbox.into(), + (1, _) => DataType::Thread.into(), + (2, true) => DataType::Calendar.into(), + (2, false) => DataType::CalendarEvent.into(), + (3, true) => DataType::AddressBook.into(), + (3, false) => DataType::ContactCard.into(), + (4, _) => DataType::FileNode.into(), + (5, _) => DataType::Identity.into(), + (6, _) => DataType::EmailSubmission.into(), + (7, _) => DataType::SieveScript.into(), + _ => None, } } } diff --git a/crates/jmap/src/blob/download.rs b/crates/jmap/src/blob/download.rs index 9189f833..475186f7 100644 --- a/crates/jmap/src/blob/download.rs +++ b/crates/jmap/src/blob/download.rs @@ -4,15 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::ops::Range; - use common::{Server, auth::AccessToken}; -use email::{ - mailbox::cache::MessageMailboxCache, - message::cache::{MessageCacheFetch, MessageCacheAccess}, -}; +use email::cache::MessageCacheFetch; +use email::cache::email::MessageCacheAccess; use jmap_proto::types::{acl::Acl, blob::BlobId, collection::Collection}; use std::future::Future; +use std::ops::Range; use store::BlobClass; use trc::AddContext; use utils::BlobHash; @@ -67,14 +64,7 @@ impl BlobDownload for Server { .get_cached_messages(*account_id) .await .caused_by(trc::location!())? - .shared_messages( - access_token, - self.get_cached_mailboxes(*account_id) - .await - .caused_by(trc::location!())? - .as_ref(), - Acl::ReadItems, - ) + .shared_messages(access_token, Acl::ReadItems) .contains(*document_id) { return Ok(None); @@ -142,14 +132,7 @@ impl BlobDownload for Server { .get_cached_messages(*account_id) .await .caused_by(trc::location!())? - .shared_messages( - access_token, - self.get_cached_mailboxes(*account_id) - .await - .caused_by(trc::location!())? - .as_ref(), - Acl::ReadItems, - ) + .shared_messages(access_token, Acl::ReadItems) .contains(*document_id) } else { access_token.is_member(*account_id) diff --git a/crates/jmap/src/changes/get.rs b/crates/jmap/src/changes/get.rs index 541a4a42..49051cba 100644 --- a/crates/jmap/src/changes/get.rs +++ b/crates/jmap/src/changes/get.rs @@ -7,10 +7,14 @@ use common::{Server, auth::AccessToken}; use jmap_proto::{ method::changes::{ChangesRequest, ChangesResponse, RequestArguments}, - types::{collection::Collection, property::Property, state::State}, + types::{ + collection::{Collection, SyncCollection}, + property::Property, + state::State, + }, }; use std::future::Future; -use store::query::log::{Change, Changes, Query}; +use store::query::log::{Change, Query}; pub trait ChangesLookup: Sync + Send { fn changes( @@ -27,30 +31,30 @@ impl ChangesLookup for Server { access_token: &AccessToken, ) -> trc::Result { // Map collection and validate ACLs - let collection = match request.arguments { + let (collection, is_container) = match request.arguments { RequestArguments::Email => { access_token.assert_has_access(request.account_id, Collection::Email)?; - Collection::Email + (SyncCollection::Email, false) } RequestArguments::Mailbox => { access_token.assert_has_access(request.account_id, Collection::Mailbox)?; - Collection::Mailbox + (SyncCollection::Email, true) } RequestArguments::Thread => { access_token.assert_has_access(request.account_id, Collection::Email)?; - Collection::Thread + (SyncCollection::Thread, true) } RequestArguments::Identity => { access_token.assert_is_member(request.account_id)?; - Collection::Identity + (SyncCollection::Identity, false) } RequestArguments::EmailSubmission => { access_token.assert_is_member(request.account_id)?; - Collection::EmailSubmission + (SyncCollection::EmailSubmission, false) } RequestArguments::Quota => { access_token.assert_is_member(request.account_id)?; @@ -59,13 +63,13 @@ impl ChangesLookup for Server { } }; - let max_changes = if self.core.jmap.changes_max_results > 0 - && self.core.jmap.changes_max_results < request.max_changes.unwrap_or(0) - { - self.core.jmap.changes_max_results - } else { - request.max_changes.unwrap_or(0) - }; + let max_changes = std::cmp::min( + request + .max_changes + .filter(|n| *n != 0) + .unwrap_or(usize::MAX), + self.core.jmap.changes_max_results.unwrap_or(usize::MAX), + ); let mut response = ChangesResponse { account_id: request.account_id, old_state: request.since_state.clone(), @@ -78,10 +82,12 @@ impl ChangesLookup for Server { }; let account_id = request.account_id.document_id(); - let (items_sent, mut changelog) = match &request.since_state { + let (items_sent, changelog) = match &request.since_state { State::Initial => { - let changelog = - changes(self, account_id, collection, Query::All, &mut response).await?; + let changelog = self + .store() + .changes(account_id, collection, Query::All) + .await?; if changelog.changes.is_empty() && changelog.from_change_id == 0 { return Ok(response); } @@ -90,81 +96,108 @@ impl ChangesLookup for Server { } State::Exact(change_id) => ( 0, - changes( - self, - account_id, - collection, - Query::Since(*change_id), - &mut response, - ) - .await?, + self.store() + .changes(account_id, collection, Query::Since(*change_id)) + .await?, ), State::Intermediate(intermediate_state) => { - let mut changelog = changes( - self, - account_id, - collection, - Query::RangeInclusive(intermediate_state.from_id, intermediate_state.to_id), - &mut response, - ) - .await?; - if intermediate_state.items_sent >= changelog.changes.len() { + let changelog = self + .store() + .changes( + account_id, + collection, + Query::RangeInclusive(intermediate_state.from_id, intermediate_state.to_id), + ) + .await?; + if (is_container + && intermediate_state.items_sent >= changelog.total_container_changes()) + || (!is_container + && intermediate_state.items_sent >= changelog.total_item_changes()) + { ( 0, - changes( - self, - account_id, - collection, - Query::Since(intermediate_state.to_id), - &mut response, - ) - .await?, + self.store() + .changes( + account_id, + collection, + Query::Since(intermediate_state.to_id), + ) + .await?, ) } else { - changelog.changes.drain( - (changelog.changes.len() - intermediate_state.items_sent) - ..changelog.changes.len(), - ); (intermediate_state.items_sent, changelog) } } }; - if max_changes > 0 && changelog.changes.len() > max_changes { - changelog - .changes - .drain(0..(changelog.changes.len() - max_changes)); - response.has_more_changes = true; - }; + let mut changes = changelog + .changes + .into_iter() + .filter(|change| { + (is_container && change.is_container_change()) + || (!is_container && change.is_item_change()) + }) + .skip(items_sent) + .peekable(); - let total_changes = changelog.changes.len(); - if total_changes > 0 { - for change in changelog.changes { - match change { - Change::Insert(item) => response.created.push(item.into()), - Change::Update(item) => response.updated.push(item.into()), - Change::Delete(item) => response.destroyed.push(item.into()), - }; - } + let mut items_changed = false; + for change in (&mut changes).take(max_changes) { + match change { + Change::InsertContainer(item) | Change::InsertItem(item) => { + response.created.push(item.into()); + } + Change::UpdateContainer(item) | Change::UpdateItem(item) => { + response.updated.push(item.into()); + items_changed = true; + } + Change::DeleteContainer(item) | Change::DeleteItem(item) => { + response.destroyed.push(item.into()); + } + Change::UpdateContainerProperty(item) => { + response.updated.push(item.into()); + } + }; } + + let change_id = (if is_container { + changelog.container_change_id + } else { + changelog.item_change_id + }) + .unwrap_or(changelog.to_change_id); + + response.has_more_changes = changes.peek().is_some(); response.new_state = if response.has_more_changes { State::new_intermediate( changelog.from_change_id, - changelog.to_change_id, + change_id, items_sent + max_changes, ) } else { - State::new_exact(changelog.to_change_id) + State::new_exact(change_id) }; + if is_container + && !response.updated.is_empty() + && !items_changed + && collection == SyncCollection::Email + { + response.updated_properties = vec![ + Property::TotalEmails, + Property::UnreadEmails, + Property::TotalThreads, + Property::UnreadThreads, + ] + .into() + } Ok(response) } } -async fn changes( +/*async fn changes( server: &Server, account_id: u32, - collection: Collection, + collection: SyncCollection, query: Query, response: &mut ChangesResponse, ) -> trc::Result { @@ -207,3 +240,4 @@ async fn changes( } Ok(main_changes) } +*/ diff --git a/crates/jmap/src/changes/state.rs b/crates/jmap/src/changes/state.rs index 266ef2bc..c841ff6e 100644 --- a/crates/jmap/src/changes/state.rs +++ b/crates/jmap/src/changes/state.rs @@ -4,8 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::Server; -use jmap_proto::types::{collection::Collection, state::State}; +use common::{MessageStoreCache, Server}; +use jmap_proto::types::{collection::SyncCollection, state::State}; use std::future::Future; use trc::AddContext; @@ -13,24 +13,25 @@ pub trait StateManager: Sync + Send { fn get_state( &self, account_id: u32, - collection: impl Into + Send, + collection: SyncCollection, ) -> impl Future> + Send; fn assert_state( &self, account_id: u32, - collection: Collection, + collection: SyncCollection, if_in_state: &Option, ) -> impl Future> + Send; } +pub trait MessageCacheState: Sync + Send { + fn get_state(&self, is_mailbox: bool) -> State; + + fn assert_state(&self, is_mailbox: bool, if_in_state: &Option) -> trc::Result; +} + impl StateManager for Server { - async fn get_state( - &self, - account_id: u32, - collection: impl Into + Send, - ) -> trc::Result { - let collection = collection.into(); + async fn get_state(&self, account_id: u32, collection: SyncCollection) -> trc::Result { self.core .storage .data @@ -43,7 +44,7 @@ impl StateManager for Server { async fn assert_state( &self, account_id: u32, - collection: Collection, + collection: SyncCollection, if_in_state: &Option, ) -> trc::Result { let old_state: State = self.get_state(account_id, collection).await?; @@ -56,3 +57,23 @@ impl StateManager for Server { Ok(old_state) } } + +impl MessageCacheState for MessageStoreCache { + fn get_state(&self, is_mailbox: bool) -> State { + if is_mailbox { + State::from(self.mailboxes.change_id) + } else { + State::from(self.emails.change_id) + } + } + + fn assert_state(&self, is_mailbox: bool, if_in_state: &Option) -> trc::Result { + let old_state: State = self.get_state(is_mailbox); + if let Some(if_in_state) = if_in_state { + if &old_state != if_in_state { + return Err(trc::JmapEvent::StateMismatch.into_err()); + } + } + Ok(old_state) + } +} diff --git a/crates/jmap/src/email/copy.rs b/crates/jmap/src/email/copy.rs index 8be5ec1d..f00c75c4 100644 --- a/crates/jmap/src/email/copy.rs +++ b/crates/jmap/src/email/copy.rs @@ -4,14 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::changes::state::MessageCacheState; use common::{Server, auth::AccessToken}; - use email::{ - mailbox::cache::{MailboxCacheAccess, MessageMailboxCache}, - message::{ - cache::{MessageCacheAccess, MessageCacheFetch}, - copy::EmailCopy, - }, + cache::{MessageCacheFetch, email::MessageCacheAccess, mailbox::MailboxCacheAccess}, + message::copy::EmailCopy, }; use http_proto::HttpSessionData; use jmap_proto::{ @@ -28,15 +25,12 @@ use jmap_proto::{ response::references::EvalObjectReferences, types::{ acl::Acl, - collection::Collection, property::Property, value::{MaybePatchValue, Value}, }, }; -use trc::AddContext; - -use crate::changes::state::StateManager; use std::future::Future; +use trc::AddContext; use utils::map::vec_map::VecMap; pub trait JmapEmailCopy: Sync + Send { @@ -65,9 +59,8 @@ impl JmapEmailCopy for Server { .into_err() .details("From accountId is equal to fromAccountId")); } - let old_state = self - .assert_state(account_id, Collection::Email, &request.if_in_state) - .await?; + let cache = self.get_cached_messages(account_id).await?; + let old_state = cache.assert_state(false, &request.if_in_state)?; let mut response = CopyResponse { from_account_id: request.from_account_id, account_id: request.account_id, @@ -77,29 +70,18 @@ impl JmapEmailCopy for Server { not_created: VecMap::new(), }; - let from_cached_messages = self + let from_cache = self .get_cached_messages(from_account_id) .await .caused_by(trc::location!())?; let from_message_ids = if access_token.is_member(from_account_id) { - from_cached_messages.document_ids() + from_cache.email_document_ids() } else { - let from_cached_mailboxes = self - .get_cached_mailboxes(from_account_id) - .await - .caused_by(trc::location!())?; - from_cached_messages.shared_messages( - access_token, - &from_cached_mailboxes, - Acl::ReadItems, - ) + from_cache.shared_messages(access_token, Acl::ReadItems) }; - let cached_mailboxes = self.get_cached_mailboxes(account_id).await?; let can_add_mailbox_ids = if access_token.is_shared(account_id) { - cached_mailboxes - .shared_mailboxes(access_token, Acl::AddItems) - .into() + cache.shared_mailboxes(access_token, Acl::AddItems).into() } else { None }; @@ -205,7 +187,7 @@ impl JmapEmailCopy for Server { // Verify that the mailboxIds are valid for mailbox_id in &mailboxes { - if !cached_mailboxes.has_id(mailbox_id) { + if !cache.has_mailbox_id(mailbox_id) { response.not_created.append( id, SetError::invalid_properties() @@ -253,7 +235,7 @@ impl JmapEmailCopy for Server { // Update state if !response.created.is_empty() { - response.new_state = self.get_state(account_id, Collection::Email).await?; + response.new_state = self.get_cached_messages(account_id).await?.get_state(false); } // Destroy ids diff --git a/crates/jmap/src/email/get.rs b/crates/jmap/src/email/get.rs index 5321723e..1b500496 100644 --- a/crates/jmap/src/email/get.rs +++ b/crates/jmap/src/email/get.rs @@ -4,15 +4,18 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::{ + body::{ToBodyPart, truncate_html, truncate_plain}, + headers::IntoForm, +}; +use crate::{ + blob::download::BlobDownload, changes::state::MessageCacheState, email::headers::HeaderToValue, +}; use common::{Server, auth::AccessToken}; - use email::{ - mailbox::cache::MessageMailboxCache, - message::{ - cache::{MessageCacheAccess, MessageCacheFetch}, - metadata::{ - ArchivedGetHeader, ArchivedHeaderName, ArchivedMetadataPartType, MessageMetadata, - }, + cache::{MessageCacheFetch, email::MessageCacheAccess}, + message::metadata::{ + ArchivedGetHeader, ArchivedHeaderName, ArchivedMetadataPartType, MessageMetadata, }, }; use jmap_proto::{ @@ -28,21 +31,11 @@ use jmap_proto::{ value::{Object, Value}, }, }; - +use std::{borrow::Cow, future::Future}; use store::BlobClass; use trc::{AddContext, StoreEvent}; use utils::BlobHash; -use crate::{ - blob::download::BlobDownload, changes::state::StateManager, email::headers::HeaderToValue, -}; -use std::{borrow::Cow, future::Future}; - -use super::{ - body::{ToBodyPart, truncate_html, truncate_plain}, - headers::IntoForm, -}; - pub trait EmailGet: Sync + Send { fn email_get( &self, @@ -104,24 +97,21 @@ impl EmailGet for Server { let max_body_value_bytes = request.arguments.max_body_value_bytes.unwrap_or(0); let account_id = request.account_id.document_id(); - let cached_messages = self + let cache = self .get_cached_messages(account_id) .await .caused_by(trc::location!())?; let message_ids = if access_token.is_member(account_id) { - cached_messages.document_ids() + cache.email_document_ids() } else { - let cached_mailboxes = self - .get_cached_mailboxes(account_id) - .await - .caused_by(trc::location!())?; - cached_messages.shared_messages(access_token, &cached_mailboxes, Acl::ReadItems) + cache.shared_messages(access_token, Acl::ReadItems) }; let ids = if let Some(ids) = ids { ids } else { - cached_messages + cache + .emails .items .iter() .take(self.core.jmap.get_max_objects) @@ -130,7 +120,7 @@ impl EmailGet for Server { }; let mut response = GetResponse { account_id: request.account_id.into(), - state: self.get_state(account_id, Collection::Email).await?.into(), + state: cache.get_state(false).into(), list: Vec::with_capacity(ids.len()), not_found: vec![], }; @@ -177,7 +167,7 @@ impl EmailGet for Server { .caused_by(trc::location!())?; // Obtain message data - let data = match cached_messages.by_id(&id.document_id()) { + let data = match cache.email_by_id(&id.document_id()) { Some(data) => data, None => { response.not_found.push(id.into()); @@ -243,7 +233,7 @@ impl EmailGet for Server { } Property::Keywords => { let mut obj = Object::with_capacity(2); - for keyword in cached_messages.expand_keywords(data) { + for keyword in cache.expand_keywords(data) { obj.append(Property::_T(keyword.to_string()), true); } email.append(property.clone(), Value::Object(obj)); diff --git a/crates/jmap/src/email/import.rs b/crates/jmap/src/email/import.rs index 36537a23..8065dc2d 100644 --- a/crates/jmap/src/email/import.rs +++ b/crates/jmap/src/email/import.rs @@ -4,23 +4,21 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::{blob::download::BlobDownload, changes::state::MessageCacheState}; use common::{Server, auth::AccessToken}; use email::{ - mailbox::cache::{MailboxCacheAccess, MessageMailboxCache}, + cache::{MessageCacheFetch, mailbox::MailboxCacheAccess}, message::ingest::{EmailIngest, IngestEmail, IngestSource}, }; use http_proto::HttpSessionData; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::import::{ImportEmailRequest, ImportEmailResponse}, - types::{acl::Acl, collection::Collection, id::Id, property::Property, state::State}, + types::{acl::Acl, id::Id, property::Property, state::State}, }; use mail_parser::MessageParser; -use utils::map::vec_map::VecMap; - -use crate::{blob::download::BlobDownload, changes::state::StateManager}; - use std::future::Future; +use utils::map::vec_map::VecMap; pub trait EmailImport: Sync + Send { fn email_import( @@ -40,15 +38,10 @@ impl EmailImport for Server { ) -> trc::Result { // Validate state let account_id = request.account_id.document_id(); - let old_state: State = self - .assert_state(account_id, Collection::Email, &request.if_in_state) - .await?; - - let cached_mailboxes = self.get_cached_mailboxes(account_id).await?; + let cache = self.get_cached_messages(account_id).await?; + let old_state: State = cache.assert_state(false, &request.if_in_state)?; let can_add_mailbox_ids = if access_token.is_shared(account_id) { - cached_mailboxes - .shared_mailboxes(access_token, Acl::AddItems) - .into() + cache.shared_mailboxes(access_token, Acl::AddItems).into() } else { None }; @@ -83,7 +76,7 @@ impl EmailImport for Server { continue; } for mailbox_id in &mailbox_ids { - if !cached_mailboxes.has_id(mailbox_id) { + if !cache.has_mailbox_id(mailbox_id) { response.not_created.append( id, SetError::invalid_properties() @@ -165,7 +158,7 @@ impl EmailImport for Server { // Update state if !response.created.is_empty() { - response.new_state = self.get_state(account_id, Collection::Email).await?; + response.new_state = self.get_cached_messages(account_id).await?.get_state(false); } Ok(response) diff --git a/crates/jmap/src/email/query.rs b/crates/jmap/src/email/query.rs index 4511ad24..0f9a7d51 100644 --- a/crates/jmap/src/email/query.rs +++ b/crates/jmap/src/email/query.rs @@ -4,11 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::{JmapMethods, changes::state::MessageCacheState}; use common::{MessageStoreCache, Server, auth::AccessToken}; -use email::{ - mailbox::cache::MessageMailboxCache, - message::cache::{MessageCacheAccess, MessageCacheFetch}, -}; +use email::cache::{MessageCacheFetch, email::MessageCacheAccess}; use jmap_proto::{ method::query::{Comparator, Filter, QueryRequest, QueryResponse, SortProperty}, object::email::QueryArguments, @@ -26,8 +24,6 @@ use store::{ }; use trc::AddContext; -use crate::JmapMethods; - pub trait EmailQuery: Sync + Send { fn email_query( &self, @@ -305,17 +301,11 @@ impl EmailQuery for Server { let mut result_set = self.filter(account_id, Collection::Email, filters).await?; if access_token.is_shared(account_id) { - let cached_mailboxes = self - .get_cached_mailboxes(account_id) - .await - .caused_by(trc::location!())?; - result_set.apply_mask(cached_messages.shared_messages( - access_token, - &cached_mailboxes, - Acl::ReadItems, - )); + result_set.apply_mask(cached_messages.shared_messages(access_token, Acl::ReadItems)); } - let (response, paginate) = self.build_query_response(&result_set, &request).await?; + let (response, paginate) = self + .build_query_response(&result_set, cached_messages.get_state(false), &request) + .await?; if let Some(paginate) = paginate { // Parse sort criteria @@ -382,13 +372,13 @@ impl EmailQuery for Server { } // Sort results - let cache = self.get_cached_messages(account_id).await?; self.sort( result_set, comparators, paginate .with_prefix_map( - &cache + &cached_messages + .emails .items .iter() .map(|item| (item.document_id, item.thread_id)) @@ -415,14 +405,14 @@ fn thread_keywords(cache: &MessageStoreCache, keyword: Keyword, match_all: bool) let mut thread_map: AHashMap = AHashMap::new(); - for item in &cache.items { + for item in &cache.emails.items { thread_map .entry(item.thread_id) .or_default() .insert(item.document_id); } - for item in &cache.items { + for item in &cache.emails.items { let keyword_doc_id = item.document_id; if !keyword_doc_ids.contains(keyword_doc_id) || matched_ids.contains(keyword_doc_id) diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index 0d2c8356..1db2182f 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -6,14 +6,13 @@ use std::{borrow::Cow, collections::HashMap}; +use super::headers::{BuildHeader, ValueToHeader}; +use crate::{JmapMethods, blob::download::BlobDownload, changes::state::MessageCacheState}; use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; use email::{ - mailbox::{ - UidMailbox, - cache::{MailboxCacheAccess, MessageMailboxCache}, - }, + cache::{MessageCacheFetch, email::MessageCacheAccess, mailbox::MailboxCacheAccess}, + mailbox::UidMailbox, message::{ - cache::{MessageCacheAccess, MessageCacheFetch}, delete::EmailDeletion, ingest::{EmailIngest, IngestEmail, IngestSource}, metadata::MessageData, @@ -26,7 +25,7 @@ use jmap_proto::{ response::references::EvalObjectReferences, types::{ acl::Acl, - collection::Collection, + collection::{Collection, SyncCollection}, keyword::Keyword, property::Property, state::{State, StateChange}, @@ -43,14 +42,10 @@ use mail_builder::{ mime::{BodyPart, MimePart}, }; use mail_parser::MessageParser; +use std::future::Future; use store::{ahash::AHashSet, roaring::RoaringBitmap, write::BatchBuilder}; use trc::AddContext; -use crate::{JmapMethods, blob::download::BlobDownload}; -use std::future::Future; - -use super::headers::{BuildHeader, ValueToHeader}; - pub trait EmailSet: Sync + Send { fn email_set( &self, @@ -69,26 +64,21 @@ impl EmailSet for Server { ) -> trc::Result { // Prepare response let account_id = request.account_id.document_id(); + let cache = self.get_cached_messages(account_id).await?; let mut response = self - .prepare_set_response(&request, Collection::Email) + .prepare_set_response(&request, cache.assert_state(false, &request.if_in_state)?) .await?; let can_train_spam = self.email_bayes_can_train(access_token); // Obtain mailboxIds - let cached_mailboxes = self.get_cached_mailboxes(account_id).await?; - let cached_messages = self.get_cached_messages(account_id).await?; let (can_add_mailbox_ids, can_delete_mailbox_ids, can_modify_message_ids) = if access_token.is_shared(account_id) { ( - cached_mailboxes - .shared_mailboxes(access_token, Acl::AddItems) - .into(), - cached_mailboxes + cache.shared_mailboxes(access_token, Acl::AddItems).into(), + cache .shared_mailboxes(access_token, Acl::RemoveItems) .into(), - cached_messages - .shared_messages(access_token, &cached_mailboxes, Acl::ModifyItems) - .into(), + cache.shared_messages(access_token, Acl::ModifyItems).into(), ) } else { (None, None, None) @@ -662,7 +652,7 @@ impl EmailSet for Server { // Verify that the mailboxIds are valid for mailbox_id in &mailboxes { - if !cached_mailboxes.has_id(mailbox_id) { + if !cache.has_mailbox_id(mailbox_id) { response.not_created.append( id, SetError::invalid_properties() @@ -878,7 +868,7 @@ impl EmailSet for Server { // Make sure all new mailboxIds are valid for mailbox_id in new_data.added_mailboxes(data.inner) { - if cached_mailboxes.has_id(&mailbox_id.mailbox_id) { + if cache.has_mailbox_id(&mailbox_id.mailbox_id) { // Verify permissions on shared accounts if !matches!(&can_add_mailbox_ids, Some(ids) if !ids.contains(mailbox_id.mailbox_id)) { @@ -958,7 +948,7 @@ impl EmailSet for Server { if !batch.is_empty() { // Log mailbox changes for parent_id in changed_mailboxes { - batch.log_parent_update(Collection::Mailbox.as_child_update(), parent_id); + batch.log_container_property_change(SyncCollection::Email, parent_id); } match self.commit_batch(batch).await { @@ -986,11 +976,9 @@ impl EmailSet for Server { // Process deletions if !will_destroy.is_empty() { - let email_ids = cached_messages.document_ids(); + let email_ids = cache.email_document_ids(); let can_destroy_message_ids = if access_token.is_shared(account_id) { - cached_messages - .shared_messages(access_token, &cached_mailboxes, Acl::RemoveItems) - .into() + cache.shared_messages(access_token, Acl::RemoveItems).into() } else { None }; diff --git a/crates/jmap/src/email/snippet.rs b/crates/jmap/src/email/snippet.rs index 4d4673df..6ab5872d 100644 --- a/crates/jmap/src/email/snippet.rs +++ b/crates/jmap/src/email/snippet.rs @@ -4,15 +4,13 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::blob::download::BlobDownload; use common::{Server, auth::AccessToken}; use email::{ - mailbox::cache::MessageMailboxCache, - message::{ - cache::{MessageCacheFetch, MessageCacheAccess}, - metadata::{ - ArchivedGetHeader, ArchivedHeaderName, ArchivedMetadataPartType, DecodedPartContent, - MessageMetadata, - }, + cache::{MessageCacheFetch, email::MessageCacheAccess}, + message::metadata::{ + ArchivedGetHeader, ArchivedHeaderName, ArchivedMetadataPartType, DecodedPartContent, + MessageMetadata, }, }; use jmap_proto::{ @@ -24,14 +22,11 @@ use jmap_proto::{ }; use mail_parser::decoders::html::html_to_text; use nlp::language::{Language, search_snippet::generate_snippet, stemmer::Stemmer}; +use std::future::Future; use store::backend::MAX_TOKEN_LENGTH; use trc::AddContext; use utils::BlobHash; -use crate::blob::download::BlobDownload; - -use std::future::Future; - pub trait EmailSearchSnippet: Sync + Send { fn email_search_snippet( &self, @@ -97,13 +92,9 @@ impl EmailSearchSnippet for Server { .await .caused_by(trc::location!())?; let document_ids = if access_token.is_member(account_id) { - cached_messages.document_ids() + cached_messages.email_document_ids() } else { - let cached_mailboxes = self - .get_cached_mailboxes(account_id) - .await - .caused_by(trc::location!())?; - cached_messages.shared_messages(access_token, &cached_mailboxes, Acl::ReadItems) + cached_messages.shared_messages(access_token, Acl::ReadItems) }; let email_ids = request.email_ids.unwrap(); diff --git a/crates/jmap/src/identity/get.rs b/crates/jmap/src/identity/get.rs index ada3c658..69b60dd6 100644 --- a/crates/jmap/src/identity/get.rs +++ b/crates/jmap/src/identity/get.rs @@ -10,7 +10,7 @@ use email::identity::{ArchivedEmailAddress, Identity}; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, types::{ - collection::Collection, + collection::{Collection, SyncCollection}, property::Property, value::{Object, Value}, }, @@ -69,7 +69,7 @@ impl IdentityGet for Server { let mut response = GetResponse { account_id: request.account_id.into(), state: self - .get_state(account_id, Collection::Identity) + .get_state(account_id, SyncCollection::Identity) .await? .into(), list: Vec::with_capacity(ids.len()), diff --git a/crates/jmap/src/identity/set.rs b/crates/jmap/src/identity/set.rs index bca47765..6ddbce58 100644 --- a/crates/jmap/src/identity/set.rs +++ b/crates/jmap/src/identity/set.rs @@ -12,7 +12,7 @@ use jmap_proto::{ method::set::{RequestArguments, SetRequest, SetResponse}, response::references::EvalObjectReferences, types::{ - collection::Collection, + collection::{Collection, SyncCollection}, property::Property, state::State, value::{MaybePatchValue, Value}, @@ -161,7 +161,7 @@ impl IdentitySet for Server { .with_collection(Collection::Identity) .delete_document(document_id) .clear(Property::Value) - .log_delete(None) + .log_item_delete(SyncCollection::Identity, None) .commit_point(); response.destroyed.push(id); } else { diff --git a/crates/jmap/src/lib.rs b/crates/jmap/src/lib.rs index 9a5bfdff..8846b5a9 100644 --- a/crates/jmap/src/lib.rs +++ b/crates/jmap/src/lib.rs @@ -6,18 +6,15 @@ #![warn(clippy::large_futures)] -use std::{fmt::Display, future::Future}; - -use changes::state::StateManager; use common::Server; use jmap_proto::{ method::{ query::{QueryRequest, QueryResponse}, set::{SetRequest, SetResponse}, }, - types::collection::Collection, + types::{collection::Collection, state::State}, }; - +use std::{fmt::Display, future::Future}; use store::{ fts::FtsFilter, query::{Comparator, Filter, ResultSet, SortedResultSet, sort::Pagination}, @@ -44,17 +41,11 @@ impl JmapMethods for Server { async fn prepare_set_response( &self, request: &SetRequest, - collection: Collection, + asserted_state: State, ) -> trc::Result { Ok( - SetResponse::from_request(request, self.core.jmap.set_max_objects)?.with_state( - self.assert_state( - request.account_id.document_id(), - collection, - &request.if_in_state, - ) - .await?, - ), + SetResponse::from_request(request, self.core.jmap.set_max_objects)? + .with_state(asserted_state), ) } @@ -97,6 +88,7 @@ impl JmapMethods for Server { async fn build_query_response( &self, result_set: &ResultSet, + query_state: State, request: &QueryRequest, ) -> trc::Result<(QueryResponse, Option)> { let total = result_set.results.len() as usize; @@ -116,9 +108,7 @@ impl JmapMethods for Server { Ok(( QueryResponse { account_id: request.account_id, - query_state: self - .get_state(result_set.account_id, result_set.collection) - .await?, + query_state, can_calculate_changes: true, position: 0, ids: vec![], @@ -174,7 +164,7 @@ pub trait JmapMethods: Sync + Send { fn prepare_set_response( &self, request: &SetRequest, - collection: Collection, + asserted_state: State, ) -> impl Future> + Send; fn filter( @@ -194,6 +184,7 @@ pub trait JmapMethods: Sync + Send { fn build_query_response( &self, result_set: &ResultSet, + query_state: State, request: &QueryRequest, ) -> impl Future)>> + Send; diff --git a/crates/jmap/src/mailbox/get.rs b/crates/jmap/src/mailbox/get.rs index eaeef7af..da097dcc 100644 --- a/crates/jmap/src/mailbox/get.rs +++ b/crates/jmap/src/mailbox/get.rs @@ -5,10 +5,7 @@ */ use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; -use email::{ - mailbox::cache::{MailboxCacheAccess, MessageMailboxCache}, - message::cache::{MessageCacheAccess, MessageCacheFetch}, -}; +use email::cache::{MessageCacheFetch, email::MessageCacheAccess, mailbox::MailboxCacheAccess}; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, types::{ @@ -50,19 +47,17 @@ impl MailboxGet for Server { Property::MyRights, ]); let account_id = request.account_id.document_id(); - let mailbox_cache = self.get_cached_mailboxes(account_id).await?; - let message_cache = self.get_cached_messages(account_id).await?; + let cache = self.get_cached_messages(account_id).await?; let shared_ids = if access_token.is_shared(account_id) { - mailbox_cache - .shared_mailboxes(access_token, Acl::Read) - .into() + cache.shared_mailboxes(access_token, Acl::Read).into() } else { None }; let ids = if let Some(ids) = ids { ids } else { - mailbox_cache + cache + .mailboxes .index .keys() .filter(|id| shared_ids.as_ref().is_none_or(|ids| ids.contains(**id))) @@ -73,7 +68,7 @@ impl MailboxGet for Server { }; let mut response = GetResponse { account_id: request.account_id.into(), - state: Some(mailbox_cache.change_id.into()), + state: Some(cache.mailboxes.change_id.into()), list: Vec::with_capacity(ids.len()), not_found: vec![], }; @@ -82,7 +77,7 @@ impl MailboxGet for Server { // Obtain the mailbox object let document_id = id.document_id(); let cached_mailbox = if let Some(mailbox) = - mailbox_cache.by_id(&document_id).filter(|_| { + cache.mailbox_by_id(&document_id).filter(|_| { shared_ids .as_ref() .is_none_or(|ids| ids.contains(document_id)) @@ -115,22 +110,22 @@ impl MailboxGet for Server { } } Property::TotalEmails => { - Value::UnsignedInt(message_cache.in_mailbox(document_id).count() as u64) + Value::UnsignedInt(cache.in_mailbox(document_id).count() as u64) } Property::UnreadEmails => Value::UnsignedInt( - message_cache + cache .in_mailbox_without_keyword(document_id, &Keyword::Seen) .count() as u64, ), Property::TotalThreads => Value::UnsignedInt( - message_cache + cache .in_mailbox(document_id) .map(|m| m.thread_id) .collect::>() .len() as u64, ), Property::UnreadThreads => Value::UnsignedInt( - message_cache + cache .in_mailbox_without_keyword(document_id, &Keyword::Seen) .map(|m| m.thread_id) .collect::>() diff --git a/crates/jmap/src/mailbox/query.rs b/crates/jmap/src/mailbox/query.rs index 987277a6..3b0d17e3 100644 --- a/crates/jmap/src/mailbox/query.rs +++ b/crates/jmap/src/mailbox/query.rs @@ -4,23 +4,22 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::{JmapMethods, changes::state::MessageCacheState}; use common::{Server, auth::AccessToken, config::jmap::settings::SpecialUse}; -use email::mailbox::cache::{MailboxCacheAccess, MessageMailboxCache}; +use email::cache::{MessageCacheFetch, mailbox::MailboxCacheAccess}; use jmap_proto::{ method::query::{Comparator, Filter, QueryRequest, QueryResponse, SortProperty}, object::mailbox::QueryArguments, types::{acl::Acl, collection::Collection}, }; -use store::{ - query::{self}, - roaring::RoaringBitmap, -}; - -use crate::JmapMethods; use std::{ collections::{BTreeMap, BTreeSet}, future::Future, }; +use store::{ + query::{self}, + roaring::RoaringBitmap, +}; pub trait MailboxQuery: Sync + Send { fn mailbox_query( @@ -40,7 +39,7 @@ impl MailboxQuery for Server { let sort_as_tree = request.arguments.sort_as_tree.unwrap_or(false); let filter_as_tree = request.arguments.filter_as_tree.unwrap_or(false); let mut filters = Vec::with_capacity(request.filter.len()); - let mailboxes = self.get_cached_mailboxes(account_id).await?; + let mailboxes = self.get_cached_messages(account_id).await?; for cond in std::mem::take(&mut request.filter) { match cond { @@ -48,6 +47,7 @@ impl MailboxQuery for Server { let parent_id = parent_id.map(|id| id.document_id()).unwrap_or(u32::MAX); filters.push(query::Filter::is_in_set( mailboxes + .mailboxes .items .iter() .filter(|mailbox| mailbox.parent_id == parent_id) @@ -66,6 +66,7 @@ impl MailboxQuery for Server { let name = name.to_lowercase(); filters.push(query::Filter::is_in_set( mailboxes + .mailboxes .items .iter() .filter(|mailbox| mailbox.name.to_lowercase().contains(&name)) @@ -77,6 +78,7 @@ impl MailboxQuery for Server { if let Some(role) = role { filters.push(query::Filter::is_in_set( mailboxes + .mailboxes .items .iter() .filter(|mailbox| mailbox.role.as_str().is_some_and(|r| r == role)) @@ -87,6 +89,7 @@ impl MailboxQuery for Server { filters.push(query::Filter::Not); filters.push(query::Filter::is_in_set( mailboxes + .mailboxes .items .iter() .filter(|mailbox| matches!(mailbox.role, SpecialUse::None)) @@ -102,6 +105,7 @@ impl MailboxQuery for Server { } filters.push(query::Filter::is_in_set( mailboxes + .mailboxes .items .iter() .filter(|mailbox| !matches!(mailbox.role, SpecialUse::None)) @@ -118,6 +122,7 @@ impl MailboxQuery for Server { } filters.push(query::Filter::is_in_set( mailboxes + .mailboxes .items .iter() .filter(|mailbox| { @@ -148,7 +153,9 @@ impl MailboxQuery for Server { if access_token.is_shared(account_id) { result_set.apply_mask(mailboxes.shared_mailboxes(access_token, Acl::Read)); } - let (mut response, mut paginate) = self.build_query_response(&result_set, &request).await?; + let (mut response, mut paginate) = self + .build_query_response(&result_set, mailboxes.get_state(true), &request) + .await?; // Filter as tree if filter_as_tree { @@ -157,7 +164,7 @@ impl MailboxQuery for Server { for document_id in &result_set.results { let mut check_id = document_id; for _ in 0..self.core.jmap.mailbox_max_depth { - if let Some(mailbox) = mailboxes.by_id(&check_id) { + if let Some(mailbox) = mailboxes.mailbox_by_id(&check_id) { if let Some(parent_id) = mailbox.parent_id() { if result_set.results.contains(parent_id) { check_id = parent_id; @@ -190,6 +197,7 @@ impl MailboxQuery for Server { // Sort as tree if sort_as_tree { let sorted_list = mailboxes + .mailboxes .items .iter() .map(|mailbox| (mailbox.path.as_str(), mailbox.document_id)) @@ -209,6 +217,7 @@ impl MailboxQuery for Server { comparators.push(match comparator.property { SortProperty::Name => { let sorted_list = mailboxes + .mailboxes .items .iter() .map(|mailbox| (mailbox.name.as_str(), mailbox.document_id)) @@ -221,6 +230,7 @@ impl MailboxQuery for Server { } SortProperty::SortOrder => { let sorted_list = mailboxes + .mailboxes .items .iter() .map(|mailbox| (mailbox.sort_order, mailbox.document_id)) @@ -233,6 +243,7 @@ impl MailboxQuery for Server { } SortProperty::ParentId => { let sorted_list = mailboxes + .mailboxes .items .iter() .map(|mailbox| { diff --git a/crates/jmap/src/mailbox/set.rs b/crates/jmap/src/mailbox/set.rs index 4b6c15e3..3f46306d 100644 --- a/crates/jmap/src/mailbox/set.rs +++ b/crates/jmap/src/mailbox/set.rs @@ -4,15 +4,14 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::{JmapMethods, changes::state::MessageCacheState}; use common::{ Server, auth::AccessToken, config::jmap::settings::SpecialUse, sharing::EffectiveAcl, storage::index::ObjectIndexBuilder, }; - -use email::mailbox::{ - Mailbox, - cache::{MailboxCacheAccess, MessageMailboxCache}, - destroy::MailboxDestroy, +use email::{ + cache::{MessageCacheFetch, mailbox::MailboxCacheAccess}, + mailbox::{Mailbox, destroy::MailboxDestroy}, }; use jmap_proto::{ error::set::SetError, @@ -35,8 +34,6 @@ use store::{ use trc::AddContext; use utils::config::utils::ParseValue; -use crate::JmapMethods; - #[allow(unused_imports)] use email::mailbox::{INBOX_ID, JUNK_ID, TRASH_ID, UidMailbox}; use std::future::Future; @@ -75,16 +72,15 @@ impl MailboxSet for Server { // Prepare response let account_id = request.account_id.document_id(); let on_destroy_remove_emails = request.arguments.on_destroy_remove_emails.unwrap_or(false); + let cache = self.get_cached_messages(account_id).await?; let mut ctx = SetContext { account_id, is_shared: access_token.is_shared(account_id), access_token, response: self - .prepare_set_response(&request, Collection::Mailbox) + .prepare_set_response(&request, cache.assert_state(true, &request.if_in_state)?) .await?, - mailbox_ids: RoaringBitmap::from_iter( - self.get_cached_mailboxes(account_id).await?.index.keys(), - ), + mailbox_ids: RoaringBitmap::from_iter(cache.mailboxes.index.keys()), will_destroy: request.unwrap_destroy(), }; let mut change_id = None; @@ -433,7 +429,7 @@ impl MailboxSet for Server { } } - let cached_mailboxes = self.get_cached_mailboxes(ctx.account_id).await?; + let cached_mailboxes = self.get_cached_messages(ctx.account_id).await?; // Verify that the mailbox role is unique. if !matches!(changes.role, SpecialUse::None) @@ -441,7 +437,7 @@ impl MailboxSet for Server { .as_ref() .is_none_or(|(_, m)| m.inner.role != changes.role) { - if cached_mailboxes.by_role(&changes.role).is_some() { + if cached_mailboxes.mailbox_by_role(&changes.role).is_some() { return Ok(Err(SetError::invalid_properties() .with_property(Property::Role) .with_description(format!( @@ -469,7 +465,7 @@ impl MailboxSet for Server { if update .as_ref() .is_none_or(|(_, m)| m.inner.name != changes.name) - && cached_mailboxes.items.iter().any(|m| { + && cached_mailboxes.mailboxes.items.iter().any(|m| { m.name.to_lowercase() == lower_name && m.parent_id().map_or(0, |id| id + 1) == changes.parent_id }) diff --git a/crates/jmap/src/principal/query.rs b/crates/jmap/src/principal/query.rs index 66238862..d26953df 100644 --- a/crates/jmap/src/principal/query.rs +++ b/crates/jmap/src/principal/query.rs @@ -9,7 +9,7 @@ use directory::QueryBy; use http_proto::HttpSessionData; use jmap_proto::{ method::query::{Filter, QueryRequest, QueryResponse, RequestArguments}, - types::collection::Collection, + types::{collection::Collection, state::State}, }; use store::{query::ResultSet, roaring::RoaringBitmap}; @@ -90,7 +90,9 @@ impl PrincipalQuery for Server { .unwrap_or_default(); } - let (response, paginate) = self.build_query_response(&result_set, &request).await?; + let (response, paginate) = self + .build_query_response(&result_set, State::Initial, &request) + .await?; if let Some(paginate) = paginate { self.sort(result_set, Vec::new(), paginate, response).await diff --git a/crates/jmap/src/sieve/get.rs b/crates/jmap/src/sieve/get.rs index 7639732b..c3cd26d1 100644 --- a/crates/jmap/src/sieve/get.rs +++ b/crates/jmap/src/sieve/get.rs @@ -10,7 +10,7 @@ use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, types::{ blob::{BlobId, BlobSection}, - collection::Collection, + collection::{Collection, SyncCollection}, property::Property, value::{Object, Value}, }, @@ -58,7 +58,7 @@ impl SieveScriptGet for Server { let mut response = GetResponse { account_id: request.account_id.into(), state: self - .get_state(account_id, Collection::SieveScript) + .get_state(account_id, SyncCollection::SieveScript) .await? .into(), list: Vec::with_capacity(ids.len()), diff --git a/crates/jmap/src/sieve/query.rs b/crates/jmap/src/sieve/query.rs index a9b24ff4..16f4345f 100644 --- a/crates/jmap/src/sieve/query.rs +++ b/crates/jmap/src/sieve/query.rs @@ -9,12 +9,15 @@ use jmap_proto::{ method::query::{ Comparator, Filter, QueryRequest, QueryResponse, RequestArguments, SortProperty, }, - types::{collection::Collection, property::Property}, + types::{ + collection::{Collection, SyncCollection}, + property::Property, + }, }; use std::future::Future; use store::query::{self}; -use crate::JmapMethods; +use crate::{JmapMethods, changes::state::StateManager}; pub trait SieveScriptQuery: Sync + Send { fn sieve_script_query( @@ -52,7 +55,14 @@ impl SieveScriptQuery for Server { .filter(account_id, Collection::SieveScript, filters) .await?; - let (response, paginate) = self.build_query_response(&result_set, &request).await?; + let (response, paginate) = self + .build_query_response( + &result_set, + self.get_state(account_id, SyncCollection::SieveScript) + .await?, + &request, + ) + .await?; if let Some(paginate) = paginate { // Parse sort criteria diff --git a/crates/jmap/src/sieve/set.rs b/crates/jmap/src/sieve/set.rs index 0927422c..5efc3be0 100644 --- a/crates/jmap/src/sieve/set.rs +++ b/crates/jmap/src/sieve/set.rs @@ -22,7 +22,7 @@ use jmap_proto::{ response::references::EvalObjectReferences, types::{ blob::{BlobId, BlobSection}, - collection::Collection, + collection::{Collection, SyncCollection}, id::Id, property::Property, state::State, @@ -39,7 +39,7 @@ use store::{ }; use trc::AddContext; -use crate::{JmapMethods, blob::download::BlobDownload}; +use crate::{JmapMethods, blob::download::BlobDownload, changes::state::StateManager}; use std::future::Future; pub struct SetContext<'x> { @@ -92,7 +92,15 @@ impl SieveScriptSet for Server { resource_token: self.get_resource_token(access_token, account_id).await?, access_token, response: self - .prepare_set_response(&request, Collection::SieveScript) + .prepare_set_response( + &request, + self.assert_state( + account_id, + SyncCollection::SieveScript, + &request.if_in_state, + ) + .await?, + ) .await?, }; let will_destroy = request.unwrap_destroy(); diff --git a/crates/jmap/src/submission/get.rs b/crates/jmap/src/submission/get.rs index c43d08ab..8b50d1ec 100644 --- a/crates/jmap/src/submission/get.rs +++ b/crates/jmap/src/submission/get.rs @@ -11,7 +11,7 @@ use email::submission::{ use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, types::{ - collection::Collection, + collection::{Collection, SyncCollection}, date::UTCDate, id::Id, property::Property, @@ -69,7 +69,7 @@ impl EmailSubmissionGet for Server { let mut response = GetResponse { account_id: request.account_id.into(), state: self - .get_state(account_id, Collection::EmailSubmission) + .get_state(account_id, SyncCollection::EmailSubmission) .await? .into(), list: Vec::with_capacity(ids.len()), diff --git a/crates/jmap/src/submission/query.rs b/crates/jmap/src/submission/query.rs index c059a873..9459188f 100644 --- a/crates/jmap/src/submission/query.rs +++ b/crates/jmap/src/submission/query.rs @@ -10,7 +10,10 @@ use jmap_proto::{ method::query::{ Comparator, Filter, QueryRequest, QueryResponse, RequestArguments, SortProperty, }, - types::{collection::Collection, property::Property}, + types::{ + collection::{Collection, SyncCollection}, + property::Property, + }, }; use std::future::Future; use store::{ @@ -18,7 +21,7 @@ use store::{ query::{self}, }; -use crate::JmapMethods; +use crate::{JmapMethods, changes::state::StateManager}; pub trait EmailSubmissionQuery: Sync + Send { fn email_submission_query( @@ -94,7 +97,14 @@ impl EmailSubmissionQuery for Server { .filter(account_id, Collection::EmailSubmission, filters) .await?; - let (response, paginate) = self.build_query_response(&result_set, &request).await?; + let (response, paginate) = self + .build_query_response( + &result_set, + self.get_state(account_id, SyncCollection::EmailSubmission) + .await?, + &request, + ) + .await?; if let Some(paginate) = paginate { // Parse sort criteria diff --git a/crates/jmap/src/thread/get.rs b/crates/jmap/src/thread/get.rs index 73629a31..e107d3ce 100644 --- a/crates/jmap/src/thread/get.rs +++ b/crates/jmap/src/thread/get.rs @@ -4,11 +4,17 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::changes::state::StateManager; use common::Server; -use email::message::cache::MessageCacheFetch; +use email::cache::MessageCacheFetch; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, - types::{collection::Collection, id::Id, property::Property, value::Object}, + types::{ + collection::{Collection, SyncCollection}, + id::Id, + property::Property, + value::Object, + }, }; use std::future::Future; use store::{ @@ -18,8 +24,6 @@ use store::{ }; use trc::AddContext; -use crate::changes::state::StateManager; - pub trait ThreadGet: Sync + Send { fn thread_get( &self, @@ -38,6 +42,7 @@ impl ThreadGet for Server { .get_cached_messages(account_id) .await .caused_by(trc::location!())? + .emails .items { thread_map @@ -61,7 +66,10 @@ impl ThreadGet for Server { .is_none_or(|p| p.unwrap().contains(&Property::EmailIds)); let mut response = GetResponse { account_id: request.account_id.into(), - state: self.get_state(account_id, Collection::Thread).await?.into(), + state: self + .get_state(account_id, SyncCollection::Thread) + .await? + .into(), list: Vec::with_capacity(ids.len()), not_found: vec![], }; diff --git a/crates/jmap/src/vacation/get.rs b/crates/jmap/src/vacation/get.rs index a60dbc0d..60cf673c 100644 --- a/crates/jmap/src/vacation/get.rs +++ b/crates/jmap/src/vacation/get.rs @@ -11,7 +11,7 @@ use jmap_proto::{ request::reference::MaybeReference, types::{ any_id::AnyId, - collection::Collection, + collection::{Collection, SyncCollection}, date::UTCDate, id::Id, property::Property, @@ -54,7 +54,7 @@ impl VacationResponseGet for Server { let mut response = GetResponse { account_id: request.account_id.into(), state: self - .get_state(account_id, Collection::SieveScript) + .get_state(account_id, SyncCollection::SieveScript) .await? .into(), list: Vec::with_capacity(1), diff --git a/crates/jmap/src/vacation/set.rs b/crates/jmap/src/vacation/set.rs index 91064522..438eb92c 100644 --- a/crates/jmap/src/vacation/set.rs +++ b/crates/jmap/src/vacation/set.rs @@ -6,6 +6,8 @@ use std::borrow::Cow; +use super::get::VacationResponseGet; +use crate::{JmapMethods, changes::state::StateManager}; use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; use email::sieve::{ SieveScript, VacationResponse, activate::SieveScriptActivate, delete::SieveScriptDelete, @@ -15,7 +17,7 @@ use jmap_proto::{ method::set::{RequestArguments, SetRequest, SetResponse}, response::references::EvalObjectReferences, types::{ - collection::Collection, + collection::{Collection, SyncCollection}, date::UTCDate, id::Id, property::Property, @@ -31,10 +33,6 @@ use store::{ }; use trc::AddContext; -use crate::JmapMethods; - -use super::get::VacationResponseGet; - pub trait VacationResponseSet: Sync + Send { fn vacation_response_set( &self, @@ -53,7 +51,15 @@ impl VacationResponseSet for Server { ) -> trc::Result { let account_id = request.account_id.document_id(); let mut response = self - .prepare_set_response(&request, Collection::SieveScript) + .prepare_set_response( + &request, + self.assert_state( + account_id, + SyncCollection::SieveScript, + &request.if_in_state, + ) + .await?, + ) .await?; let will_destroy = request.unwrap_destroy(); let resource_token = self.get_resource_token(access_token, account_id).await?; diff --git a/crates/pop3/src/mailbox.rs b/crates/pop3/src/mailbox.rs index 908bf6b5..5c290d22 100644 --- a/crates/pop3/src/mailbox.rs +++ b/crates/pop3/src/mailbox.rs @@ -8,11 +8,8 @@ use std::collections::BTreeMap; use common::{config::jmap::settings::SpecialUse, listener::SessionStream}; use email::{ - mailbox::{ - INBOX_ID, - cache::{MailboxCacheAccess, MessageMailboxCache}, - }, - message::cache::MessageCacheFetch, + cache::{MessageCacheFetch, mailbox::MailboxCacheAccess}, + mailbox::INBOX_ID, }; use jmap_proto::types::{collection::Collection, property::Property}; use store::{ @@ -42,23 +39,18 @@ pub struct Message { impl Session { pub async fn fetch_mailbox(&self, account_id: u32) -> trc::Result { // Obtain UID validity - let message_cache = self + let cache = self .server .get_cached_messages(account_id) .await .caused_by(trc::location!())?; - if message_cache.items.is_empty() { + if cache.emails.items.is_empty() { return Ok(Mailbox::default()); } - let mailbox_cache = self - .server - .get_cached_mailboxes(account_id) - .await - .caused_by(trc::location!())?; - let uid_validity = mailbox_cache - .by_role(&SpecialUse::Inbox) + let uid_validity = cache + .mailbox_by_role(&SpecialUse::Inbox) .map(|x| x.uid_validity) .unwrap_or_default(); @@ -87,13 +79,10 @@ impl Session { ) .no_values(), |key, _| { - let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; - if mailbox_cache.has_id(&document_id) { - message_sizes.insert( - document_id, - key.deserialize_be_u32(key.len() - (U32_LEN * 2))?, - ); - } + message_sizes.insert( + key.deserialize_be_u32(key.len() - U32_LEN)?, + key.deserialize_be_u32(key.len() - (U32_LEN * 2))?, + ); Ok(true) }, @@ -102,7 +91,8 @@ impl Session { .caused_by(trc::location!())?; // Sort by UID - let message_map = message_cache + let message_map = cache + .emails .items .iter() .filter_map(|message| { diff --git a/crates/store/src/query/log.rs b/crates/store/src/query/log.rs index 9a966e7f..1092ed5e 100644 --- a/crates/store/src/query/log.rs +++ b/crates/store/src/query/log.rs @@ -11,9 +11,13 @@ use crate::{IterateParams, LogKey, Store, U64_LEN, write::key::DeserializeBigEnd #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum Change { - Insert(u64), - Update(u64), - Delete(u64), + InsertContainer(u64), + UpdateContainer(u64), + UpdateContainerProperty(u64), + DeleteContainer(u64), + InsertItem(u64), + UpdateItem(u64), + DeleteItem(u64), } #[derive(Debug)] @@ -21,6 +25,8 @@ pub struct Changes { pub changes: Vec, pub from_change_id: u64, pub to_change_id: u64, + pub container_change_id: Option, + pub item_change_id: Option, } #[derive(Debug, Clone, Copy)] @@ -37,6 +43,8 @@ impl Default for Changes { changes: Vec::with_capacity(10), from_change_id: 0, to_change_id: 0, + container_change_id: None, + item_change_id: None, } } } @@ -79,9 +87,16 @@ impl Store { changelog.from_change_id = change_id; } changelog.to_change_id = change_id; - changelog.deserialize(value).ok_or_else(|| { - trc::Error::corrupted_key(key, value.into(), trc::location!()) - })?; + let (has_container_changes, has_item_changes) = + changelog.deserialize(value).ok_or_else(|| { + trc::Error::corrupted_key(key, value.into(), trc::location!()) + })?; + if has_container_changes { + changelog.container_change_id = Some(change_id); + } + if has_item_changes { + changelog.item_change_id = Some(change_id); + } } Ok(true) }, @@ -139,29 +154,48 @@ impl Store { } impl Changes { - pub fn deserialize(&mut self, bytes: &[u8]) -> Option<()> { + pub fn deserialize(&mut self, bytes: &[u8]) -> Option<(bool, bool)> { let mut bytes_it = bytes.iter(); - let total_inserts: usize = bytes_it.next_leb128()?; - let total_updates: usize = bytes_it.next_leb128()?; - let total_deletes: usize = bytes_it.next_leb128()?; - if total_inserts > 0 { - for _ in 0..total_inserts { - self.changes.push(Change::Insert(bytes_it.next_leb128()?)); + let container_inserts: usize = bytes_it.next_leb128()?; + let container_updates: usize = bytes_it.next_leb128()?; + let container_property_changes: usize = bytes_it.next_leb128()?; + let container_deletes: usize = bytes_it.next_leb128()?; + + let item_inserts: usize = bytes_it.next_leb128()?; + let item_updates: usize = bytes_it.next_leb128()?; + let item_deletes: usize = bytes_it.next_leb128()?; + + let has_container_changes = + container_inserts + container_updates + container_property_changes + container_deletes + > 0; + let has_item_changes = item_inserts + item_updates + item_deletes > 0; + + if container_inserts > 0 { + for _ in 0..container_inserts { + self.changes + .push(Change::InsertContainer(bytes_it.next_leb128()?)); } } - if total_updates > 0 { - 'update_outer: for _ in 0..total_updates { + if container_updates > 0 || container_property_changes > 0 { + 'update_outer: for change_pos in 0..(container_updates + container_property_changes) { let id = bytes_it.next_leb128()?; + let mut is_property_change = change_pos >= container_updates; for (idx, change) in self.changes.iter().enumerate() { match change { - Change::Insert(insert_id) if *insert_id == id => { + Change::InsertContainer(insert_id) if *insert_id == id => { // Item updated after inserted, no need to count this change. continue 'update_outer; } - Change::Update(update_id) if *update_id == id => { + Change::UpdateContainer(update_id) if *update_id == id => { + // Move update to the front + is_property_change = false; + self.changes.remove(idx); + break; + } + Change::UpdateContainerProperty(update_id) if *update_id == id => { // Move update to the front self.changes.remove(idx); break; @@ -170,21 +204,25 @@ impl Changes { } } - self.changes.push(Change::Update(id)); + self.changes.push(if !is_property_change { + Change::UpdateContainer(id) + } else { + Change::UpdateContainerProperty(id) + }); } } - if total_deletes > 0 { - 'delete_outer: for _ in 0..total_deletes { + if container_deletes > 0 { + 'delete_outer: for _ in 0..container_deletes { let id = bytes_it.next_leb128()?; 'delete_inner: for (idx, change) in self.changes.iter().enumerate() { match change { - Change::Insert(insert_id) if *insert_id == id => { + Change::InsertContainer(insert_id) if *insert_id == id => { self.changes.remove(idx); continue 'delete_outer; } - Change::Update(update_id) if *update_id == id => { + Change::UpdateContainer(update_id) if *update_id == id => { self.changes.remove(idx); break 'delete_inner; } @@ -192,28 +230,136 @@ impl Changes { } } - self.changes.push(Change::Delete(id)); + self.changes.push(Change::DeleteContainer(id)); } } - Some(()) + // Item changes + if item_inserts > 0 { + for _ in 0..item_inserts { + self.changes + .push(Change::InsertItem(bytes_it.next_leb128()?)); + } + } + + if item_updates > 0 { + 'update_outer: for _ in 0..item_updates { + let id = bytes_it.next_leb128()?; + + for (idx, change) in self.changes.iter().enumerate() { + match change { + Change::InsertItem(insert_id) if *insert_id == id => { + // Item updated after inserted, no need to count this change. + continue 'update_outer; + } + Change::UpdateItem(update_id) if *update_id == id => { + // Move update to the front + self.changes.remove(idx); + break; + } + _ => (), + } + } + + self.changes.push(Change::UpdateItem(id)); + } + } + + if item_deletes > 0 { + 'delete_outer: for _ in 0..item_deletes { + let id = bytes_it.next_leb128()?; + + 'delete_inner: for (idx, change) in self.changes.iter().enumerate() { + match change { + Change::InsertItem(insert_id) if *insert_id == id => { + self.changes.remove(idx); + continue 'delete_outer; + } + Change::UpdateItem(update_id) if *update_id == id => { + self.changes.remove(idx); + break 'delete_inner; + } + _ => (), + } + } + + self.changes.push(Change::DeleteItem(id)); + } + } + + Some((has_container_changes, has_item_changes)) + } +} + +impl Changes { + pub fn total_container_changes(&self) -> usize { + self.changes + .iter() + .filter(|change| change.is_container_change()) + .count() + } + + pub fn total_item_changes(&self) -> usize { + self.changes + .iter() + .filter(|change| change.is_item_change()) + .count() } } impl Change { - pub fn id(&self) -> u64 { + pub fn item_id(&self) -> Option { match self { - Change::Insert(id) => *id, - Change::Update(id) => *id, - Change::Delete(id) => *id, + Change::InsertItem(id) => Some(*id), + Change::UpdateItem(id) => Some(*id), + Change::DeleteItem(id) => Some(*id), + _ => None, } } - pub fn unwrap_id(self) -> u64 { + pub fn container_id(&self) -> Option { match self { - Change::Insert(id) => id, - Change::Update(id) => id, - Change::Delete(id) => id, + Change::InsertContainer(id) => Some(*id), + Change::UpdateContainer(id) => Some(*id), + Change::UpdateContainerProperty(id) => Some(*id), + Change::DeleteContainer(id) => Some(*id), + _ => None, } } + + pub fn try_unwrap_item_id(self) -> Option { + match self { + Change::InsertItem(id) => Some(id), + Change::UpdateItem(id) => Some(id), + Change::DeleteItem(id) => Some(id), + _ => None, + } + } + + pub fn try_unwrap_container_id(self) -> Option { + match self { + Change::InsertContainer(id) => Some(id), + Change::UpdateContainer(id) => Some(id), + Change::UpdateContainerProperty(id) => Some(id), + Change::DeleteContainer(id) => Some(id), + _ => None, + } + } + + pub fn is_container_change(&self) -> bool { + matches!( + self, + Change::InsertContainer(_) + | Change::UpdateContainer(_) + | Change::UpdateContainerProperty(_) + | Change::DeleteContainer(_) + ) + } + + pub fn is_item_change(&self) -> bool { + matches!( + self, + Change::InsertItem(_) | Change::UpdateItem(_) | Change::DeleteItem(_) + ) + } } diff --git a/crates/store/src/write/batch.rs b/crates/store/src/write/batch.rs index 3ece7921..432ac946 100644 --- a/crates/store/src/write/batch.rs +++ b/crates/store/src/write/batch.rs @@ -4,26 +4,20 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::{ + Batch, BatchBuilder, BitmapClass, ChangedCollection, IntoOperations, Operation, TagValue, + ValueClass, ValueOp, assert::ToAssertValue, +}; +use crate::{SerializeInfallible, U32_LEN}; use std::sync::{ LazyLock, atomic::{AtomicU64, Ordering}, }; - use utils::{ - map::{ - bitmap::{Bitmap, ShortId}, - vec_map::VecMap, - }, + map::{bitmap::ShortId, vec_map::VecMap}, snowflake::SnowflakeIdGenerator, }; -use crate::U32_LEN; - -use super::{ - Batch, BatchBuilder, BitmapClass, IntoOperations, Operation, TagValue, ValueClass, ValueOp, - assert::ToAssertValue, -}; - static CHANGE_SEQ: AtomicU64 = AtomicU64::new(0); static NODE_MUM: LazyLock = LazyLock::new(|| CHANGE_SEQ.swap(0, Ordering::Relaxed) as u16); @@ -250,15 +244,15 @@ impl BatchBuilder { self } - pub fn log_insert(&mut self, prefix: Option) -> &mut Self { - if let (Some(account_id), Some(collection), Some(document_id)) = ( - self.current_account_id, - self.current_collection, - self.current_document_id, - ) { - self.changes - .get_mut_or_insert(account_id) - .log_insert(collection, prefix, document_id); + pub fn log_item_insert(&mut self, collection: impl Into, prefix: Option) -> &mut Self { + if let (Some(account_id), Some(document_id)) = + (self.current_account_id, self.current_document_id) + { + self.changes.get_mut_or_insert(account_id).log_item_insert( + collection.into(), + prefix, + document_id, + ); } if self.current_change_id.is_none() { self.generate_change_id(); @@ -267,15 +261,15 @@ impl BatchBuilder { self } - pub fn log_update(&mut self, prefix: Option) -> &mut Self { - if let (Some(account_id), Some(collection), Some(document_id)) = ( - self.current_account_id, - self.current_collection, - self.current_document_id, - ) { - self.changes - .get_mut_or_insert(account_id) - .log_update(collection, prefix, document_id); + pub fn log_item_update(&mut self, collection: impl Into, prefix: Option) -> &mut Self { + if let (Some(account_id), Some(document_id)) = + (self.current_account_id, self.current_document_id) + { + self.changes.get_mut_or_insert(account_id).log_item_update( + collection.into(), + prefix, + document_id, + ); } if self.current_change_id.is_none() { self.generate_change_id(); @@ -284,15 +278,15 @@ impl BatchBuilder { self } - pub fn log_delete(&mut self, prefix: Option) -> &mut Self { - if let (Some(account_id), Some(collection), Some(document_id)) = ( - self.current_account_id, - self.current_collection, - self.current_document_id, - ) { - self.changes - .get_mut_or_insert(account_id) - .log_delete(collection, prefix, document_id); + pub fn log_item_delete(&mut self, collection: impl Into, prefix: Option) -> &mut Self { + if let (Some(account_id), Some(document_id)) = + (self.current_account_id, self.current_document_id) + { + self.changes.get_mut_or_insert(account_id).log_item_delete( + collection.into(), + prefix, + document_id, + ); } if self.current_change_id.is_none() { self.generate_change_id(); @@ -301,13 +295,60 @@ impl BatchBuilder { self } - pub fn log_parent_update(&mut self, collection: impl Into, parent_id: u32) -> &mut Self { - let collection = collection.into(); + pub fn log_container_insert(&mut self, collection: impl Into) -> &mut Self { + if let (Some(account_id), Some(document_id)) = + (self.current_account_id, self.current_document_id) + { + self.changes + .get_mut_or_insert(account_id) + .log_container_insert(collection.into(), document_id); + } + if self.current_change_id.is_none() { + self.generate_change_id(); + self.batch_ops += 1; + } + self + } + pub fn log_container_update(&mut self, collection: impl Into) -> &mut Self { + if let (Some(account_id), Some(document_id)) = + (self.current_account_id, self.current_document_id) + { + self.changes + .get_mut_or_insert(account_id) + .log_container_update(collection.into(), document_id); + } + if self.current_change_id.is_none() { + self.generate_change_id(); + self.batch_ops += 1; + } + self + } + + pub fn log_container_delete(&mut self, collection: impl Into) -> &mut Self { + if let (Some(account_id), Some(document_id)) = + (self.current_account_id, self.current_document_id) + { + self.changes + .get_mut_or_insert(account_id) + .log_container_delete(collection.into(), document_id); + } + if self.current_change_id.is_none() { + self.generate_change_id(); + self.batch_ops += 1; + } + self + } + + pub fn log_container_property_change( + &mut self, + collection: impl Into, + document_id: u32, + ) -> &mut Self { if let Some(account_id) = self.current_account_id { self.changes .get_mut_or_insert(account_id) - .log_update(collection, None, parent_id); + .log_container_property_update(collection.into(), document_id); } if self.current_change_id.is_none() { self.generate_change_id(); @@ -322,19 +363,20 @@ impl BatchBuilder { for (account_id, changelog) in std::mem::take(&mut self.changes) { self.with_account_id(account_id); - for (collection, set) in changelog.serialize() { + for (collection, changes) in changelog.into_iterator() { let cc = self.changed_collections.get_mut_or_insert(account_id); - cc.0 = change_id; - if collection < 64 { - cc.1.insert(ShortId(collection)); - } else { - cc.1.insert(ShortId(u8::MAX - collection)); + cc.change_id = change_id; + if changes.has_container_changes() { + cc.changed_containers.insert(ShortId(collection)); + } + if changes.has_item_changes() { + cc.changed_items.insert(ShortId(collection)); } self.ops.push(Operation::Log { change_id, collection, - set, + set: changes.serialize(), }); } } @@ -409,7 +451,7 @@ impl BatchBuilder { } } - pub fn changes(self) -> Option)>> { + pub fn changes(self) -> Option> { if self.has_changes() { Some(self.changed_collections) } else { diff --git a/crates/store/src/write/log.rs b/crates/store/src/write/log.rs index 463a82a3..7e686e62 100644 --- a/crates/store/src/write/log.rs +++ b/crates/store/src/write/log.rs @@ -16,37 +16,84 @@ pub(crate) struct ChangeLogBuilder { #[derive(Default, Debug)] pub struct Changes { - pub inserts: AHashSet, - pub updates: AHashSet, - pub deletes: AHashSet, + pub item_inserts: AHashSet, + pub item_updates: AHashSet, + pub item_deletes: AHashSet, + + pub container_inserts: AHashSet, + pub container_updates: AHashSet, + pub container_deletes: AHashSet, + pub container_property_changes: AHashSet, } impl ChangeLogBuilder { - pub fn serialize(self) -> impl Iterator)> { - self.changes - .into_iter() - .map(|(collection, changes)| (collection, changes.serialize())) + pub fn into_iterator(self) -> impl Iterator { + self.changes.into_iter() } - pub fn log_insert(&mut self, collection: impl Into, prefix: Option, document_id: u32) { + pub fn log_container_insert(&mut self, collection: impl Into, document_id: u32) { self.changes .get_mut_or_insert(collection.into()) - .inserts + .container_inserts + .insert(document_id); + } + + pub fn log_item_insert( + &mut self, + collection: impl Into, + prefix: Option, + document_id: u32, + ) { + self.changes + .get_mut_or_insert(collection.into()) + .item_inserts .insert(build_id(prefix, document_id)); } - pub fn log_update(&mut self, collection: impl Into, prefix: Option, document_id: u32) { + pub fn log_container_update(&mut self, collection: impl Into, document_id: u32) { self.changes .get_mut_or_insert(collection.into()) - .updates + .container_updates + .insert(document_id); + } + + pub fn log_container_property_update(&mut self, collection: impl Into, document_id: u32) { + self.changes + .get_mut_or_insert(collection.into()) + .container_property_changes + .insert(document_id); + } + + pub fn log_item_update( + &mut self, + collection: impl Into, + prefix: Option, + document_id: u32, + ) { + self.changes + .get_mut_or_insert(collection.into()) + .item_updates .insert(build_id(prefix, document_id)); } - pub fn log_delete(&mut self, collection: impl Into, prefix: Option, document_id: u32) { + pub fn log_container_delete(&mut self, collection: impl Into, document_id: u32) { + let changes = self.changes.get_mut_or_insert(collection.into()); + let id = document_id; + changes.container_updates.remove(&id); + changes.container_property_changes.remove(&id); + changes.container_deletes.insert(id); + } + + pub fn log_item_delete( + &mut self, + collection: impl Into, + prefix: Option, + document_id: u32, + ) { let changes = self.changes.get_mut_or_insert(collection.into()); let id = build_id(prefix, document_id); - changes.updates.remove(&id); - changes.deletes.insert(id); + changes.item_updates.remove(&id); + changes.item_deletes.insert(id); } } @@ -60,56 +107,58 @@ fn build_id(prefix: Option, document_id: u32) -> u64 { } impl Changes { - pub fn insert(id: T) -> Self - where - T: IntoIterator, - I: Into, - { - Changes { - inserts: id.into_iter().map(Into::into).collect(), - ..Default::default() - } + pub fn has_container_changes(&self) -> bool { + !self.container_inserts.is_empty() + || !self.container_updates.is_empty() + || !self.container_property_changes.is_empty() + || !self.container_deletes.is_empty() } - pub fn update(id: T) -> Self - where - T: IntoIterator, - I: Into, - { - Changes { - updates: id.into_iter().map(Into::into).collect(), - ..Default::default() - } - } - - pub fn delete(id: T) -> Self - where - T: IntoIterator, - I: Into, - { - Changes { - deletes: id.into_iter().map(Into::into).collect(), - ..Default::default() - } + pub fn has_item_changes(&self) -> bool { + !self.item_inserts.is_empty() + || !self.item_updates.is_empty() + || !self.item_deletes.is_empty() } } impl SerializeInfallible for Changes { fn serialize(&self) -> Vec { let mut buf = Vec::with_capacity( - 1 + (self.inserts.len() + self.updates.len() + self.deletes.len() + 4) + 1 + (self.item_inserts.len() + + self.item_updates.len() + + self.item_deletes.len() + + self.container_inserts.len() + + self.container_updates.len() + + self.container_property_changes.len() + + self.container_deletes.len() + + 4) * std::mem::size_of::(), ); - buf.push_leb128(self.inserts.len()); - buf.push_leb128(self.updates.len()); - buf.push_leb128(self.deletes.len()); + buf.push_leb128(self.container_inserts.len()); + buf.push_leb128(self.container_updates.len()); + buf.push_leb128(self.container_property_changes.len()); + buf.push_leb128(self.container_deletes.len()); + buf.push_leb128(self.item_inserts.len()); + buf.push_leb128(self.item_updates.len()); + buf.push_leb128(self.item_deletes.len()); - for list in [&self.inserts, &self.updates, &self.deletes] { + for list in [ + &self.container_inserts, + &self.container_updates, + &self.container_property_changes, + &self.container_deletes, + ] { for id in list { buf.push_leb128(*id); } } + for list in [&self.item_inserts, &self.item_updates, &self.item_deletes] { + for id in list { + buf.push_leb128(*id); + } + } + buf } } diff --git a/crates/store/src/write/mod.rs b/crates/store/src/write/mod.rs index 8af60c1b..5d806679 100644 --- a/crates/store/src/write/mod.rs +++ b/crates/store/src/write/mod.rs @@ -96,7 +96,7 @@ pub struct BatchBuilder { current_collection: Option, current_document_id: Option, changes: VecMap, - changed_collections: VecMap)>, + changed_collections: VecMap, has_assertions: bool, batch_size: usize, batch_ops: usize, @@ -104,6 +104,13 @@ pub struct BatchBuilder { ops: Vec, } +#[derive(Debug, Default)] +pub struct ChangedCollection { + pub change_id: u64, + pub changed_containers: Bitmap, + pub changed_items: Bitmap, +} + #[derive(Debug, PartialEq, Eq, Hash)] pub enum Operation { AccountId { diff --git a/tests/resources/jmap/email_set/headers.eml b/tests/resources/jmap/email_set/headers.eml index e6c688ba..0880e29b 100644 --- a/tests/resources/jmap/email_set/headers.eml +++ b/tests/resources/jmap/email_set/headers.eml @@ -21,11 +21,11 @@ Subject: Headers test To: "Greg Vaudreuil" , "Ned Freed" , "Keith Moore" X-AddressesGroup: "A Group": "Ed Jones" , - , "John" + , "John" ; X-AddressesGroup: "List 1": , - ; "List 2": , - ; , - + ;"List 2": , + ;, + ; X-References: <1234@local.machine.example> <3456@example.net> X-References: <789@local.machine.example> X-Text: a b diff --git a/tests/resources/jmap/email_set/headers.jmap b/tests/resources/jmap/email_set/headers.jmap index 58fd5ebd..6287d497 100644 --- a/tests/resources/jmap/email_set/headers.jmap +++ b/tests/resources/jmap/email_set/headers.jmap @@ -140,11 +140,11 @@ }, { "name": "X-AddressesGroup", - "value": " \"A Group\": \"Ed Jones\" , \r\n\t, \"John\" " + "value": " \"A Group\": \"Ed Jones\" , \r\n\t, \"John\" ;" }, { "name": "X-AddressesGroup", - "value": " \"List 1\": , \r\n\t; \"List 2\": , \r\n\t; , \r\n\t" + "value": " \"List 1\": , \r\n\t;\"List 2\": , \r\n\t;, \r\n\t;" }, { "name": "X-References", diff --git a/tests/resources/jmap/email_set/mixed.eml b/tests/resources/jmap/email_set/mixed.eml index 115cd2ad..13e41872 100644 --- a/tests/resources/jmap/email_set/mixed.eml +++ b/tests/resources/jmap/email_set/mixed.eml @@ -2,9 +2,9 @@ Date: Sat, 20 Nov 2021 22:22:01 +0000 From: "Art Vandelay (Vandelay Industries)" Message-ID: Subject: =?utf-8?Q?Why_not_both_importing_AND_exporting=3F_=E2=98=BA?= -To: "Colleagues": "James Smythe" ; +To: "Colleagues": "James Smythe" ; "Friends": , - "=?utf-8?Q?John_Sm=C3=AEth?=" + "=?utf-8?Q?John_Sm=C3=AEth?=" ; MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="boundary_0" diff --git a/tests/resources/jmap/email_set/mixed.jmap b/tests/resources/jmap/email_set/mixed.jmap index 723cc1ec..23ee2597 100644 --- a/tests/resources/jmap/email_set/mixed.jmap +++ b/tests/resources/jmap/email_set/mixed.jmap @@ -53,7 +53,7 @@ }, { "name": "To", - "value": " \"Colleagues\": \"James Smythe\" ; \r\n\t\"Friends\": , \r\n\t\"=?utf-8?Q?John_Sm=C3=AEth?=\" " + "value": " \"Colleagues\": \"James Smythe\" ;\r\n\t\"Friends\": , \r\n\t\"=?utf-8?Q?John_Sm=C3=AEth?=\" ;" }, { "name": "MIME-Version", diff --git a/tests/src/imap/basic.rs b/tests/src/imap/basic.rs index f3509f4d..d76c55ba 100644 --- a/tests/src/imap/basic.rs +++ b/tests/src/imap/basic.rs @@ -26,7 +26,7 @@ pub async fn test(imap: &mut ImapConnection, _imap_check: &mut ImapConnection) { imap.send("ID").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await - .assert_contains("* ID (\"name\" \"Stalwart IMAP\" \"version\" "); + .assert_contains("* ID (\"name\" \"Stalwart\" \"version\" "); // Login should be disabled imap.send("LOGIN jdoe@example.com secret").await; diff --git a/tests/src/imap/mod.rs b/tests/src/imap/mod.rs index ccc418a5..5e61a46d 100644 --- a/tests/src/imap/mod.rs +++ b/tests/src/imap/mod.rs @@ -20,13 +20,12 @@ pub mod search; pub mod store; pub mod thread; -use std::{ - path::PathBuf, - sync::Arc, - time::{Duration, Instant}, +use crate::{ + AssertConfig, add_test_certs, directory::internal::TestInternalDirectory, store::TempDir, }; - use ::managesieve::core::ManageSieveSessionManager; +use ::store::Stores; +use ahash::AHashSet; use common::{ Caches, Core, Data, Inner, Server, config::{ @@ -37,14 +36,16 @@ use common::{ manager::boot::build_ipc, }; use http::HttpSessionManager; - -use ::store::Stores; -use ahash::AHashSet; use imap::core::ImapSessionManager; use imap_proto::ResponseType; use pop3::Pop3SessionManager; use services::SpawnServices; use smtp::{SpawnQueueManager, core::SmtpSessionManager}; +use std::{ + path::PathBuf, + sync::Arc, + time::{Duration, Instant}, +}; use tokio::{ io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines, ReadHalf, WriteHalf}, net::TcpStream, @@ -52,250 +53,83 @@ use tokio::{ }; use utils::config::Config; -use crate::{ - AssertConfig, add_test_certs, directory::internal::TestInternalDirectory, store::TempDir, -}; +#[tokio::test] +pub async fn imap_tests() { + // Prepare settings + let start_time = Instant::now(); + let delete = true; + let handle = init_imap_tests( + &std::env::var("STORE") + .expect("Missing store type. Try running `STORE= cargo test`"), + delete, + ) + .await; -const SERVER: &str = r#" -[server] -hostname = "imap.example.org" + // Connect to IMAP server + let mut imap_check = ImapConnection::connect(b"_y ").await; + let mut imap = ImapConnection::connect(b"_x ").await; + for imap in [&mut imap, &mut imap_check] { + imap.assert_read(Type::Untagged, ResponseType::Ok).await; + } -[server.listener.imap] -bind = ["127.0.0.1:9991"] -protocol = "imap" -max-connections = 81920 + // Unauthenticated tests + basic::test(&mut imap, &mut imap_check).await; -[server.listener.imaptls] -bind = ["127.0.0.1:9992"] -protocol = "imap" -max-connections = 81920 -tls.implicit = true + // Login + for imap in [&mut imap, &mut imap_check] { + imap.send("AUTHENTICATE PLAIN {32+}\r\nAGpkb2VAZXhhbXBsZS5jb20Ac2VjcmV0") + .await; + imap.assert_read(Type::Tagged, ResponseType::Ok).await; + } -[server.listener.sieve] -bind = ["127.0.0.1:4190"] -protocol = "managesieve" -max-connections = 81920 -tls.implicit = true + // Delete folders + for mailbox in ["Drafts", "Junk Mail", "Sent Items"] { + imap.send(&format!("DELETE \"{}\"", mailbox)).await; + imap.assert_read(Type::Tagged, ResponseType::Ok).await; + } -[server.listener.pop3] -bind = ["127.0.0.1:4110"] -protocol = "pop3" -max-connections = 81920 -tls.implicit = true + mailbox::test(&mut imap, &mut imap_check).await; + append::test(&mut imap, &mut imap_check, &handle).await; + search::test(&mut imap, &mut imap_check).await; + fetch::test(&mut imap, &mut imap_check).await; + store::test(&mut imap, &mut imap_check, &handle).await; + copy_move::test(&mut imap, &mut imap_check).await; + thread::test(&mut imap, &mut imap_check).await; + idle::test(&mut imap, &mut imap_check).await; + condstore::test(&mut imap, &mut imap_check).await; + acl::test(&mut imap, &mut imap_check).await; -[server.listener.lmtp-debug] -bind = ['127.0.0.1:11201'] -greeting = 'Test LMTP instance' -protocol = 'lmtp' -tls.implicit = false + // Logout + for imap in [&mut imap, &mut imap_check] { + imap.send("UNAUTHENTICATE").await; + imap.assert_read(Type::Tagged, ResponseType::Ok).await; -[server.socket] -reuse-addr = true + imap.send("LOGOUT").await; + imap.assert_read(Type::Untagged, ResponseType::Bye).await; + } -[server.tls] -enable = true -implicit = false -certificate = "default" + // Bayes training + bayes::test(&handle).await; -[session.ehlo] -reject-non-fqdn = false + // Run ManageSieve tests + managesieve::test().await; -[session.rcpt] -relay = [ { if = "!is_empty(authenticated_as)", then = true }, - { else = false } ] -directory = "'{STORE}'" + // Run POP3 tests + pop::test().await; -[session.rcpt.errors] -total = 5 -wait = "1ms" + // Print elapsed time + let elapsed = start_time.elapsed(); + println!( + "Elapsed: {}.{:03}s", + elapsed.as_secs(), + elapsed.subsec_millis() + ); -[spam-filter] -enable = true - -[spam-filter.bayes.account] -enable = true - -[spam-filter.bayes.classify] -balance = "0.0" -learns = 10 - -[queue] -path = "{TMP}" -hash = 64 - -[report] -path = "{TMP}" -hash = 64 - -[resolver] -type = "system" - -[queue.outbound] -next-hop = [ { if = "rcpt_domain == 'example.com'", then = "'local'" }, - { if = "contains(['remote.org', 'foobar.com', 'test.com', 'other_domain.com'], rcpt_domain)", then = "'mock-smtp'" }, - { else = false } ] - -[remote."mock-smtp"] -address = "localhost" -port = 9999 -protocol = "smtp" - -[remote."mock-smtp".tls] -enable = false -allow-invalid-certs = true - -[session.data] -spam-filter = "recipients[0] != 'popper@example.com'" - -[session.data.add-headers] -delivered-to = false - -[session.extensions] -future-release = [ { if = "!is_empty(authenticated_as)", then = "99999999d"}, - { else = false } ] - -[store."sqlite"] -type = "sqlite" -path = "{TMP}/sqlite.db" - -[store."rocksdb"] -type = "rocksdb" -path = "{TMP}/rocks.db" - -[store."foundationdb"] -type = "foundationdb" - -[store."postgresql"] -type = "postgresql" -host = "localhost" -port = 5432 -database = "stalwart" -user = "postgres" -password = "mysecretpassword" - -[store."psql-replica"] -type = "sql-read-replica" -primary = "postgresql" -replicas = "postgresql" - -[store."mysql"] -type = "mysql" -host = "localhost" -port = 3307 -database = "stalwart" -user = "root" -password = "password" - -[store."elastic"] -type = "elasticsearch" -url = "https://localhost:9200" -user = "elastic" -password = "RtQ-Lu6+o4rxx=XJplVJ" -disable = true - -[store."elastic".tls] -allow-invalid-certs = true - -[certificate.default] -cert = "%{file:{CERT}}%" -private-key = "%{file:{PK}}%" - -[imap.protocol] -uidplus = true - -[storage] -data = "{STORE}" -fts = "{STORE}" -blob = "{STORE}" -lookup = "{STORE}" -directory = "{STORE}" - -[jmap.protocol] -set.max-objects = 100000 - -[jmap.protocol.request] -max-concurrent = 8 - -[jmap.protocol.upload] -max-size = 5000000 -max-concurrent = 4 -ttl = "1m" - -[jmap.protocol.upload.quota] -files = 3 -size = 50000 - -[jmap.rate-limit] -account = "1000/1m" -authentication = "100/2s" -anonymous = "100/1m" - -[jmap.event-source] -throttle = "500ms" - -[jmap.web-sockets] -throttle = "500ms" - -[jmap.push] -throttle = "500ms" -attempts.interval = "500ms" - -[jmap.folders.inbox] -name = "Inbox" -subscribe = false - -[jmap.folders.sent] -name = "Sent Items" -subscribe = false - -[jmap.folders.trash] -name = "Deleted Items" -subscribe = false - -[jmap.folders.junk] -name = "Junk Mail" -subscribe = false - -[jmap.folders.drafts] -name = "Drafts" -subscribe = false - -[store."auth"] -type = "sqlite" -path = "{TMP}/auth.db" - -[store."auth".query] -name = "SELECT name, type, secret, description, quota FROM accounts WHERE name = ? AND active = true" -members = "SELECT member_of FROM group_members WHERE name = ?" -recipients = "SELECT name FROM emails WHERE address = ?" -emails = "SELECT address FROM emails WHERE name = ? AND type != 'list' ORDER BY type DESC, address ASC" -verify = "SELECT address FROM emails WHERE address LIKE '%' || ? || '%' AND type = 'primary' ORDER BY address LIMIT 5" -expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = ? AND l.type = 'list' ORDER BY p.address LIMIT 50" -domains = "SELECT 1 FROM emails WHERE address LIKE '%@' || ? LIMIT 1" - -[directory."{STORE}"] -type = "internal" -store = "{STORE}" - -[oauth] -key = "parerga_und_paralipomena" -[oauth.auth] -max-attempts = 1 - -[oauth.expiry] -user-code = "1s" -token = "1s" -refresh-token = "3s" -refresh-token-renew = "2s" - -[tracer.console] -type = "console" -level = "{LEVEL}" -multiline = false -ansi = true -disabled-events = ["network.*"] - -"#; + // Remove test data + if delete { + handle.temp_dir.delete(); + } +} #[allow(dead_code)] pub struct IMAPTest { @@ -456,84 +290,6 @@ async fn init_imap_tests(store_id: &str, delete_if_exists: bool) -> IMAPTest { } } -#[tokio::test] -pub async fn imap_tests() { - // Prepare settings - let start_time = Instant::now(); - let delete = true; - let handle = init_imap_tests( - &std::env::var("STORE") - .expect("Missing store type. Try running `STORE= cargo test`"), - delete, - ) - .await; - - // Connect to IMAP server - let mut imap_check = ImapConnection::connect(b"_y ").await; - let mut imap = ImapConnection::connect(b"_x ").await; - for imap in [&mut imap, &mut imap_check] { - imap.assert_read(Type::Untagged, ResponseType::Ok).await; - } - - // Unauthenticated tests - basic::test(&mut imap, &mut imap_check).await; - - // Login - for imap in [&mut imap, &mut imap_check] { - imap.send("AUTHENTICATE PLAIN {32+}\r\nAGpkb2VAZXhhbXBsZS5jb20Ac2VjcmV0") - .await; - imap.assert_read(Type::Tagged, ResponseType::Ok).await; - } - - // Delete folders - for mailbox in ["Drafts", "Junk Mail", "Sent Items"] { - imap.send(&format!("DELETE \"{}\"", mailbox)).await; - imap.assert_read(Type::Tagged, ResponseType::Ok).await; - } - - mailbox::test(&mut imap, &mut imap_check).await; - append::test(&mut imap, &mut imap_check, &handle).await; - search::test(&mut imap, &mut imap_check).await; - fetch::test(&mut imap, &mut imap_check).await; - store::test(&mut imap, &mut imap_check, &handle).await; - copy_move::test(&mut imap, &mut imap_check).await; - thread::test(&mut imap, &mut imap_check).await; - idle::test(&mut imap, &mut imap_check).await; - condstore::test(&mut imap, &mut imap_check).await; - acl::test(&mut imap, &mut imap_check).await; - - // Logout - for imap in [&mut imap, &mut imap_check] { - imap.send("UNAUTHENTICATE").await; - imap.assert_read(Type::Tagged, ResponseType::Ok).await; - - imap.send("LOGOUT").await; - imap.assert_read(Type::Untagged, ResponseType::Bye).await; - } - - // Bayes training - bayes::test(&handle).await; - - // Run ManageSieve tests - managesieve::test().await; - - // Run POP3 tests - pop::test().await; - - // Print elapsed time - let elapsed = start_time.elapsed(); - println!( - "Elapsed: {}.{:03}s", - elapsed.as_secs(), - elapsed.subsec_millis() - ); - - // Remove test data - if delete { - handle.temp_dir.delete(); - } -} - pub struct ImapConnection { tag: &'static [u8], reader: Lines>>, @@ -832,3 +588,244 @@ fn resources_dir() -> PathBuf { resources.push("imap"); resources } + +const SERVER: &str = r#" +[server] +hostname = "imap.example.org" + +[server.listener.imap] +bind = ["127.0.0.1:9991"] +protocol = "imap" +max-connections = 81920 + +[server.listener.imaptls] +bind = ["127.0.0.1:9992"] +protocol = "imap" +max-connections = 81920 +tls.implicit = true + +[server.listener.sieve] +bind = ["127.0.0.1:4190"] +protocol = "managesieve" +max-connections = 81920 +tls.implicit = true + +[server.listener.pop3] +bind = ["127.0.0.1:4110"] +protocol = "pop3" +max-connections = 81920 +tls.implicit = true + +[server.listener.lmtp-debug] +bind = ['127.0.0.1:11201'] +greeting = 'Test LMTP instance' +protocol = 'lmtp' +tls.implicit = false + +[server.socket] +reuse-addr = true + +[server.tls] +enable = true +implicit = false +certificate = "default" + +[session.ehlo] +reject-non-fqdn = false + +[session.rcpt] +relay = [ { if = "!is_empty(authenticated_as)", then = true }, + { else = false } ] +directory = "'{STORE}'" + +[session.rcpt.errors] +total = 5 +wait = "1ms" + +[spam-filter] +enable = true + +[spam-filter.bayes.account] +enable = true + +[spam-filter.bayes.classify] +balance = "0.0" +learns = 10 + +[queue] +path = "{TMP}" +hash = 64 + +[report] +path = "{TMP}" +hash = 64 + +[resolver] +type = "system" + +[queue.outbound] +next-hop = [ { if = "rcpt_domain == 'example.com'", then = "'local'" }, + { if = "contains(['remote.org', 'foobar.com', 'test.com', 'other_domain.com'], rcpt_domain)", then = "'mock-smtp'" }, + { else = false } ] + +[remote."mock-smtp"] +address = "localhost" +port = 9999 +protocol = "smtp" + +[remote."mock-smtp".tls] +enable = false +allow-invalid-certs = true + +[session.data] +spam-filter = "recipients[0] != 'popper@example.com'" + +[session.data.add-headers] +delivered-to = false + +[session.extensions] +future-release = [ { if = "!is_empty(authenticated_as)", then = "99999999d"}, + { else = false } ] + +[store."sqlite"] +type = "sqlite" +path = "{TMP}/sqlite.db" + +[store."rocksdb"] +type = "rocksdb" +path = "{TMP}/rocks.db" + +[store."foundationdb"] +type = "foundationdb" + +[store."postgresql"] +type = "postgresql" +host = "localhost" +port = 5432 +database = "stalwart" +user = "postgres" +password = "mysecretpassword" + +[store."psql-replica"] +type = "sql-read-replica" +primary = "postgresql" +replicas = "postgresql" + +[store."mysql"] +type = "mysql" +host = "localhost" +port = 3307 +database = "stalwart" +user = "root" +password = "password" + +[store."elastic"] +type = "elasticsearch" +url = "https://localhost:9200" +user = "elastic" +password = "RtQ-Lu6+o4rxx=XJplVJ" +disable = true + +[store."elastic".tls] +allow-invalid-certs = true + +[certificate.default] +cert = "%{file:{CERT}}%" +private-key = "%{file:{PK}}%" + +[imap.protocol] +uidplus = true + +[storage] +data = "{STORE}" +fts = "{STORE}" +blob = "{STORE}" +lookup = "{STORE}" +directory = "{STORE}" + +[jmap.protocol] +set.max-objects = 100000 + +[jmap.protocol.request] +max-concurrent = 8 + +[jmap.protocol.upload] +max-size = 5000000 +max-concurrent = 4 +ttl = "1m" + +[jmap.protocol.upload.quota] +files = 3 +size = 50000 + +[jmap.rate-limit] +account = "1000/1m" +authentication = "100/2s" +anonymous = "100/1m" + +[jmap.event-source] +throttle = "500ms" + +[jmap.web-sockets] +throttle = "500ms" + +[jmap.push] +throttle = "500ms" +attempts.interval = "500ms" + +[jmap.folders.inbox] +name = "Inbox" +subscribe = false + +[jmap.folders.sent] +name = "Sent Items" +subscribe = false + +[jmap.folders.trash] +name = "Deleted Items" +subscribe = false + +[jmap.folders.junk] +name = "Junk Mail" +subscribe = false + +[jmap.folders.drafts] +name = "Drafts" +subscribe = false + +[store."auth"] +type = "sqlite" +path = "{TMP}/auth.db" + +[store."auth".query] +name = "SELECT name, type, secret, description, quota FROM accounts WHERE name = ? AND active = true" +members = "SELECT member_of FROM group_members WHERE name = ?" +recipients = "SELECT name FROM emails WHERE address = ?" +emails = "SELECT address FROM emails WHERE name = ? AND type != 'list' ORDER BY type DESC, address ASC" +verify = "SELECT address FROM emails WHERE address LIKE '%' || ? || '%' AND type = 'primary' ORDER BY address LIMIT 5" +expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = ? AND l.type = 'list' ORDER BY p.address LIMIT 50" +domains = "SELECT 1 FROM emails WHERE address LIKE '%@' || ? LIMIT 1" + +[directory."{STORE}"] +type = "internal" +store = "{STORE}" + +[oauth] +key = "parerga_und_paralipomena" +[oauth.auth] +max-attempts = 1 + +[oauth.expiry] +user-code = "1s" +token = "1s" +refresh-token = "3s" +refresh-token-renew = "2s" + +[tracer.console] +type = "console" +level = "{LEVEL}" +multiline = false +ansi = true +disabled-events = ["network.*"] + +"#; diff --git a/tests/src/jmap/delivery.rs b/tests/src/jmap/delivery.rs index 3419ec00..450b0f23 100644 --- a/tests/src/jmap/delivery.rs +++ b/tests/src/jmap/delivery.rs @@ -4,25 +4,22 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::time::Duration; - use email::{ + cache::{MessageCacheFetch, email::MessageCacheAccess}, mailbox::{INBOX_ID, JUNK_ID}, - message::cache::{MessageCacheAccess, MessageCacheFetch}, }; use jmap_proto::types::{collection::Collection, id::Id}; +use std::time::Duration; -use tokio::{ - io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines, ReadHalf, WriteHalf}, - net::TcpStream, -}; - +use super::JMAPTest; use crate::{ directory::internal::TestInternalDirectory, jmap::{assert_is_empty, mailbox::destroy_all_mailboxes}, }; - -use super::JMAPTest; +use tokio::{ + io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines, ReadHalf, WriteHalf}, + net::TcpStream, +}; pub async fn test(params: &mut JMAPTest) { println!("Running message delivery tests..."); diff --git a/tests/src/jmap/email_changes.rs b/tests/src/jmap/email_changes.rs index 5a75c5af..5e82c5b2 100644 --- a/tests/src/jmap/email_changes.rs +++ b/tests/src/jmap/email_changes.rs @@ -6,7 +6,11 @@ use jmap_proto::{ parser::{JsonObjectParser, json::Parser}, - types::{collection::Collection, id::Id, state::State}, + types::{ + collection::{Collection, SyncCollection}, + id::Id, + state::State, + }, }; use store::{ahash::AHashSet, write::BatchBuilder}; @@ -139,23 +143,29 @@ pub async fn test(params: &mut JMAPTest) { for change in changes { match change { LogAction::Insert(id) => { - batch.update_document(id as u32).log_insert(None); + batch + .update_document(id as u32) + .log_item_insert(SyncCollection::Email, None); } LogAction::Update(id) => { - batch.update_document(id as u32).log_update(None); + batch + .update_document(id as u32) + .log_item_update(SyncCollection::Email, None); } LogAction::Delete(id) => { - batch.update_document(id as u32).log_delete(None); + batch + .update_document(id as u32) + .log_item_delete(SyncCollection::Email, None); } LogAction::UpdateChild(id) => { - batch.log_parent_update(Collection::Email, id as u32); + batch.log_container_property_change(SyncCollection::Email, id as u32); } LogAction::Move(old_id, new_id) => { batch .update_document(old_id as u32) - .log_delete(None) + .log_item_delete(SyncCollection::Email, None) .update_document(new_id as u32) - .log_insert(None); + .log_item_insert(SyncCollection::Email, None); } } } @@ -297,7 +307,7 @@ pub async fn test(params: &mut JMAPTest) { let changes = params .client - .email_changes(State::Initial.to_string(), 0.into()) + .email_changes(State::Initial.to_string(), None) .await .unwrap(); let mut created = changes diff --git a/tests/src/jmap/email_query.rs b/tests/src/jmap/email_query.rs index d9e6e611..49f88f75 100644 --- a/tests/src/jmap/email_query.rs +++ b/tests/src/jmap/email_query.rs @@ -6,12 +6,12 @@ use std::{collections::hash_map::Entry, time::Instant}; +use super::JMAPTest; use crate::{ jmap::{assert_is_empty, mailbox::destroy_all_mailboxes, wait_for_index}, store::{deflate_test_resource, query::FIELDS}, }; - -use ::email::{mailbox::Mailbox, message::cache::MessageCacheFetch}; +use ::email::{cache::MessageCacheFetch, mailbox::Mailbox}; use ahash::AHashSet; use common::{config::jmap::settings::SpecialUse, storage::index::ObjectIndexBuilder}; use jmap_client::{ @@ -21,14 +21,11 @@ use jmap_client::{ }; use jmap_proto::types::{collection::Collection, id::Id}; use mail_parser::{DateTime, HeaderName}; - use store::{ ahash::AHashMap, write::{BatchBuilder, now}, }; -use super::JMAPTest; - const MAX_THREADS: usize = 100; const MAX_MESSAGES: usize = 1000; const MAX_MESSAGES_PER_THREAD: usize = 100; @@ -79,6 +76,7 @@ pub async fn test(params: &mut JMAPTest, insert: bool) { .get_cached_messages(account_id) .await .unwrap() + .emails .items .iter() .map(|m| m.thread_id) diff --git a/tests/src/jmap/email_query_changes.rs b/tests/src/jmap/email_query_changes.rs index 99ad3703..42e06722 100644 --- a/tests/src/jmap/email_query_changes.rs +++ b/tests/src/jmap/email_query_changes.rs @@ -11,7 +11,11 @@ use jmap_client::{ email, mailbox::Role, }; -use jmap_proto::types::{collection::Collection, id::Id, state::State}; +use jmap_proto::types::{ + collection::{Collection, SyncCollection}, + id::Id, + state::State, +}; use store::{ ahash::{AHashMap, AHashSet}, @@ -123,7 +127,7 @@ pub async fn test(params: &mut JMAPTest) { let mut batch = BatchBuilder::new(); batch .update_document(id.document_id()) - .log_update(id.prefix_id().into()); + .log_item_update(SyncCollection::Email, id.prefix_id().into()); server.store().write(batch.build_all()).await.unwrap(); updated_ids.insert(id); } diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 13953a84..26b28580 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -80,289 +80,6 @@ pub mod vacation_response; pub mod webhooks; pub mod websocket; -const SERVER: &str = r#" -[server] -hostname = "'jmap.example.org'" -http.url = "'https://127.0.0.1:8899'" - -[server.listener.jmap] -bind = ["127.0.0.1:8899"] -protocol = "http" -max-connections = 81920 -tls.implicit = true - -[server.listener.imap] -bind = ["127.0.0.1:9991"] -protocol = "imap" -max-connections = 81920 - -[server.listener.lmtp-debug] -bind = ['127.0.0.1:11200'] -greeting = 'Test LMTP instance' -protocol = 'lmtp' -tls.implicit = false - -[server.listener.pop3] -bind = ["127.0.0.1:4110"] -protocol = "pop3" -max-connections = 81920 -tls.implicit = true - -[server.socket] -reuse-addr = true - -[server.tls] -enable = true -implicit = false -certificate = "default" - -[server.fail2ban] -authentication = "100/5s" - -[authentication] -rate-limit = "100/2s" - -[session.ehlo] -reject-non-fqdn = false - -[session.rcpt] -relay = [ { if = "!is_empty(authenticated_as)", then = true }, - { else = false } ] -directory = "'{STORE}'" - -[session.rcpt.errors] -total = 5 -wait = "1ms" - -[session.auth] -mechanisms = "[plain, login, oauthbearer]" -directory = "'{STORE}'" - -[session.data] -spam-filter = "recipients[0] != 'robert@example.com'" - -[session.data.add-headers] -delivered-to = false - -[queue] -path = "{TMP}" -hash = 64 - -[report] -path = "{TMP}" -hash = 64 - -[resolver] -type = "system" - -[queue.outbound] -next-hop = [ { if = "rcpt_domain == 'example.com'", then = "'local'" }, - { if = "contains(['remote.org', 'foobar.com', 'test.com', 'other_domain.com'], rcpt_domain)", then = "'mock-smtp'" }, - { else = false } ] - -[remote."mock-smtp"] -address = "localhost" -port = 9999 -protocol = "smtp" - -[remote."mock-smtp".tls] -implicit = false -allow-invalid-certs = true - -[session.extensions] -future-release = [ { if = "!is_empty(authenticated_as)", then = "99999999d"}, - { else = false } ] - -[store."sqlite"] -type = "sqlite" -path = "{TMP}/sqlite.db" - -[store."rocksdb"] -type = "rocksdb" -path = "{TMP}/rocks.db" - -[store."foundationdb"] -type = "foundationdb" - -[store."postgresql"] -type = "postgresql" -host = "localhost" -port = 5432 -database = "stalwart" -user = "postgres" -password = "mysecretpassword" - -[store."mysql"] -type = "mysql" -host = "localhost" -port = 3307 -database = "stalwart" -user = "root" -password = "password" - -[store."elastic"] -type = "elasticsearch" -url = "https://localhost:9200" -user = "elastic" -password = "changeme" -tls.allow-invalid-certs = true -disable = true - -[certificate.default] -cert = "%{file:{CERT}}%" -private-key = "%{file:{PK}}%" - -[storage] -data = "{STORE}" -fts = "{STORE}" -blob = "{STORE}" -lookup = "{STORE}" -directory = "{STORE}" - -[jmap.protocol.get] -max-objects = 100000 - -[jmap.protocol.set] -max-objects = 100000 - -[jmap.protocol.request] -max-concurrent = 8 - -[jmap.protocol.upload] -max-size = 5000000 -max-concurrent = 4 -ttl = "1m" - -[jmap.protocol.upload.quota] -files = 3 -size = 50000 - -[jmap.rate-limit] -account = "1000/1m" -anonymous = "100/1m" - -[jmap.event-source] -throttle = "500ms" - -[jmap.web-sockets] -throttle = "500ms" - -[jmap.push] -throttle = "500ms" -attempts.interval = "500ms" - -[jmap.email] -auto-expunge = "1s" - -[jmap.protocol.changes] -max-history = "1s" - -[store."auth"] -type = "sqlite" -path = "{TMP}/auth.db" - -[store."auth".query] -name = "SELECT name, type, secret, description, quota FROM accounts WHERE name = ? AND active = true" -members = "SELECT member_of FROM group_members WHERE name = ?" -recipients = "SELECT name FROM emails WHERE address = ?" -emails = "SELECT address FROM emails WHERE name = ? AND type != 'list' ORDER BY type DESC, address ASC" -verify = "SELECT address FROM emails WHERE address LIKE '%' || ? || '%' AND type = 'primary' ORDER BY address LIMIT 5" -expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = ? AND l.type = 'list' ORDER BY p.address LIMIT 50" -domains = "SELECT 1 FROM emails WHERE address LIKE '%@' || ? LIMIT 1" - -[directory."{STORE}"] -type = "internal" -store = "{STORE}" - -[imap.auth] -allow-plain-text = true - -[oauth] -key = "parerga_und_paralipomena" - -[oauth.auth] -max-attempts = 1 - -[oauth.expiry] -user-code = "1s" -token = "1s" -refresh-token = "3s" -refresh-token-renew = "2s" - -[oauth.client-registration] -anonymous = true -require = true - -[oauth.oidc] -signature-key = '''-----BEGIN PRIVATE KEY----- -MIIEuwIBADANBgkqhkiG9w0BAQEFAASCBKUwggShAgEAAoIBAQDMXJI1bL3z8gaF -Ze/6493VjL+jHkFMP2Pc7fLwRF1fhkuIdYTp69LabzrSEJCRCz0UI2NHqPOgtOta -+zRHKAMr7c7Z6uKO0K+aXiQYHw4Y70uSG8CnmNl7kb4OM/CAcoO6fePmvBsyESfn -TmkJ5bfHEZQFDQEAoDlDjtjxuwYsAQQVQXuAydi8j8pyTWKAJ1RDgnUT+HbOub7j -JrQ7sPe6MPCjXv5N76v9RMHKktfYwRNMlkLkxImQU55+vlvghNztgFlIlJDFfNiy -UQPV5FTEZJli9BzMoj1JQK3sZyV8WV0W1zN41QQ+glAAC6+K7iTDPRMINBSwbHyn -6Lb9Q6U7AgMBAAECggEAB93qZ5xrhYgEFeoyKO4mUdGsu4qZyJB0zNeWGgdaXCfZ -zC4l8zFM+R6osix0EY6lXRtC95+6h9hfFQNa5FWseupDzmIQiEnim1EowjWef87l -Eayi0nDRB8TjqZKjR/aLOUhzrPlXHKrKEUk/RDkacCiDklwz9S0LIfLOSXlByBDM -/n/eczfX2gUATexMHSeIXs8vN2jpuiVv0r+FPXcRvqdzDZnYSzS8BJ9k6RYXVQ4o -NzCbfqgFIpVryB7nHgSTrNX9G7299If8/dXmesXWSFEJvvDSSpcBoINKbfgSlrxd -6ubjiotcEIBUSlbaanRrydwShhLHnXyupNAb7tlvyQKBgQDsIipSK4+H9FGl1rAk -Gg9DLJ7P/94sidhoq1KYnj/CxwGLoRq22khZEUYZkSvYXDu1Qkj9Avi3TRhw8uol -l2SK1VylL5FQvTLKhWB7b2hjrUd5llMRgS3/NIdLhOgDMB7w3UxJnCA/df/Rj+dM -WhkyS1f0x3t7XPLwWGurW0nJcwKBgQDdjhrNfabrK7OQvDpAvNJizuwZK9WUL7CD -rR0V0MpDGYW12BTEOY6tUK6XZgiRitAXf4EkEI6R0Q0bFzwDDLrg7TvGdTuzNeg/ -8vm8IlRlOkrdihtHZI4uRB7Ytmz24vzywEBE0p6enA7v4oniscUks/KKmDGr0V90 -yT9gIVrjGQKBgQCjnWC5otlHGLDiOgm+WhgtMWOxN9dYAQNkMyF+Alinu4CEoVKD -VGhA3sk1ufMpbW8pvw4X0dFIITFIQeift3DBCemxw23rBc2FqjkaDi3EszINO22/ -eUTHyjvcxfCFFPi7aHsNnhJyJm7lY9Kegudmg/Ij93zGE7d5darVBuHvpQKBgBBY -YovUgFMLR1UfPeD2zUKy52I4BKrJFemxBNtOKw3mPSIcTfPoFymcMTVENs+eARoq -svlZK1uAo8ni3e+Pqd3cQrOyhHQFPxwwrdH+amGJemp7vOV4erDZH7l3Q/S27Fhw -bI1nSIKFGukBupB58wRxLiyha9C0QqmYC0/pRg5JAn8Rbj5tP26oVCXjZEfWJL8J -axxSxsGA4Vol6i6LYnVgZG+1ez2rP8vUORo1lRzmdeP4o1BSJf9TPwXkuppE5J+t -UZVKtYGlEn1RqwGNd8I9TiWvU84rcY9nsxlDR86xwKRWFvYqVOiGYtzRyewYRdjU -rTs9aqB3v1+OVxGxR6Na ------END PRIVATE KEY----- -''' -signature-algorithm = "RS256" - -[oauth.oidc-ignore] -signature-key = '''-----BEGIN PRIVATE KEY----- -MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQggybcqc86ulFFiOon -WiYrLO4z8/kmkqvA7wGElBok9IqhRANCAAQxZK68FnQtHC0eyh8CA05xRIvxhVHn -0ymka6XBh9aFtW4wfeoKhTkSKjHc/zjh9Rr2dr3kvmYe80fMGhW4ycGA ------END PRIVATE KEY----- -''' -signature-algorithm = "ES256" - -[session.extensions] -expn = true -vrfy = true - -[spam-filter] -enable = true - -[tracer.console] -type = "console" -level = "{LEVEL}" -multiline = false -ansi = true -disabled-events = ["network.*", "telemetry.webhook-error", "http.request-body"] - -[webhook."test"] -url = "http://127.0.0.1:8821/hook" -events = ["auth.*", "delivery.dsn*", "message-ingest.*", "security.authentication-ban"] -signature-key = "ovos-moles" -throttle = "100ms" - -[sieve.untrusted.scripts."common"] -contents = ''' -require "reject"; - -reject "Rejected from a global script."; -stop; -''' -"#; - #[test] fn jmap_tests() { tokio::runtime::Builder::new_multi_thread() @@ -403,12 +120,12 @@ async fn jmap_tests_() { event_source::test(&mut params).await; push_subscription::test(&mut params).await; sieve_script::test(&mut params).await; - vacation_response::test(&mut params).await;*/ + vacation_response::test(&mut params).await; email_submission::test(&mut params).await; websocket::test(&mut params).await; quota::test(&mut params).await; crypto::test(&mut params).await; - blob::test(&mut params).await; + blob::test(&mut params).await;*/ permissions::test(¶ms).await; purge::test(&mut params).await; enterprise::test(&mut params).await; @@ -522,7 +239,6 @@ pub async fn assert_is_empty(server: Server) { .await; // Clean cache - server.inner.cache.mailboxes.clear(); server.inner.cache.messages.clear(); } @@ -1030,3 +746,286 @@ impl Response { } } } + +const SERVER: &str = r#" +[server] +hostname = "'jmap.example.org'" +http.url = "'https://127.0.0.1:8899'" + +[server.listener.jmap] +bind = ["127.0.0.1:8899"] +protocol = "http" +max-connections = 81920 +tls.implicit = true + +[server.listener.imap] +bind = ["127.0.0.1:9991"] +protocol = "imap" +max-connections = 81920 + +[server.listener.lmtp-debug] +bind = ['127.0.0.1:11200'] +greeting = 'Test LMTP instance' +protocol = 'lmtp' +tls.implicit = false + +[server.listener.pop3] +bind = ["127.0.0.1:4110"] +protocol = "pop3" +max-connections = 81920 +tls.implicit = true + +[server.socket] +reuse-addr = true + +[server.tls] +enable = true +implicit = false +certificate = "default" + +[server.fail2ban] +authentication = "100/5s" + +[authentication] +rate-limit = "100/2s" + +[session.ehlo] +reject-non-fqdn = false + +[session.rcpt] +relay = [ { if = "!is_empty(authenticated_as)", then = true }, + { else = false } ] +directory = "'{STORE}'" + +[session.rcpt.errors] +total = 5 +wait = "1ms" + +[session.auth] +mechanisms = "[plain, login, oauthbearer]" +directory = "'{STORE}'" + +[session.data] +spam-filter = "recipients[0] != 'robert@example.com'" + +[session.data.add-headers] +delivered-to = false + +[queue] +path = "{TMP}" +hash = 64 + +[report] +path = "{TMP}" +hash = 64 + +[resolver] +type = "system" + +[queue.outbound] +next-hop = [ { if = "rcpt_domain == 'example.com'", then = "'local'" }, + { if = "contains(['remote.org', 'foobar.com', 'test.com', 'other_domain.com'], rcpt_domain)", then = "'mock-smtp'" }, + { else = false } ] + +[remote."mock-smtp"] +address = "localhost" +port = 9999 +protocol = "smtp" + +[remote."mock-smtp".tls] +implicit = false +allow-invalid-certs = true + +[session.extensions] +future-release = [ { if = "!is_empty(authenticated_as)", then = "99999999d"}, + { else = false } ] + +[store."sqlite"] +type = "sqlite" +path = "{TMP}/sqlite.db" + +[store."rocksdb"] +type = "rocksdb" +path = "{TMP}/rocks.db" + +[store."foundationdb"] +type = "foundationdb" + +[store."postgresql"] +type = "postgresql" +host = "localhost" +port = 5432 +database = "stalwart" +user = "postgres" +password = "mysecretpassword" + +[store."mysql"] +type = "mysql" +host = "localhost" +port = 3307 +database = "stalwart" +user = "root" +password = "password" + +[store."elastic"] +type = "elasticsearch" +url = "https://localhost:9200" +user = "elastic" +password = "changeme" +tls.allow-invalid-certs = true +disable = true + +[certificate.default] +cert = "%{file:{CERT}}%" +private-key = "%{file:{PK}}%" + +[storage] +data = "{STORE}" +fts = "{STORE}" +blob = "{STORE}" +lookup = "{STORE}" +directory = "{STORE}" + +[jmap.protocol.get] +max-objects = 100000 + +[jmap.protocol.set] +max-objects = 100000 + +[jmap.protocol.request] +max-concurrent = 8 + +[jmap.protocol.upload] +max-size = 5000000 +max-concurrent = 4 +ttl = "1m" + +[jmap.protocol.upload.quota] +files = 3 +size = 50000 + +[jmap.rate-limit] +account = "1000/1m" +anonymous = "100/1m" + +[jmap.event-source] +throttle = "500ms" + +[jmap.web-sockets] +throttle = "500ms" + +[jmap.push] +throttle = "500ms" +attempts.interval = "500ms" + +[jmap.email] +auto-expunge = "1s" + +[jmap.protocol.changes] +max-history = "1s" + +[store."auth"] +type = "sqlite" +path = "{TMP}/auth.db" + +[store."auth".query] +name = "SELECT name, type, secret, description, quota FROM accounts WHERE name = ? AND active = true" +members = "SELECT member_of FROM group_members WHERE name = ?" +recipients = "SELECT name FROM emails WHERE address = ?" +emails = "SELECT address FROM emails WHERE name = ? AND type != 'list' ORDER BY type DESC, address ASC" +verify = "SELECT address FROM emails WHERE address LIKE '%' || ? || '%' AND type = 'primary' ORDER BY address LIMIT 5" +expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = ? AND l.type = 'list' ORDER BY p.address LIMIT 50" +domains = "SELECT 1 FROM emails WHERE address LIKE '%@' || ? LIMIT 1" + +[directory."{STORE}"] +type = "internal" +store = "{STORE}" + +[imap.auth] +allow-plain-text = true + +[oauth] +key = "parerga_und_paralipomena" + +[oauth.auth] +max-attempts = 1 + +[oauth.expiry] +user-code = "1s" +token = "1s" +refresh-token = "3s" +refresh-token-renew = "2s" + +[oauth.client-registration] +anonymous = true +require = true + +[oauth.oidc] +signature-key = '''-----BEGIN PRIVATE KEY----- +MIIEuwIBADANBgkqhkiG9w0BAQEFAASCBKUwggShAgEAAoIBAQDMXJI1bL3z8gaF +Ze/6493VjL+jHkFMP2Pc7fLwRF1fhkuIdYTp69LabzrSEJCRCz0UI2NHqPOgtOta ++zRHKAMr7c7Z6uKO0K+aXiQYHw4Y70uSG8CnmNl7kb4OM/CAcoO6fePmvBsyESfn +TmkJ5bfHEZQFDQEAoDlDjtjxuwYsAQQVQXuAydi8j8pyTWKAJ1RDgnUT+HbOub7j +JrQ7sPe6MPCjXv5N76v9RMHKktfYwRNMlkLkxImQU55+vlvghNztgFlIlJDFfNiy +UQPV5FTEZJli9BzMoj1JQK3sZyV8WV0W1zN41QQ+glAAC6+K7iTDPRMINBSwbHyn +6Lb9Q6U7AgMBAAECggEAB93qZ5xrhYgEFeoyKO4mUdGsu4qZyJB0zNeWGgdaXCfZ +zC4l8zFM+R6osix0EY6lXRtC95+6h9hfFQNa5FWseupDzmIQiEnim1EowjWef87l +Eayi0nDRB8TjqZKjR/aLOUhzrPlXHKrKEUk/RDkacCiDklwz9S0LIfLOSXlByBDM +/n/eczfX2gUATexMHSeIXs8vN2jpuiVv0r+FPXcRvqdzDZnYSzS8BJ9k6RYXVQ4o +NzCbfqgFIpVryB7nHgSTrNX9G7299If8/dXmesXWSFEJvvDSSpcBoINKbfgSlrxd +6ubjiotcEIBUSlbaanRrydwShhLHnXyupNAb7tlvyQKBgQDsIipSK4+H9FGl1rAk +Gg9DLJ7P/94sidhoq1KYnj/CxwGLoRq22khZEUYZkSvYXDu1Qkj9Avi3TRhw8uol +l2SK1VylL5FQvTLKhWB7b2hjrUd5llMRgS3/NIdLhOgDMB7w3UxJnCA/df/Rj+dM +WhkyS1f0x3t7XPLwWGurW0nJcwKBgQDdjhrNfabrK7OQvDpAvNJizuwZK9WUL7CD +rR0V0MpDGYW12BTEOY6tUK6XZgiRitAXf4EkEI6R0Q0bFzwDDLrg7TvGdTuzNeg/ +8vm8IlRlOkrdihtHZI4uRB7Ytmz24vzywEBE0p6enA7v4oniscUks/KKmDGr0V90 +yT9gIVrjGQKBgQCjnWC5otlHGLDiOgm+WhgtMWOxN9dYAQNkMyF+Alinu4CEoVKD +VGhA3sk1ufMpbW8pvw4X0dFIITFIQeift3DBCemxw23rBc2FqjkaDi3EszINO22/ +eUTHyjvcxfCFFPi7aHsNnhJyJm7lY9Kegudmg/Ij93zGE7d5darVBuHvpQKBgBBY +YovUgFMLR1UfPeD2zUKy52I4BKrJFemxBNtOKw3mPSIcTfPoFymcMTVENs+eARoq +svlZK1uAo8ni3e+Pqd3cQrOyhHQFPxwwrdH+amGJemp7vOV4erDZH7l3Q/S27Fhw +bI1nSIKFGukBupB58wRxLiyha9C0QqmYC0/pRg5JAn8Rbj5tP26oVCXjZEfWJL8J +axxSxsGA4Vol6i6LYnVgZG+1ez2rP8vUORo1lRzmdeP4o1BSJf9TPwXkuppE5J+t +UZVKtYGlEn1RqwGNd8I9TiWvU84rcY9nsxlDR86xwKRWFvYqVOiGYtzRyewYRdjU +rTs9aqB3v1+OVxGxR6Na +-----END PRIVATE KEY----- +''' +signature-algorithm = "RS256" + +[oauth.oidc-ignore] +signature-key = '''-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQggybcqc86ulFFiOon +WiYrLO4z8/kmkqvA7wGElBok9IqhRANCAAQxZK68FnQtHC0eyh8CA05xRIvxhVHn +0ymka6XBh9aFtW4wfeoKhTkSKjHc/zjh9Rr2dr3kvmYe80fMGhW4ycGA +-----END PRIVATE KEY----- +''' +signature-algorithm = "ES256" + +[session.extensions] +expn = true +vrfy = true + +[spam-filter] +enable = true + +[tracer.console] +type = "console" +level = "{LEVEL}" +multiline = false +ansi = true +disabled-events = ["network.*", "telemetry.webhook-error", "http.request-body"] + +[webhook."test"] +url = "http://127.0.0.1:8821/hook" +events = ["auth.*", "delivery.dsn*", "message-ingest.*", "security.authentication-ban"] +signature-key = "ovos-moles" +throttle = "100ms" + +[sieve.untrusted.scripts."common"] +contents = ''' +require "reject"; + +reject "Rejected from a global script."; +stop; +''' +"#; diff --git a/tests/src/jmap/purge.rs b/tests/src/jmap/purge.rs index 452525c2..d9293186 100644 --- a/tests/src/jmap/purge.rs +++ b/tests/src/jmap/purge.rs @@ -4,27 +4,23 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use ahash::AHashSet; -use common::Server; -use directory::{QueryBy, backend::internal::manage::ManageDirectory}; -use email::{ - mailbox::{INBOX_ID, JUNK_ID, TRASH_ID}, - message::{ - cache::{MessageCacheAccess, MessageCacheFetch}, - delete::EmailDeletion, - }, -}; -use imap_proto::ResponseType; -use jmap_proto::types::{collection::Collection, id::Id}; -use store::{IterateParams, LogKey, U32_LEN, U64_LEN, write::key::DeserializeBigEndian}; - +use super::JMAPTest; use crate::{ directory::internal::TestInternalDirectory, imap::{AssertResult, ImapConnection, Type}, jmap::assert_is_empty, }; - -use super::JMAPTest; +use ahash::AHashSet; +use common::Server; +use directory::{QueryBy, backend::internal::manage::ManageDirectory}; +use email::{ + cache::{MessageCacheFetch, email::MessageCacheAccess}, + mailbox::{INBOX_ID, JUNK_ID, TRASH_ID}, + message::delete::EmailDeletion, +}; +use imap_proto::ResponseType; +use jmap_proto::types::{collection::Collection, id::Id}; +use store::{IterateParams, LogKey, U32_LEN, U64_LEN, write::key::DeserializeBigEndian}; pub async fn test(params: &mut JMAPTest) { println!("Running purge tests..."); diff --git a/tests/src/jmap/stress_test.rs b/tests/src/jmap/stress_test.rs index 52264fb1..8c4dbabb 100644 --- a/tests/src/jmap/stress_test.rs +++ b/tests/src/jmap/stress_test.rs @@ -6,12 +6,13 @@ use std::{sync::Arc, time::Duration}; +use super::assert_is_empty; use crate::jmap::{mailbox::destroy_all_mailboxes_no_wait, wait_for_index}; use common::Server; use directory::backend::internal::manage::ManageDirectory; -use email::message::{ - cache::{MessageCacheAccess, MessageCacheFetch}, - metadata::MessageData, +use email::{ + cache::{MessageCacheFetch, email::MessageCacheAccess}, + message::metadata::MessageData, }; use futures::future::join_all; use jmap_client::{ @@ -25,8 +26,6 @@ use store::{ roaring::RoaringBitmap, }; -use super::assert_is_empty; - const TEST_USER_ID: u32 = 1; const NUM_PASSES: usize = 1; diff --git a/tests/src/store/import_export.rs b/tests/src/store/import_export.rs index 13898efc..d926370e 100644 --- a/tests/src/store/import_export.rs +++ b/tests/src/store/import_export.rs @@ -6,7 +6,10 @@ use ahash::AHashSet; use common::{Core, manager::backup::BackupParams}; -use jmap_proto::types::{collection::Collection, property::Property}; +use jmap_proto::types::{ + collection::{Collection, SyncCollection}, + property::Property, +}; use store::{ rand, write::{ @@ -98,7 +101,7 @@ pub async fn test(db: Store) { ); } - batch.log_insert(None); + batch.log_item_insert(SyncCollection::from(collection), None); /*batch.any_op(Operation::ChangeId { change_id: document_id as u64 + account_id as u64 + collection as u64,