From 2826ff1548669959f34c3687f5eefd9412a21dfc Mon Sep 17 00:00:00 2001 From: mdecimus Date: Tue, 25 Feb 2025 17:38:02 +0100 Subject: [PATCH] How I stopped worrying and learned to love zero-copy deserialization --- Cargo.lock | 207 ++++++++++++-- Cargo.toml | 1 + crates/common/Cargo.toml | 1 + crates/common/src/config/inner.rs | 8 +- crates/common/src/config/jmap/settings.rs | 36 ++- crates/common/src/core.rs | 7 +- crates/common/src/lib.rs | 12 +- crates/common/src/listener/acme/directory.rs | 4 +- crates/common/src/manager/backup.rs | 17 +- crates/common/src/manager/restore.rs | 14 +- .../src/auth => common/src/sharing}/acl.rs | 268 +----------------- crates/common/src/sharing/document.rs | 169 +++++++++++ crates/common/src/sharing/mod.rs | 47 +++ crates/common/src/storage/blob.rs | 35 +++ .../object => common/src/storage}/index.rs | 154 +++------- crates/common/src/storage/mod.rs | 10 + crates/common/src/storage/state.rs | 39 +++ crates/common/src/storage/tag.rs | 143 ++++++++++ .../directory/src/backend/internal/manage.rs | 27 +- crates/directory/src/backend/internal/mod.rs | 21 +- crates/email/Cargo.toml | 1 + crates/email/src/identity/mod.rs | 16 +- crates/email/src/identity/serialize.rs | 23 -- crates/email/src/mailbox/destroy.rs | 239 ++++++++++++++++ crates/email/src/mailbox/index.rs | 31 +- crates/email/src/mailbox/manage.rs | 35 ++- crates/email/src/mailbox/mod.rs | 9 +- crates/email/src/mailbox/serialize.rs | 46 +-- crates/email/src/message/copy.rs | 235 +++++++++++++++ crates/email/src/message/crypto.rs | 23 +- crates/email/src/message/delete.rs | 28 +- crates/email/src/message/index.rs | 160 +++++++---- crates/email/src/message/ingest.rs | 39 +-- crates/email/src/message/mod.rs | 1 + crates/email/src/push/mod.rs | 17 +- crates/email/src/push/serialize.rs | 23 -- crates/email/src/sieve/activate.rs | 127 +++++++++ crates/email/src/sieve/delete.rs | 81 ++++++ crates/email/src/sieve/index.rs | 14 +- crates/email/src/sieve/ingest.rs | 89 +++--- crates/email/src/sieve/mod.rs | 56 +++- crates/email/src/sieve/serialize.rs | 32 +-- crates/email/src/submission/index.rs | 10 +- crates/email/src/submission/mod.rs | 78 ++++- crates/email/src/submission/serialize.rs | 23 -- crates/groupware/Cargo.toml | 20 ++ crates/groupware/src/calendar/mod.rs | 91 ++++++ crates/groupware/src/contact/mod.rs | 51 ++++ crates/groupware/src/file/mod.rs | 13 + crates/groupware/src/lib.rs | 3 + crates/imap-proto/src/protocol/capability.rs | 4 +- crates/imap-proto/src/protocol/expunge.rs | 2 +- crates/imap-proto/src/protocol/select.rs | 4 +- crates/imap/Cargo.toml | 1 - crates/imap/src/core/mailbox.rs | 91 ++++-- crates/imap/src/core/message.rs | 8 +- crates/imap/src/op/acl.rs | 56 ++-- crates/imap/src/op/copy_move.rs | 14 +- crates/imap/src/op/create.rs | 24 +- crates/imap/src/op/delete.rs | 4 +- crates/imap/src/op/expunge.rs | 16 +- crates/imap/src/op/fetch.rs | 19 +- crates/imap/src/op/idle.rs | 9 +- crates/imap/src/op/rename.rs | 30 +- crates/imap/src/op/search.rs | 62 ++-- crates/imap/src/op/status.rs | 51 +--- crates/imap/src/op/store.rs | 15 +- crates/imap/src/op/subscribe.rs | 22 +- crates/jmap-proto/Cargo.toml | 3 +- crates/jmap-proto/src/object/mod.rs | 1 - crates/jmap-proto/src/types/acl.rs | 53 ++-- crates/jmap-proto/src/types/collection.rs | 78 ++++- crates/jmap-proto/src/types/date.rs | 6 +- crates/jmap-proto/src/types/keyword.rs | 44 +-- crates/jmap-proto/src/types/property.rs | 8 +- crates/jmap-proto/src/types/type_state.rs | 28 +- crates/jmap-proto/src/types/value.rs | 36 ++- crates/jmap/src/api/event_source.rs | 8 +- crates/jmap/src/api/form.rs | 2 +- crates/jmap/src/api/management/stores.rs | 28 +- crates/jmap/src/api/request.rs | 18 +- crates/jmap/src/api/session.rs | 6 +- crates/jmap/src/auth/mod.rs | 1 - crates/jmap/src/auth/oauth/auth.rs | 9 +- crates/jmap/src/blob/copy.rs | 6 +- crates/jmap/src/blob/download.rs | 52 +--- crates/jmap/src/changes/get.rs | 50 ++-- crates/jmap/src/email/copy.rs | 241 +--------------- crates/jmap/src/email/crypto.rs | 16 +- crates/jmap/src/email/get.rs | 10 +- crates/jmap/src/email/import.rs | 3 +- crates/jmap/src/email/query.rs | 25 +- crates/jmap/src/email/set.rs | 151 ++-------- crates/jmap/src/email/snippet.rs | 9 +- crates/jmap/src/identity/get.rs | 45 +-- crates/jmap/src/identity/set.rs | 22 +- crates/jmap/src/mailbox/get.rs | 52 ++-- crates/jmap/src/mailbox/query.rs | 29 +- crates/jmap/src/mailbox/set.rs | 256 +++-------------- crates/jmap/src/push/get.rs | 24 +- crates/jmap/src/push/set.rs | 20 +- crates/jmap/src/services/index.rs | 2 +- crates/jmap/src/services/state.rs | 48 +--- crates/jmap/src/sieve/get.rs | 31 +- crates/jmap/src/sieve/query.rs | 21 +- crates/jmap/src/sieve/set.rs | 244 ++++------------ crates/jmap/src/submission/get.rs | 111 +++++--- crates/jmap/src/submission/query.rs | 24 +- crates/jmap/src/submission/set.rs | 49 +++- crates/jmap/src/vacation/get.rs | 43 +-- crates/jmap/src/vacation/set.rs | 33 ++- crates/jmap/src/websocket/stream.rs | 18 +- crates/main/Cargo.toml | 2 + crates/managesieve/Cargo.toml | 1 - crates/managesieve/src/core/client.rs | 11 +- crates/managesieve/src/op/deletescript.rs | 2 +- crates/managesieve/src/op/getscript.rs | 26 +- crates/managesieve/src/op/listscripts.rs | 8 +- crates/managesieve/src/op/putscript.rs | 65 ++--- crates/managesieve/src/op/renamescript.rs | 20 +- crates/managesieve/src/op/setactive.rs | 2 +- crates/pop3/Cargo.toml | 1 - crates/pop3/src/mailbox.rs | 45 +-- crates/pop3/src/op/fetch.rs | 4 +- crates/smtp/src/queue/spool.rs | 103 ++++--- crates/smtp/src/reporting/analysis.rs | 22 +- crates/smtp/src/reporting/dmarc.rs | 53 ++-- crates/smtp/src/reporting/tls.rs | 60 ++-- crates/spam-filter/src/analysis/reputation.rs | 20 +- crates/store/Cargo.toml | 8 +- crates/store/src/backend/foundationdb/read.rs | 10 +- crates/store/src/backend/mysql/read.rs | 14 +- crates/store/src/backend/postgres/read.rs | 8 +- crates/store/src/backend/redis/lookup.rs | 4 +- crates/store/src/backend/rocksdb/read.rs | 6 +- crates/store/src/dispatch/lookup.rs | 15 +- crates/store/src/fts/index.rs | 10 +- crates/store/src/fts/postings.rs | 10 +- crates/store/src/lib.rs | 17 +- crates/store/src/query/filter.rs | 26 +- crates/store/src/query/mod.rs | 35 ++- crates/store/src/write/assert.rs | 13 +- crates/store/src/write/batch.rs | 128 ++++++--- crates/store/src/write/log.rs | 9 +- crates/store/src/write/mod.rs | 249 +++++++--------- crates/trc/Cargo.toml | 1 + crates/trc/src/event/conv.rs | 8 + crates/utils/Cargo.toml | 1 + crates/utils/src/lib.rs | 22 +- crates/utils/src/map/bitmap.rs | 39 ++- crates/utils/src/map/vec_map.rs | 27 +- tests/Cargo.toml | 6 +- tests/src/jmap/email_changes.rs | 5 +- tests/src/jmap/email_query_changes.rs | 11 +- tests/src/store/blob.rs | 146 +++++----- tests/src/store/ops.rs | 13 +- tests/src/store/query.rs | 80 +++--- 157 files changed, 3822 insertions(+), 2774 deletions(-) rename crates/{jmap/src/auth => common/src/sharing}/acl.rs (50%) create mode 100644 crates/common/src/sharing/document.rs create mode 100644 crates/common/src/sharing/mod.rs create mode 100644 crates/common/src/storage/blob.rs rename crates/{jmap-proto/src/object => common/src/storage}/index.rs (75%) create mode 100644 crates/common/src/storage/mod.rs create mode 100644 crates/common/src/storage/state.rs create mode 100644 crates/common/src/storage/tag.rs delete mode 100644 crates/email/src/identity/serialize.rs create mode 100644 crates/email/src/mailbox/destroy.rs create mode 100644 crates/email/src/message/copy.rs delete mode 100644 crates/email/src/push/serialize.rs create mode 100644 crates/email/src/sieve/activate.rs create mode 100644 crates/email/src/sieve/delete.rs delete mode 100644 crates/email/src/submission/serialize.rs create mode 100644 crates/groupware/Cargo.toml create mode 100644 crates/groupware/src/calendar/mod.rs create mode 100644 crates/groupware/src/contact/mod.rs create mode 100644 crates/groupware/src/file/mod.rs create mode 100644 crates/groupware/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 54e73c77..e9bb4465 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -932,8 +932,20 @@ version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" dependencies = [ - "bytecheck_derive", - "ptr_meta", + "bytecheck_derive 0.6.12", + "ptr_meta 0.1.4", + "simdutf8", +] + +[[package]] +name = "bytecheck" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50690fb3370fb9fe3550372746084c46f2ac8c9685c583d2be10eefd89d3d1a3" +dependencies = [ + "bytecheck_derive 0.8.1", + "ptr_meta 0.3.0", + "rancor", "simdutf8", ] @@ -948,6 +960,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "bytecheck_derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efb7846e0cb180355c2dec69e721edafa36919850f1a9f52ffba4ebc0393cb71" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "bytemuck" version = "1.21.0" @@ -987,6 +1010,15 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "calcard" +version = "0.1.0" +dependencies = [ + "hashify", + "mail-builder", + "mail-parser", +] + [[package]] name = "camellia" version = "0.1.0" @@ -1250,6 +1282,7 @@ dependencies = [ "regex", "reqwest 0.12.12", "ring 0.17.8", + "rkyv 0.8.10", "rsa", "rustls 0.23.21", "rustls-pemfile 2.2.0", @@ -2071,6 +2104,7 @@ dependencies = [ "rasn", "rasn-cms", "rasn-pkix", + "rkyv 0.8.10", "rsa", "sequoia-openpgp", "serde", @@ -2591,6 +2625,16 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "gethostname" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fd4b8790c0792e3b11895efdf5f289ebe8b59107a6624f1cce68f24ff8c7035" +dependencies = [ + "rustix", + "windows-targets 0.52.6", +] + [[package]] name = "getrandom" version = "0.1.16" @@ -2660,6 +2704,19 @@ dependencies = [ "subtle", ] +[[package]] +name = "groupware" +version = "0.11.5" +dependencies = [ + "calcard", + "common", + "directory", + "hashify", + "jmap_proto", + "tokio", + "utils", +] + [[package]] name = "h2" version = "0.3.26" @@ -2741,11 +2798,11 @@ dependencies = [ [[package]] name = "hashlink" -version = "0.9.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" dependencies = [ - "hashbrown 0.14.5", + "hashbrown 0.15.2", ] [[package]] @@ -3306,7 +3363,6 @@ dependencies = [ "directory", "email", "imap_proto", - "jmap", "jmap_proto", "mail-parser", "mail-send", @@ -3629,7 +3685,9 @@ version = "0.11.8" dependencies = [ "ahash 0.8.11", "fast-float", + "hashify", "mail-parser", + "rkyv 0.8.10", "serde", "serde_json", "store", @@ -3841,9 +3899,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.30.1" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +checksum = "ad8935b44e7c13394a179a438e0cebba0fe08fe01b54f152e29a93b5cf993fd4" dependencies = [ "cc", "pkg-config", @@ -3969,11 +4027,11 @@ dependencies = [ [[package]] name = "mail-builder" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d75c72a0bf2070b4c2aa384f173439569a9e0e97293b90aad3469b6d9c183ec4" +checksum = "f2d2c992d0b4d0acedb466c5524bb09327f4d0bb5bcfacea13005ec01c09f2ac" dependencies = [ - "gethostname", + "gethostname 1.0.0", ] [[package]] @@ -3994,7 +4052,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b12277cdcacfc15af67fe9cf155f31ff68ad8c301304573ea116ed8870f192d5" dependencies = [ "base64 0.22.1", - "gethostname", + "gethostname 0.5.0", "md5", "rustls 0.23.21", "rustls-pki-types", @@ -4010,6 +4068,7 @@ version = "0.11.8" dependencies = [ "common", "directory", + "email", "imap", "jemallocator", "jmap", @@ -4035,7 +4094,6 @@ dependencies = [ "email", "imap", "imap_proto", - "jmap", "jmap_proto", "mail-parser", "mail-send", @@ -4181,6 +4239,26 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "munge" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8743b8dfaf66acac79aca9ff2440e8680fef745b6260e6a31d1772b14cfa2862" +dependencies = [ + "munge_macro", +] + +[[package]] +name = "munge_macro" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66191390a55bb9830fa8468c12634442ea4199c6e390ddf08ddcace35b3cd5da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "mysql-common-derive" version = "0.31.2" @@ -4856,7 +4934,6 @@ dependencies = [ "directory", "email", "imap", - "jmap", "jmap_proto", "mail-parser", "mail-send", @@ -5084,7 +5161,16 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" dependencies = [ - "ptr_meta_derive", + "ptr_meta_derive 0.1.4", +] + +[[package]] +name = "ptr_meta" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe9e76f66d3f9606f44e45598d155cb13ecf09f4a28199e48daf8c8fc937ea90" +dependencies = [ + "ptr_meta_derive 0.3.0", ] [[package]] @@ -5098,6 +5184,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "ptr_meta_derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca414edb151b4c8d125c12566ab0d74dc9cdba36fb80eb7b848c15f495fd32d1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "pwhash" version = "1.0.0" @@ -5248,6 +5345,15 @@ dependencies = [ "nibble_vec", ] +[[package]] +name = "rancor" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf5f7161924b9d1cea0e4cabc97c372cea92b5f927fc13c6bca67157a0ad947" +dependencies = [ + "ptr_meta 0.3.0", +] + [[package]] name = "rand" version = "0.7.3" @@ -5453,28 +5559,24 @@ dependencies = [ [[package]] name = "redis" -version = "0.27.6" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d8f99a4090c89cc489a94833c901ead69bfbf3877b4867d5482e321ee875bc" +checksum = "9568894e8bdefd16512bca9e286a9d2abc27773609aa4eb7f428497d64df4373" dependencies = [ "arc-swap", - "async-trait", "bytes", "combine", "crc16", - "futures", + "futures-sink", "futures-util", - "itertools 0.13.0", "itoa", "log", "num-bigint", "percent-encoding", "pin-project-lite", - "rand 0.8.5", + "rand 0.9.0", "rustls 0.23.21", "rustls-native-certs 0.7.3", - "rustls-pemfile 2.2.0", - "rustls-pki-types", "ryu", "sha1_smol", "socket2", @@ -5540,7 +5642,16 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" dependencies = [ - "bytecheck", + "bytecheck 0.6.12", +] + +[[package]] +name = "rend" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a35e8a6bf28cd121053a66aa2e6a2e3eaffad4a60012179f0e864aa5ffeff215" +dependencies = [ + "bytecheck 0.8.1", ] [[package]] @@ -5710,17 +5821,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" dependencies = [ "bitvec", - "bytecheck", + "bytecheck 0.6.12", "bytes", "hashbrown 0.12.3", - "ptr_meta", - "rend", - "rkyv_derive", + "ptr_meta 0.1.4", + "rend 0.4.2", + "rkyv_derive 0.7.45", "seahash", "tinyvec", "uuid", ] +[[package]] +name = "rkyv" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e147371c75553e1e2fcdb483944a8540b8438c31426279553b9a8182a9b7b65" +dependencies = [ + "bytecheck 0.8.1", + "bytes", + "hashbrown 0.15.2", + "indexmap 2.7.1", + "munge", + "ptr_meta 0.3.0", + "rancor", + "rend 0.5.2", + "rkyv_derive 0.8.10", + "tinyvec", + "uuid", +] + [[package]] name = "rkyv_derive" version = "0.7.45" @@ -5732,6 +5862,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "rkyv_derive" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "246b40ac189af6c675d124b802e8ef6d5246c53e17367ce9501f8f66a81abb7a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "rle-decode-fast" version = "1.0.3" @@ -5807,9 +5948,9 @@ checksum = "ad8388ea1a9e0ea807e442e8263a699e7edcb320ecbcd21b4fa8ff859acce3ba" [[package]] name = "rusqlite" -version = "0.32.1" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +checksum = "1c6d5e5acb6f6129fe3f7ba0a7fc77bca1942cb568535e18e7bc40262baf3110" dependencies = [ "bitflags 2.8.0", "fallible-iterator 0.3.0", @@ -5888,7 +6029,7 @@ dependencies = [ "bytes", "num-traits", "rand 0.8.5", - "rkyv", + "rkyv 0.7.45", "serde", "serde_json", ] @@ -6734,6 +6875,7 @@ dependencies = [ "futures", "lru-cache", "lz4_flex", + "memchr", "mysql_async", "nlp", "num_cpus", @@ -6745,6 +6887,7 @@ dependencies = [ "regex", "reqwest 0.12.12", "ring 0.17.8", + "rkyv 0.8.10", "roaring", "rocksdb", "rusqlite", @@ -7371,6 +7514,7 @@ dependencies = [ "mail-parser", "parking_lot", "reqwest 0.12.12", + "rkyv 0.8.10", "rtrb", "serde", "serde_json", @@ -7638,6 +7782,7 @@ dependencies = [ "regex", "reqwest 0.12.12", "ring 0.17.8", + "rkyv 0.8.10", "rustls 0.23.21", "rustls-pemfile 2.2.0", "rustls-pki-types", diff --git a/Cargo.toml b/Cargo.toml index f055474f..e5619fc1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "crates/smtp", "crates/managesieve", "crates/pop3", + "crates/groupware", "crates/spam-filter", "crates/nlp", "crates/store", diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index d3890197..b8d56032 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -67,6 +67,7 @@ p256 = { version = "0.13", features = ["ecdh"] } p384 = { version = "0.13", features = ["ecdh"] } num_cpus = "1.13.1" hashify = "0.2" +rkyv = { version = "0.8.10", features = ["little_endian"] } [target.'cfg(unix)'.dependencies] privdrop = "0.5.3" diff --git a/crates/common/src/config/inner.rs b/crates/common/src/config/inner.rs index f424e374..31e2c286 100644 --- a/crates/common/src/config/inner.rs +++ b/crates/common/src/config/inner.rs @@ -11,7 +11,7 @@ use std::{ use ahash::{AHashMap, AHashSet}; use arc_swap::ArcSwap; -use mail_auth::{Parameters, Txt, MX}; +use mail_auth::{MX, Parameters, Txt}; use mail_send::smtp::tls::build_tls_connector; use nlp::bayes::{TokenHash, Weights}; use parking_lot::RwLock; @@ -22,12 +22,12 @@ use utils::{ }; use crate::{ - auth::{roles::RolePermissions, AccessToken}, + Account, AccountId, Caches, Data, Mailbox, MailboxId, MailboxState, NextMailboxState, Threads, + TlsConnectors, + auth::{AccessToken, roles::RolePermissions}, config::smtp::resolver::{Policy, Tlsa}, listener::blocked::BlockedIps, manager::webadmin::WebAdminManager, - Account, AccountId, Caches, Data, Mailbox, MailboxId, MailboxState, NextMailboxState, Threads, - TlsConnectors, }; use super::server::tls::{build_self_signed_cert, parse_certificates}; diff --git a/crates/common/src/config/jmap/settings.rs b/crates/common/src/config/jmap/settings.rs index 294f7467..b2506ebe 100644 --- a/crates/common/src/config/jmap/settings.rs +++ b/crates/common/src/config/jmap/settings.rs @@ -84,7 +84,9 @@ pub struct DefaultFolder { pub create: bool, } -#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +#[derive( + rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Clone, Copy, PartialEq, Eq, Hash, Debug, +)] pub enum SpecialUse { Inbox, Trash, @@ -403,3 +405,35 @@ impl SpecialUse { } } } + +impl ArchivedSpecialUse { + pub fn as_str(&self) -> Option<&'static str> { + match self { + ArchivedSpecialUse::Inbox => Some("inbox"), + ArchivedSpecialUse::Trash => Some("trash"), + ArchivedSpecialUse::Junk => Some("junk"), + ArchivedSpecialUse::Drafts => Some("drafts"), + ArchivedSpecialUse::Archive => Some("archive"), + ArchivedSpecialUse::Sent => Some("sent"), + ArchivedSpecialUse::Shared => Some("shared"), + ArchivedSpecialUse::Important => Some("important"), + ArchivedSpecialUse::None => None, + } + } +} + +impl From<&ArchivedSpecialUse> for SpecialUse { + fn from(value: &ArchivedSpecialUse) -> Self { + match value { + ArchivedSpecialUse::Inbox => SpecialUse::Inbox, + ArchivedSpecialUse::Trash => SpecialUse::Trash, + ArchivedSpecialUse::Junk => SpecialUse::Junk, + ArchivedSpecialUse::Drafts => SpecialUse::Drafts, + ArchivedSpecialUse::Archive => SpecialUse::Archive, + ArchivedSpecialUse::Sent => SpecialUse::Sent, + ArchivedSpecialUse::Shared => SpecialUse::Shared, + ArchivedSpecialUse::Important => SpecialUse::Important, + ArchivedSpecialUse::None => SpecialUse::None, + } + } +} diff --git a/crates/common/src/core.rs b/crates/common/src/core.rs index 7cab68b5..f6cefa87 100644 --- a/crates/common/src/core.rs +++ b/crates/common/src/core.rs @@ -13,7 +13,7 @@ use jmap_proto::types::{ use sieve::Sieve; use store::{ BitmapKey, BlobClass, BlobStore, Deserialize, FtsStore, InMemoryStore, IndexKey, IterateParams, - LogKey, Serialize, Store, U32_LEN, ValueKey, + LogKey, SerializeInfallible, Store, U32_LEN, ValueKey, dispatch::DocumentSet, roaring::RoaringBitmap, write::{ @@ -534,7 +534,10 @@ impl Server { let state = changes.change_id; let mut builder = BatchBuilder::new(); - builder.with_account_id(account_id).custom(changes); + builder + .with_account_id(account_id) + .custom(changes) + .caused_by(trc::location!())?; self.core .storage .data diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index c7ffdf0a..32e6ab32 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -9,22 +9,22 @@ use std::{ hash::{BuildHasher, Hasher}, net::{IpAddr, Ipv4Addr, Ipv6Addr}, sync::{ - atomic::{AtomicBool, AtomicU8}, Arc, + atomic::{AtomicBool, AtomicU8}, }, }; use ahash::{AHashMap, AHashSet}; use arc_swap::ArcSwap; -use auth::{oauth::config::OAuthConfig, roles::RolePermissions, AccessToken}; +use auth::{AccessToken, oauth::config::OAuthConfig, roles::RolePermissions}; use config::{ imap::ImapConfig, jmap::settings::JmapConfig, network::Network, scripts::Scripting, smtp::{ - resolver::{Policy, Tlsa}, SmtpConfig, + resolver::{Policy, Tlsa}, }, spamfilter::{IpResolver, SpamFilterConfig}, storage::Storage, @@ -35,12 +35,12 @@ use imap_proto::protocol::list::Attribute; use ipc::{HousekeeperEvent, QueueEvent, ReportingEvent, StateEvent}; use listener::{asn::AsnGeoLookupData, blocked::Security, tls::AcmeProviders}; -use mail_auth::{Txt, MX}; +use mail_auth::{MX, Txt}; use manager::webadmin::{Resource, WebAdminManager}; use nlp::bayes::{TokenHash, Weights}; use parking_lot::{Mutex, RwLock}; use rustls::sign::CertifiedKey; -use tokio::sync::{mpsc, Notify, Semaphore}; +use tokio::sync::{Notify, Semaphore, mpsc}; use tokio_rustls::TlsConnector; use utils::{ cache::{Cache, CacheItemWeight, CacheWithTtl}, @@ -59,6 +59,8 @@ pub mod ipc; pub mod listener; pub mod manager; pub mod scripts; +pub mod sharing; +pub mod storage; pub mod telemetry; pub use psl; diff --git a/crates/common/src/listener/acme/directory.rs b/crates/common/src/listener/acme/directory.rs index 6eedc5a9..dd66ecbd 100644 --- a/crates/common/src/listener/acme/directory.rs +++ b/crates/common/src/listener/acme/directory.rs @@ -190,7 +190,7 @@ impl Account { .reason(err) })?; - Ok(Bincode::new(SerializedCert { + Bincode::new(SerializedCert { certificate: cert.serialize_der().map_err(|err| { trc::EventType::Acme(trc::AcmeEvent::Error) .caused_by(trc::location!()) @@ -198,7 +198,7 @@ impl Account { })?, private_key: cert.serialize_private_key_der(), }) - .serialize()) + .serialize() } } diff --git a/crates/common/src/manager/backup.rs b/crates/common/src/manager/backup.rs index 4498cab0..b2b452b4 100644 --- a/crates/common/src/manager/backup.rs +++ b/crates/common/src/manager/backup.rs @@ -15,17 +15,18 @@ use std::{ use ahash::{AHashMap, AHashSet}; use jmap_proto::types::{collection::Collection, property::Property}; use store::{ + BitmapKey, Deserialize, IndexKey, IterateParams, LogKey, SUBSPACE_BITMAP_ID, + SUBSPACE_BITMAP_TAG, SUBSPACE_BITMAP_TEXT, SerializeInfallible, U32_LEN, U64_LEN, ValueKey, write::{ - key::DeserializeBigEndian, AnyKey, BitmapClass, BitmapHash, BlobOp, DirectoryClass, - InMemoryClass, QueueClass, QueueEvent, TagValue, ValueClass, + AnyKey, BitmapClass, BitmapHash, BlobOp, DirectoryClass, InMemoryClass, QueueClass, + QueueEvent, TagValue, ValueClass, key::DeserializeBigEndian, }, - BitmapKey, Deserialize, IndexKey, IterateParams, LogKey, Serialize, ValueKey, - SUBSPACE_BITMAP_ID, SUBSPACE_BITMAP_TAG, SUBSPACE_BITMAP_TEXT, U32_LEN, U64_LEN, }; use utils::{ - codec::leb128::{Leb128Reader, Leb128_}, - failed, BlobHash, UnwrapFailure, BLOB_HASH_LEN, + BLOB_HASH_LEN, BlobHash, UnwrapFailure, + codec::leb128::{Leb128_, Leb128Reader}, + failed, }; use crate::Core; @@ -465,8 +466,8 @@ impl Core { .failed("Failed to send key value"); } else { eprintln!( - "Warning: blob hash {hash:?} does not exist in blob store. Skipping." - ); + "Warning: blob hash {hash:?} does not exist in blob store. Skipping." + ); } } } diff --git a/crates/common/src/manager/restore.rs b/crates/common/src/manager/restore.rs index 8c9c754b..7861b51e 100644 --- a/crates/common/src/manager/restore.rs +++ b/crates/common/src/manager/restore.rs @@ -12,25 +12,25 @@ use std::{ use crate::Core; use jmap_proto::types::{collection::Collection, property::Property}; use store::{ + BlobStore, SerializeInfallible, Store, U32_LEN, roaring::RoaringBitmap, write::{ - key::DeserializeBigEndian, BatchBuilder, BitmapClass, BitmapHash, BlobOp, DirectoryClass, - InMemoryClass, MaybeDynamicId, MaybeDynamicValue, Operation, TagValue, TaskQueueClass, - ValueClass, + BatchBuilder, BitmapClass, BitmapHash, BlobOp, DirectoryClass, InMemoryClass, + MaybeDynamicId, MaybeDynamicValue, Operation, TagValue, TaskQueueClass, ValueClass, + key::DeserializeBigEndian, }, - BlobStore, Serialize, Store, U32_LEN, }; use store::{ - write::{QueueClass, QueueEvent}, Deserialize, U64_LEN, + write::{QueueClass, QueueEvent}, }; use tokio::{ fs::File, io::{AsyncReadExt, BufReader}, }; -use utils::{failed, BlobHash, UnwrapFailure}; +use utils::{BlobHash, UnwrapFailure, failed}; -use super::backup::{DeserializeBytes, Family, Op, FILE_VERSION, MAGIC_MARKER}; +use super::backup::{DeserializeBytes, FILE_VERSION, Family, MAGIC_MARKER, Op}; impl Core { pub async fn restore(&self, src: PathBuf) { diff --git a/crates/jmap/src/auth/acl.rs b/crates/common/src/sharing/acl.rs similarity index 50% rename from crates/jmap/src/auth/acl.rs rename to crates/common/src/sharing/acl.rs index 439eb4e1..c31e67c9 100644 --- a/crates/jmap/src/auth/acl.rs +++ b/crates/common/src/sharing/acl.rs @@ -4,9 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::future::Future; - -use common::{Server, auth::AccessToken}; use directory::{ QueryBy, Type, backend::internal::{PrincipalField, manage::ChangedPrincipals}, @@ -15,239 +12,16 @@ use jmap_proto::{ error::set::SetError, types::{ acl::Acl, - collection::Collection, property::Property, - value::{AclGrant, MaybePatchValue, Value}, + value::{AclGrant, ArchivedAclGrant, MaybePatchValue, Value}, }, }; -use store::{ValueKey, query::acl::AclQuery, roaring::RoaringBitmap, write::ValueClass}; -use trc::AddContext; use utils::map::bitmap::Bitmap; -pub trait AclMethods: Sync + Send { - fn shared_documents( - &self, - access_token: &AccessToken, - to_account_id: u32, - to_collection: Collection, - check_acls: impl Into> + Send, - ) -> impl Future> + Send; +use crate::{Server, auth::AccessToken}; - fn shared_messages( - &self, - access_token: &AccessToken, - to_account_id: u32, - check_acls: impl Into> + Send, - ) -> impl Future> + Send; - - fn owned_or_shared_documents( - &self, - access_token: &AccessToken, - account_id: u32, - collection: Collection, - check_acls: impl Into> + Send, - ) -> impl Future> + Send; - - fn owned_or_shared_messages( - &self, - access_token: &AccessToken, - account_id: u32, - check_acls: impl Into> + Send, - ) -> impl Future> + Send; - - fn has_access_to_document( - &self, - access_token: &AccessToken, - to_account_id: u32, - to_collection: impl Into + Send, - to_document_id: u32, - check_acls: impl Into> + Send, - ) -> impl Future> + Send; - - fn acl_set( - &self, - changes: &mut Vec, - current: Option<&[AclGrant]>, - acl_changes: MaybePatchValue, - ) -> impl Future> + Send; - - fn acl_get( - &self, - value: &[AclGrant], - access_token: &AccessToken, - account_id: u32, - ) -> impl Future + Send; - - fn refresh_acls( - &self, - changes: &[AclGrant], - current: Option<&[AclGrant]>, - ) -> impl Future + Send; - - fn map_acl_set( - &self, - acl_set: Vec, - ) -> impl Future, SetError>> + Send; - - fn map_acl_patch( - &self, - acl_patch: Vec, - ) -> impl Future), SetError>> + Send; -} - -impl AclMethods for Server { - async fn shared_documents( - &self, - access_token: &AccessToken, - to_account_id: u32, - to_collection: Collection, - check_acls: impl Into>, - ) -> trc::Result { - let check_acls = check_acls.into(); - let mut document_ids = RoaringBitmap::new(); - let to_collection = u8::from(to_collection); - for &grant_account_id in [access_token.primary_id] - .iter() - .chain(access_token.member_of.clone().iter()) - { - for acl_item in self - .core - .storage - .data - .acl_query(AclQuery::SharedWith { - grant_account_id, - to_account_id, - to_collection, - }) - .await - .caused_by(trc::location!())? - { - let mut acls = Bitmap::::from(acl_item.permissions); - - acls.intersection(&check_acls); - if !acls.is_empty() { - document_ids.insert(acl_item.to_document_id); - } - } - } - - Ok(document_ids) - } - - async fn shared_messages( - &self, - access_token: &AccessToken, - to_account_id: u32, - check_acls: impl Into>, - ) -> trc::Result { - let check_acls = check_acls.into(); - let shared_mailboxes = self - .shared_documents(access_token, to_account_id, Collection::Mailbox, check_acls) - .await?; - if shared_mailboxes.is_empty() { - return Ok(shared_mailboxes); - } - let mut shared_messages = RoaringBitmap::new(); - for mailbox_id in shared_mailboxes { - if let Some(messages_in_mailbox) = self - .get_tag( - to_account_id, - Collection::Email, - Property::MailboxIds, - mailbox_id, - ) - .await? - { - shared_messages |= messages_in_mailbox; - } - } - - Ok(shared_messages) - } - - async fn owned_or_shared_documents( - &self, - access_token: &AccessToken, - account_id: u32, - collection: Collection, - check_acls: impl Into>, - ) -> trc::Result { - let check_acls = check_acls.into(); - let mut document_ids = self - .get_document_ids(account_id, collection) - .await? - .unwrap_or_default(); - if !document_ids.is_empty() && !access_token.is_member(account_id) { - document_ids &= self - .shared_documents(access_token, account_id, collection, check_acls) - .await?; - } - Ok(document_ids) - } - - async fn owned_or_shared_messages( - &self, - access_token: &AccessToken, - account_id: u32, - check_acls: impl Into>, - ) -> trc::Result { - let check_acls = check_acls.into(); - let mut document_ids = self - .get_document_ids(account_id, Collection::Email) - .await? - .unwrap_or_default(); - if !document_ids.is_empty() && !access_token.is_member(account_id) { - document_ids &= self - .shared_messages(access_token, account_id, check_acls) - .await?; - } - Ok(document_ids) - } - - async fn has_access_to_document( - &self, - access_token: &AccessToken, - to_account_id: u32, - to_collection: impl Into, - to_document_id: u32, - check_acls: impl Into>, - ) -> trc::Result { - let to_collection = to_collection.into(); - let check_acls = check_acls.into(); - for &grant_account_id in [access_token.primary_id] - .iter() - .chain(access_token.member_of.clone().iter()) - { - match self - .core - .storage - .data - .get_value::(ValueKey { - account_id: to_account_id, - collection: to_collection, - document_id: to_document_id, - class: ValueClass::Acl(grant_account_id), - }) - .await - { - Ok(Some(acls)) => { - let mut acls = Bitmap::::from(acls); - - acls.intersection(&check_acls); - if !acls.is_empty() { - return Ok(true); - } - } - Ok(None) => (), - Err(err) => { - return Err(err.caused_by(trc::location!())); - } - } - } - Ok(false) - } - - async fn acl_set( +impl Server { + pub async fn acl_set( &self, changes: &mut Vec, current: Option<&[AclGrant]>, @@ -304,15 +78,16 @@ impl AclMethods for Server { Ok(()) } - async fn acl_get( + pub async fn acl_get( &self, - value: &[AclGrant], + value: &[ArchivedAclGrant], access_token: &AccessToken, account_id: u32, ) -> Value { if access_token.is_member(account_id) || value.iter().any(|item| { - access_token.is_member(item.account_id) && item.grants.contains(Acl::Administer) + access_token.is_member(item.account_id.into()) + && Bitmap::from(&item.grants).contains(Acl::Administer) }) { let mut acl_obj = jmap_proto::types::value::Object::with_capacity(value.len() / 2); @@ -321,13 +96,13 @@ impl AclMethods for Server { .core .storage .directory - .query(QueryBy::Id(item.account_id), false) + .query(QueryBy::Id(item.account_id.into()), false) .await .unwrap_or_default() { acl_obj.append( Property::_T(principal.take_str(PrincipalField::Name).unwrap_or_default()), - item.grants + Bitmap::from(&item.grants) .map(|acl_item| Value::Text(acl_item.to_string())) .collect::>(), ); @@ -340,7 +115,7 @@ impl AclMethods for Server { } } - async fn refresh_acls(&self, acl_changes: &[AclGrant], current: Option<&[AclGrant]>) { + pub async fn refresh_acls(&self, acl_changes: &[AclGrant], current: Option<&[AclGrant]>) { let mut changed_principals = ChangedPrincipals::new(); if let Some(acl_current) = current { for current_item in acl_current { @@ -389,7 +164,7 @@ impl AclMethods for Server { self.increment_token_revision(changed_principals).await; } - async fn map_acl_set(&self, acl_set: Vec) -> Result, SetError> { + pub async fn map_acl_set(&self, acl_set: Vec) -> Result, SetError> { let mut acls = Vec::with_capacity(acl_set.len() / 2); for item in acl_set.chunks_exact(2) { if let (Value::Text(account_name), Value::UnsignedInt(grants)) = (&item[0], &item[1]) { @@ -427,7 +202,7 @@ impl AclMethods for Server { Ok(acls) } - async fn map_acl_patch( + pub async fn map_acl_patch( &self, acl_patch: Vec, ) -> Result<(AclGrant, Option), SetError> { @@ -462,20 +237,3 @@ impl AclMethods for Server { } } } - -pub trait EffectiveAcl { - fn effective_acl(&self, access_token: &AccessToken) -> Bitmap; -} - -impl EffectiveAcl for Vec { - fn effective_acl(&self, access_token: &AccessToken) -> Bitmap { - let mut acl = Bitmap::::new(); - for item in self { - if access_token.is_member(item.account_id) { - acl.union(&item.grants); - } - } - - acl - } -} diff --git a/crates/common/src/sharing/document.rs b/crates/common/src/sharing/document.rs new file mode 100644 index 00000000..dd13df76 --- /dev/null +++ b/crates/common/src/sharing/document.rs @@ -0,0 +1,169 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use jmap_proto::types::{acl::Acl, collection::Collection}; +use store::{ValueKey, query::acl::AclQuery, roaring::RoaringBitmap, write::ValueClass}; +use trc::AddContext; +use utils::map::bitmap::Bitmap; + +use crate::{Server, auth::AccessToken}; + +impl Server { + pub async fn shared_documents( + &self, + access_token: &AccessToken, + to_account_id: u32, + to_collection: Collection, + check_acls: impl Into>, + ) -> trc::Result { + let check_acls = check_acls.into(); + let mut document_ids = RoaringBitmap::new(); + let to_collection = u8::from(to_collection); + for &grant_account_id in [access_token.primary_id] + .iter() + .chain(access_token.member_of.clone().iter()) + { + for acl_item in self + .core + .storage + .data + .acl_query(AclQuery::SharedWith { + grant_account_id, + to_account_id, + to_collection, + }) + .await + .caused_by(trc::location!())? + { + let mut acls = Bitmap::::from(acl_item.permissions); + + acls.intersection(&check_acls); + if !acls.is_empty() { + document_ids.insert(acl_item.to_document_id); + } + } + } + + Ok(document_ids) + } + + pub async fn shared_document_children( + &self, + access_token: &AccessToken, + to_account_id: u32, + to_collection: Collection, + check_acls: impl Into>, + ) -> trc::Result { + let check_acls = check_acls.into(); + let shared_documents = self + .shared_documents(access_token, to_account_id, to_collection, check_acls) + .await?; + if shared_documents.is_empty() { + return Ok(shared_documents); + } + let child_collection = to_collection.child_collection().unwrap(); + let child_property = child_collection.parent_property().unwrap(); + let mut shared_items = RoaringBitmap::new(); + for document_id in shared_documents { + if let Some(documents_in_folder) = self + .get_tag( + to_account_id, + child_collection, + child_property.clone(), + document_id, + ) + .await? + { + shared_items |= documents_in_folder; + } + } + + Ok(shared_items) + } + + pub async fn owned_or_shared_documents( + &self, + access_token: &AccessToken, + account_id: u32, + collection: Collection, + check_acls: impl Into>, + ) -> trc::Result { + let check_acls = check_acls.into(); + let mut document_ids = self + .get_document_ids(account_id, collection) + .await? + .unwrap_or_default(); + if !document_ids.is_empty() && !access_token.is_member(account_id) { + document_ids &= self + .shared_documents(access_token, account_id, collection, check_acls) + .await?; + } + Ok(document_ids) + } + + pub async fn owned_or_shared_document_children( + &self, + access_token: &AccessToken, + account_id: u32, + collection: Collection, + check_acls: impl Into>, + ) -> trc::Result { + let check_acls = check_acls.into(); + let mut document_ids = self + .get_document_ids(account_id, collection) + .await? + .unwrap_or_default(); + if !document_ids.is_empty() && !access_token.is_member(account_id) { + document_ids &= self + .shared_document_children(access_token, account_id, collection, check_acls) + .await?; + } + Ok(document_ids) + } + + pub async fn has_access_to_document( + &self, + access_token: &AccessToken, + to_account_id: u32, + to_collection: impl Into, + to_document_id: u32, + check_acls: impl Into>, + ) -> trc::Result { + let to_collection = to_collection.into(); + let check_acls = check_acls.into(); + for &grant_account_id in [access_token.primary_id] + .iter() + .chain(access_token.member_of.clone().iter()) + { + match self + .core + .storage + .data + .get_value::(ValueKey { + account_id: to_account_id, + collection: to_collection, + document_id: to_document_id, + class: ValueClass::Acl(grant_account_id), + }) + .await + { + Ok(Some(acls)) => { + let mut acls = Bitmap::::from(acls); + + acls.intersection(&check_acls); + if !acls.is_empty() { + return Ok(true); + } + } + Ok(None) => (), + Err(err) => { + return Err(err.caused_by(trc::location!())); + } + } + } + Ok(false) + } +} diff --git a/crates/common/src/sharing/mod.rs b/crates/common/src/sharing/mod.rs new file mode 100644 index 00000000..478ff1aa --- /dev/null +++ b/crates/common/src/sharing/mod.rs @@ -0,0 +1,47 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use jmap_proto::types::{ + acl::Acl, + value::{AclGrant, ArchivedAclGrant}, +}; +use rkyv::vec::ArchivedVec; +use utils::map::bitmap::Bitmap; + +use crate::auth::AccessToken; + +pub mod acl; +pub mod document; + +pub trait EffectiveAcl { + fn effective_acl(&self, access_token: &AccessToken) -> Bitmap; +} + +impl EffectiveAcl for Vec { + fn effective_acl(&self, access_token: &AccessToken) -> Bitmap { + let mut acl = Bitmap::::new(); + for item in self { + if access_token.is_member(item.account_id) { + acl.union(&item.grants); + } + } + + acl + } +} + +impl EffectiveAcl for ArchivedVec { + fn effective_acl(&self, access_token: &AccessToken) -> Bitmap { + let mut acl = Bitmap::::new(); + for item in self.iter() { + if access_token.is_member(item.account_id.into()) { + acl.union_raw(item.grants.bitmap); + } + } + + acl + } +} diff --git a/crates/common/src/storage/blob.rs b/crates/common/src/storage/blob.rs new file mode 100644 index 00000000..ec534c04 --- /dev/null +++ b/crates/common/src/storage/blob.rs @@ -0,0 +1,35 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use jmap_proto::types::blob::BlobSection; +use mail_parser::{ + Encoding, + decoders::{base64::base64_decode, quoted_printable::quoted_printable_decode}, +}; +use utils::BlobHash; + +use crate::Server; + +impl Server { + pub async fn get_blob_section( + &self, + hash: &BlobHash, + section: &BlobSection, + ) -> trc::Result>> { + Ok(self + .blob_store() + .get_blob( + hash.as_slice(), + (section.offset_start)..(section.offset_start.saturating_add(section.size)), + ) + .await? + .and_then(|bytes| match Encoding::from(section.encoding) { + Encoding::None => Some(bytes), + Encoding::Base64 => base64_decode(&bytes), + Encoding::QuotedPrintable => quoted_printable_decode(&bytes), + })) + } +} diff --git a/crates/jmap-proto/src/object/index.rs b/crates/common/src/storage/index.rs similarity index 75% rename from crates/jmap-proto/src/object/index.rs rename to crates/common/src/storage/index.rs index bc74bf31..1b771453 100644 --- a/crates/jmap-proto/src/object/index.rs +++ b/crates/common/src/storage/index.rs @@ -4,50 +4,28 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{collections::HashSet, fmt::Debug}; +use jmap_proto::types::{property::Property, value::AclGrant}; +use std::{borrow::Cow, collections::HashSet, fmt::Debug}; use store::{ - Deserialize, Serialize, + Serialize, SerializeInfallible, write::{ - BatchBuilder, BitmapClass, BitmapHash, DirectoryClass, IntoOperations, Operation, - TokenizeText, ValueClass, ValueOp, assert::HashedValue, + BatchBuilder, BitmapClass, DirectoryClass, IntoOperations, Operation, ValueOp, + assert::HashedValue, }, }; -use crate::types::{property::Property, value::AclGrant}; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum IndexValue<'x> { - Text { - field: u8, - value: &'x str, - tokenize: bool, - index: bool, - }, - U32 { - field: u8, - value: Option, - }, - U64 { - field: u8, - value: Option, - }, - U32List { - field: u8, - value: &'x [u32], - }, - Tag { - field: u8, - is_set: bool, - }, - Quota { - used: u32, - }, - Acl { - value: &'x [AclGrant], - }, + Text { field: u8, value: Cow<'x, str> }, + U32 { field: u8, value: Option }, + U64 { field: u8, value: Option }, + U32List { field: u8, value: &'x [u32] }, + Tag { field: u8, is_set: bool }, + Quota { used: u32 }, + Acl { value: &'x [AclGrant] }, } -pub trait IndexableObject: Debug + Serialize + Deserialize + Eq { +pub trait IndexableObject: Debug + Eq + Serialize + Sync + Send { fn index_values(&self) -> impl Iterator>; } @@ -111,17 +89,17 @@ impl ObjectIndexBuilder { } impl IntoOperations for ObjectIndexBuilder { - fn build(self, batch: &mut BatchBuilder) { + fn build(self, batch: &mut BatchBuilder) -> trc::Result<()> { match (self.current, self.changes) { (None, Some(changes)) => { // Insertion build_batch(batch, &changes, self.tenant_id, true); - batch.set(Property::Value, changes.serialize()); + batch.set(Property::Value, changes.serialize()?); } (Some(current), Some(changes)) => { // Update batch.assert_value(Property::Value, ¤t); - merge_batch(batch, current.inner, changes, self.tenant_id); + merge_batch(batch, current.inner, changes, self.tenant_id)?; } (Some(current), None) => { // Deletion @@ -131,6 +109,8 @@ impl IntoOperations for ObjectIndexBuilder { } (None, None) => unreachable!(), } + + Ok(()) } } @@ -142,31 +122,13 @@ fn build_batch( ) { for item in object.index_values() { match item { - IndexValue::Text { - field, - value, - tokenize, - index, - } => { + IndexValue::Text { field, value } => { if !value.is_empty() { - if index { - batch.ops.push(Operation::Index { - field, - key: value.serialize(), - set, - }); - } - if tokenize { - for token in value.to_tokens() { - batch.ops.push(Operation::Bitmap { - class: BitmapClass::Text { - field, - token: BitmapHash::new(token), - }, - set, - }); - } - } + batch.ops.push(Operation::Index { + field, + key: value.as_ref().serialize(), + set, + }); } } IndexValue::U32 { field, value } => { @@ -191,7 +153,7 @@ fn build_batch( for item in value { batch.ops.push(Operation::Index { field, - key: item.serialize(), + key: (*item).serialize(), set, }); } @@ -239,7 +201,7 @@ fn merge_batch( current: T, changes: T, tenant_id: Option, -) { +) -> trc::Result<()> { let mut has_changes = current != changes; for (current, change) in current.index_values().zip(changes.index_values()) { @@ -253,59 +215,25 @@ fn merge_batch( IndexValue::Text { field, value: old_value, - tokenize, - index, }, IndexValue::Text { value: new_value, .. }, ) => { - // Remove current text from index - let mut add_tokens = HashSet::new(); - let mut remove_tokens = HashSet::new(); - if !old_value.is_empty() { - if index { - batch.ops.push(Operation::Index { - field, - key: old_value.serialize(), - set: false, - }); - } - if tokenize { - old_value.tokenize_into(&mut remove_tokens); - } + batch.ops.push(Operation::Index { + field, + key: old_value.as_ref().serialize(), + set: false, + }); } - // Add new text to index if !new_value.is_empty() { - if index { - batch.ops.push(Operation::Index { - field, - key: new_value.serialize(), - set: true, - }); - } - if tokenize { - for token in new_value.to_tokens() { - if !remove_tokens.remove(&token) { - add_tokens.insert(token); - } - } - } - } - - // Update tokens - for (token, set) in [(add_tokens, true), (remove_tokens, false)] { - for token in token { - batch.ops.push(Operation::Bitmap { - class: BitmapClass::Text { - field, - token: BitmapHash::new(token), - }, - set, - }); - } + batch.ops.push(Operation::Index { + field, + key: new_value.as_ref().serialize(), + set: true, + }); } } ( @@ -482,13 +410,9 @@ fn merge_batch( if has_changes { batch.ops.push(Operation::Value { class: Property::Value.into(), - op: ValueOp::Set(current.serialize().into()), + op: ValueOp::Set(current.serialize()?.into()), }); } -} -impl From for ValueClass { - fn from(value: Property) -> Self { - ValueClass::Property(value.into()) - } + Ok(()) } diff --git a/crates/common/src/storage/mod.rs b/crates/common/src/storage/mod.rs new file mode 100644 index 00000000..fb47b2f1 --- /dev/null +++ b/crates/common/src/storage/mod.rs @@ -0,0 +1,10 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod blob; +pub mod index; +pub mod state; +pub mod tag; diff --git a/crates/common/src/storage/state.rs b/crates/common/src/storage/state.rs new file mode 100644 index 00000000..ecaef1a7 --- /dev/null +++ b/crates/common/src/storage/state.rs @@ -0,0 +1,39 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use jmap_proto::types::{state::StateChange, type_state::DataType}; +use tokio::sync::mpsc; +use utils::map::bitmap::Bitmap; + +use crate::{IPC_CHANNEL_BUFFER, Server, ipc::StateEvent}; + +impl Server { + pub async fn subscribe_state_manager( + &self, + account_id: u32, + types: Bitmap, + ) -> trc::Result> { + let (change_tx, change_rx) = mpsc::channel::(IPC_CHANNEL_BUFFER); + let state_tx = self.inner.ipc.state_tx.clone(); + + for event in [ + StateEvent::UpdateSharedAccounts { account_id }, + StateEvent::Subscribe { + account_id, + types, + tx: change_tx, + }, + ] { + state_tx.send(event).await.map_err(|err| { + trc::EventType::Server(trc::ServerEvent::ThreadError) + .reason(err) + .caused_by(trc::location!()) + })?; + } + + Ok(change_rx) + } +} diff --git a/crates/common/src/storage/tag.rs b/crates/common/src/storage/tag.rs new file mode 100644 index 00000000..b5911eaf --- /dev/null +++ b/crates/common/src/storage/tag.rs @@ -0,0 +1,143 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::slice::IterMut; + +use jmap_proto::types::property::Property; +use store::{ + Serialize, + write::{ + BatchBuilder, DeserializeFrom, MaybeDynamicId, SerializeInto, TagValue, ValueClass, + assert::HashedValue, + }, +}; + +pub struct TagManager< + T: Into> + + PartialEq + + Clone + + SerializeInto + + Serialize + + DeserializeFrom + + Sync + + Send, +> { + current: HashedValue>, + added: Vec, + removed: Vec, + last: LastTag, +} + +enum LastTag { + Set, + Update, + None, +} + +impl< + T: Into> + + PartialEq + + Clone + + SerializeInto + + Serialize + + DeserializeFrom + + Sync + + Send, +> TagManager +{ + pub fn new(current: HashedValue>) -> Self { + Self { + current, + added: Vec::new(), + removed: Vec::new(), + last: LastTag::None, + } + } + + pub fn set(&mut self, tags: Vec) { + if matches!(self.last, LastTag::None) { + self.added.clear(); + self.removed.clear(); + + for tag in &tags { + if !self.current.inner.contains(tag) { + self.added.push(tag.clone()); + } + } + + for tag in &self.current.inner { + if !tags.contains(tag) { + self.removed.push(tag.clone()); + } + } + + self.current.inner = tags; + self.last = LastTag::Set; + } + } + + pub fn update(&mut self, tag: T, add: bool) { + if matches!(self.last, LastTag::None | LastTag::Update) { + if add { + if !self.current.inner.contains(&tag) { + self.added.push(tag.clone()); + self.current.inner.push(tag); + } + } else if let Some(index) = self.current.inner.iter().position(|t| t == &tag) { + self.current.inner.swap_remove(index); + self.removed.push(tag); + } + self.last = LastTag::Update; + } + } + + pub fn added(&self) -> &[T] { + &self.added + } + + pub fn removed(&self) -> &[T] { + &self.removed + } + + pub fn current(&self) -> &[T] { + &self.current.inner + } + + pub fn changed_tags(&self) -> impl Iterator { + self.added.iter().chain(self.removed.iter()) + } + + pub fn inner_tags_mut(&mut self) -> IterMut<'_, T> { + self.current.inner.iter_mut() + } + + pub fn has_tags(&self) -> bool { + !self.current.inner.is_empty() + } + + pub fn has_changes(&self) -> bool { + !self.added.is_empty() || !self.removed.is_empty() + } + + pub fn update_batch(self, batch: &mut BatchBuilder, property: Property) -> trc::Result<()> { + let property = u8::from(property); + + batch + .assert_value(ValueClass::Property(property), &self.current) + .set( + ValueClass::Property(property), + self.current.inner.serialize()?, + ); + for added in self.added { + batch.tag(property, added); + } + for removed in self.removed { + batch.untag(property, removed); + } + + Ok(()) + } +} diff --git a/crates/directory/src/backend/internal/manage.rs b/crates/directory/src/backend/internal/manage.rs index f6c40a94..530d3559 100644 --- a/crates/directory/src/backend/internal/manage.rs +++ b/crates/directory/src/backend/internal/manage.rs @@ -456,7 +456,7 @@ impl ManageDirectory for Store { ) .set( ValueClass::Directory(DirectoryClass::Principal(MaybeDynamicId::Dynamic(0))), - (&principal).serialize(), + principal.serialize().caused_by(trc::location!())?, ) .set( ValueClass::Directory(DirectoryClass::NameToId( @@ -815,8 +815,12 @@ impl ManageDirectory for Store { // Prepare changes let mut batch = BatchBuilder::new(); let mut pinfo_name = - PrincipalInfo::new(principal_id, principal_type, principal.inner.tenant()).serialize(); - let pinfo_email = PrincipalInfo::new(principal_id, principal_type, None).serialize(); + PrincipalInfo::new(principal_id, principal_type, principal.inner.tenant()) + .serialize() + .caused_by(trc::location!())?; + let pinfo_email = PrincipalInfo::new(principal_id, principal_type, None) + .serialize() + .caused_by(trc::location!())?; let update_principal = !changes.is_empty() && !changes.iter().all(|c| { matches!( @@ -981,7 +985,8 @@ impl ManageDirectory for Store { principal.inner.set(PrincipalField::Tenant, tenant_info.id); pinfo_name = PrincipalInfo::new(principal_id, principal_type, tenant_info.id.into()) - .serialize(); + .serialize() + .caused_by(trc::location!())?; } else if let Some(tenant_id) = principal.inner.tenant() { // Update quota if let Some(used_quota) = used_quota { @@ -992,8 +997,9 @@ impl ManageDirectory for Store { changed_principals.add_change(principal_id, principal_type, change.field); principal.inner.remove(PrincipalField::Tenant); - pinfo_name = - PrincipalInfo::new(principal_id, principal_type, None).serialize(); + pinfo_name = PrincipalInfo::new(principal_id, principal_type, None) + .serialize() + .caused_by(trc::location!())?; } else { continue; } @@ -1722,7 +1728,7 @@ impl ManageDirectory for Store { ValueClass::Directory(DirectoryClass::Principal(MaybeDynamicId::Static( principal_id, ))), - principal.inner.serialize(), + principal.inner.serialize().caused_by(trc::location!())?, ); } @@ -2147,7 +2153,7 @@ impl SerializeWithId for Principal { fn serialize_with_id(&self, ids: &AssignedIds) -> trc::Result> { let mut principal = self.clone(); principal.id = ids.last_document_id().caused_by(trc::location!())?; - Ok(principal.serialize()) + principal.serialize() } } @@ -2383,8 +2389,9 @@ impl ChangedPrincipal { impl SerializeWithId for DynamicPrincipalInfo { fn serialize_with_id(&self, ids: &AssignedIds) -> trc::Result> { - ids.last_document_id() - .map(|principal_id| PrincipalInfo::new(principal_id, self.typ, self.tenant).serialize()) + ids.last_document_id().and_then(|principal_id| { + PrincipalInfo::new(principal_id, self.typ, self.tenant).serialize() + }) } } diff --git a/crates/directory/src/backend/internal/mod.rs b/crates/directory/src/backend/internal/mod.rs index 474e439d..8c74e782 100644 --- a/crates/directory/src/backend/internal/mod.rs +++ b/crates/directory/src/backend/internal/mod.rs @@ -32,13 +32,7 @@ pub struct PrincipalInfo { } impl Serialize for Principal { - fn serialize(self) -> Vec { - (&self).serialize() - } -} - -impl Serialize for &Principal { - fn serialize(self) -> Vec { + fn serialize(&self) -> trc::Result> { let mut serializer = KeySerializer::new( U32_LEN + 2 @@ -84,7 +78,7 @@ impl Serialize for &Principal { } } - serializer.finalize() + Ok(serializer.finalize()) } } @@ -122,8 +116,8 @@ impl PrincipalInfo { } impl Serialize for PrincipalInfo { - fn serialize(self) -> Vec { - if let Some(tenant) = self.tenant { + fn serialize(&self) -> trc::Result> { + Ok(if let Some(tenant) = self.tenant { KeySerializer::new((U32_LEN * 2) + 1) .write_leb128(self.id) .write(self.typ as u8) @@ -134,7 +128,7 @@ impl Serialize for PrincipalInfo { .write_leb128(self.id) .write(self.typ as u8) .finalize() - } + }) } } @@ -314,7 +308,7 @@ impl MigrateDirectory for Store { ValueClass::Directory(DirectoryClass::Principal(MaybeDynamicId::Static( account_id, ))), - (&principal).serialize(), + principal.serialize().caused_by(trc::location!())?, ); if principal.typ() == Type::Individual { @@ -359,7 +353,8 @@ impl MigrateDirectory for Store { Principal::new(0, Type::Domain) .with_field(PrincipalField::Name, domain.to_string()) .with_field(PrincipalField::Description, domain.to_string()) - .serialize(), + .serialize() + .caused_by(trc::location!())?, ) .set( ValueClass::Directory(DirectoryClass::NameToId(domain.as_bytes().to_vec())), diff --git a/crates/email/Cargo.toml b/crates/email/Cargo.toml index e44eef71..d899afa3 100644 --- a/crates/email/Cargo.toml +++ b/crates/email/Cargo.toml @@ -32,6 +32,7 @@ rsa = "0.9.2" rand = "0.8" sequoia-openpgp = { version = "1.16", default-features = false, features = ["crypto-rust", "allow-experimental-crypto", "allow-variable-time-crypto"] } hashify = "0.2" +rkyv = { version = "0.8.10", features = ["little_endian"] } [features] test_mode = [] diff --git a/crates/email/src/identity/mod.rs b/crates/email/src/identity/mod.rs index 5bd0e735..a4b81807 100644 --- a/crates/email/src/identity/mod.rs +++ b/crates/email/src/identity/mod.rs @@ -4,9 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -pub mod serialize; +use store::Serialize; -#[derive(Debug, Default, Clone, PartialEq, Eq)] +#[derive( + rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, +)] pub struct Identity { pub name: String, pub email: String, @@ -16,8 +18,16 @@ pub struct Identity { pub html_signature: String, } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] pub struct EmailAddress { pub name: Option, pub email: String, } + +impl Serialize for Identity { + fn serialize(&self) -> trc::Result> { + rkyv::to_bytes::(self) + .map(|r| r.into_vec()) + .map_err(Into::into) + } +} diff --git a/crates/email/src/identity/serialize.rs b/crates/email/src/identity/serialize.rs deleted file mode 100644 index 369d44ab..00000000 --- a/crates/email/src/identity/serialize.rs +++ /dev/null @@ -1,23 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use store::{Deserialize, Serialize}; - -use super::Identity; - -impl Serialize for Identity { - fn serialize(self) -> Vec { - let todo = 1; - todo!() - } -} - -impl Deserialize for Identity { - fn deserialize(bytes: &[u8]) -> trc::Result { - let todo = 1; - todo!() - } -} diff --git a/crates/email/src/mailbox/destroy.rs b/crates/email/src/mailbox/destroy.rs new file mode 100644 index 00000000..d34d4dce --- /dev/null +++ b/crates/email/src/mailbox/destroy.rs @@ -0,0 +1,239 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use common::{ + Server, auth::AccessToken, sharing::EffectiveAcl, storage::index::ObjectIndexBuilder, +}; +use directory::Permission; +use jmap_proto::{ + error::set::{SetError, SetErrorType}, + types::{acl::Acl, collection::Collection, id::Id, property::Property}, +}; +use store::{ + Serialize, SerializeInfallible, + query::Filter, + roaring::RoaringBitmap, + write::{ArchivedValue, BatchBuilder, assert::HashedValue, log::ChangeLogBuilder}, +}; +use trc::AddContext; + +use crate::message::delete::EmailDeletion; + +use super::*; + +pub trait MailboxDestroy: Sync + Send { + fn mailbox_destroy( + &self, + account_id: u32, + document_id: u32, + changes: &mut ChangeLogBuilder, + access_token: &AccessToken, + remove_emails: bool, + ) -> impl Future>> + Send; +} + +impl MailboxDestroy for Server { + async fn mailbox_destroy( + &self, + account_id: u32, + document_id: u32, + changes: &mut ChangeLogBuilder, + access_token: &AccessToken, + remove_emails: bool, + ) -> trc::Result> { + // Internal folders cannot be deleted + #[cfg(feature = "test_mode")] + if [INBOX_ID, TRASH_ID].contains(&document_id) + && !access_token.has_permission(Permission::DeleteSystemFolders) + { + return Ok(Err(SetError::forbidden().with_description( + "You are not allowed to delete Inbox, Junk or Trash folders.", + ))); + } + + #[cfg(not(feature = "test_mode"))] + if [INBOX_ID, TRASH_ID, JUNK_ID].contains(&document_id) + && !access_token.has_permission(Permission::DeleteSystemFolders) + { + return Ok(Err(SetError::forbidden().with_description( + "You are not allowed to delete Inbox, Junk or Trash folders.", + ))); + } + + // Verify that this mailbox does not have sub-mailboxes + if !self + .store() + .filter( + account_id, + Collection::Mailbox, + vec![Filter::eq( + Property::ParentId, + (document_id + 1).serialize(), + )], + ) + .await? + .results + .is_empty() + { + return Ok(Err(SetError::new(SetErrorType::MailboxHasChild) + .with_description("Mailbox has at least one children."))); + } + + // Verify that the mailbox is empty + let mut did_remove_emails = false; + if let Some(message_ids) = self + .get_tag( + account_id, + Collection::Email, + Property::MailboxIds, + document_id, + ) + .await? + { + if remove_emails { + // Flag removal for state change notification + did_remove_emails = true; + + // If the message is in multiple mailboxes, untag it from the current mailbox, + // otherwise delete it. + let mut destroy_ids = RoaringBitmap::new(); + for (message_id, mut mailbox_ids) in self + .get_properties::>, _, _>( + account_id, + Collection::Email, + &message_ids, + Property::MailboxIds, + ) + .await? + { + // Remove mailbox from list + let orig_len = mailbox_ids.inner.len(); + mailbox_ids.inner.retain(|id| id.mailbox_id != document_id); + if mailbox_ids.inner.len() == orig_len { + continue; + } + + if !mailbox_ids.inner.is_empty() { + // Obtain threadId + if let Some(thread_id) = self + .get_property::( + account_id, + Collection::Email, + message_id, + Property::ThreadId, + ) + .await? + { + // Untag message from mailbox + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .with_collection(Collection::Email) + .update_document(message_id) + .assert_value(Property::MailboxIds, &mailbox_ids) + .set( + Property::MailboxIds, + mailbox_ids.inner.serialize().caused_by(trc::location!())?, + ) + .untag(Property::MailboxIds, document_id); + match self.core.storage.data.write(batch.build()).await { + Ok(_) => changes.log_update( + Collection::Email, + Id::from_parts(thread_id, message_id), + ), + Err(err) if err.is_assertion_failure() => { + return Ok(Err(SetError::forbidden().with_description( + concat!( + "Another process modified a message in this mailbox ", + "while deleting it, please try again." + ), + ))); + } + Err(err) => { + return Err(err.caused_by(trc::location!())); + } + } + } else { + trc::event!( + Store(trc::StoreEvent::NotFound), + AccountId = account_id, + MessageId = message_id, + MailboxId = document_id, + Details = "Message does not have a threadId.", + CausedBy = trc::location!(), + ); + } + } else { + // Delete message + destroy_ids.insert(message_id); + } + } + + // Bulk delete messages + if !destroy_ids.is_empty() { + let (mut change, _) = self.emails_tombstone(account_id, destroy_ids).await?; + change.changes.remove(&(Collection::Mailbox as u8)); + changes.merge(change); + } + } else { + return Ok(Err(SetError::new(SetErrorType::MailboxHasEmail) + .with_description("Mailbox is not empty."))); + } + } + + // Obtain mailbox + if let Some(mailbox) = self + .get_property::>>( + account_id, + Collection::Mailbox, + document_id, + Property::Value, + ) + .await + .caused_by(trc::location!())? + { + let mailbox = mailbox.into_deserialized().caused_by(trc::location!())?; + // Validate ACLs + if access_token.is_shared(account_id) { + let acl = mailbox.inner.acls.effective_acl(access_token); + if !acl.contains(Acl::Administer) { + if !acl.contains(Acl::Delete) { + return Ok(Err(SetError::forbidden() + .with_description("You are not allowed to delete this mailbox."))); + } else if remove_emails && !acl.contains(Acl::RemoveItems) { + return Ok(Err(SetError::forbidden().with_description( + "You are not allowed to delete emails from this mailbox.", + ))); + } + } + } + + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .with_collection(Collection::Mailbox) + .delete_document(document_id) + .clear(Property::EmailIds) + .custom(ObjectIndexBuilder::new().with_current(mailbox)) + .caused_by(trc::location!())?; + + match self.core.storage.data.write(batch.build()).await { + Ok(_) => { + changes.log_delete(Collection::Mailbox, document_id); + Ok(Ok(did_remove_emails)) + } + Err(err) if err.is_assertion_failure() => Ok(Err(SetError::forbidden() + .with_description(concat!( + "Another process modified this mailbox ", + "while deleting it, please try again." + )))), + Err(err) => Err(err.caused_by(trc::location!())), + } + } else { + Ok(Err(SetError::not_found())) + } + } +} diff --git a/crates/email/src/mailbox/index.rs b/crates/email/src/mailbox/index.rs index a86610cd..6f9d7b7e 100644 --- a/crates/email/src/mailbox/index.rs +++ b/crates/email/src/mailbox/index.rs @@ -4,28 +4,25 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::config::jmap::settings::SpecialUse; -use jmap_proto::{ - object::index::{IndexValue, IndexableObject}, - types::property::Property, +use common::{ + config::jmap::settings::SpecialUse, + storage::index::{IndexValue, IndexableObject}, }; +use jmap_proto::types::property::Property; +use store::write::{MaybeDynamicId, TagValue}; -use super::Mailbox; +use super::{Mailbox, UidMailbox}; impl IndexableObject for Mailbox { fn index_values(&self) -> impl Iterator> { [ IndexValue::Text { field: Property::Name.into(), - value: self.name.as_str(), - tokenize: true, - index: true, + value: self.name.to_lowercase().into(), }, IndexValue::Text { field: Property::Role.into(), - value: self.role.as_str().unwrap_or_default(), - tokenize: false, - index: true, + value: self.role.as_str().unwrap_or_default().into(), }, IndexValue::Tag { field: Property::Role.into(), @@ -48,3 +45,15 @@ impl IndexableObject for Mailbox { .into_iter() } } + +impl From<&UidMailbox> for TagValue { + fn from(value: &UidMailbox) -> Self { + TagValue::Id(MaybeDynamicId::Static(value.mailbox_id)) + } +} + +impl From for TagValue { + fn from(value: UidMailbox) -> Self { + TagValue::Id(MaybeDynamicId::Static(value.mailbox_id)) + } +} diff --git a/crates/email/src/mailbox/manage.rs b/crates/email/src/mailbox/manage.rs index f4a90b52..4a46a6ad 100644 --- a/crates/email/src/mailbox/manage.rs +++ b/crates/email/src/mailbox/manage.rs @@ -6,12 +6,15 @@ use std::future::Future; -use common::{Server, config::jmap::settings::SpecialUse}; -use jmap_proto::{ - object::index::ObjectIndexBuilder, - types::{collection::Collection, keyword::Keyword, property::Property}, +use common::{Server, config::jmap::settings::SpecialUse, storage::index::ObjectIndexBuilder}; +use jmap_proto::types::{collection::Collection, keyword::Keyword, property::Property}; +use store::{ + SerializeInfallible, + ahash::AHashSet, + query::Filter, + roaring::RoaringBitmap, + write::{ArchivedValue, BatchBuilder}, }; -use store::{ahash::AHashSet, query::Filter, roaring::RoaringBitmap, write::BatchBuilder}; use trc::AddContext; use crate::thread::cache::ThreadCache; @@ -106,7 +109,8 @@ impl MailboxFnc for Server { } batch .create_document_with_id(document_id) - .custom(ObjectIndexBuilder::new().with_changes(object)); + .custom(ObjectIndexBuilder::new().with_changes(object)) + .caused_by(trc::location!())?; mailbox_ids.insert(document_id); } @@ -164,7 +168,8 @@ impl MailboxFnc for Server { .custom( ObjectIndexBuilder::new() .with_changes(Mailbox::new(name).with_parent_id(next_parent_id)), - ); + ) + .caused_by(trc::location!())?; let document_id = self .store() .write_expect_id(batch) @@ -179,7 +184,8 @@ impl MailboxFnc for Server { batch .with_account_id(account_id) .with_collection(Collection::Mailbox) - .custom(changes); + .custom(changes) + .caused_by(trc::location!())?; self.store() .write(batch.build()) .await @@ -275,7 +281,7 @@ impl MailboxFnc for Server { if pos == 0 && item.eq_ignore_ascii_case("inbox") { has_inbox = true; } else { - filter.push(Filter::eq(Property::Name, *item)); + filter.push(Filter::eq(Property::Name, item.serialize())); } } filter.push(Filter::End); @@ -299,7 +305,7 @@ impl MailboxFnc for Server { let mut found_names = Vec::new(); for document_id in document_ids { if let Some(obj) = self - .get_property::( + .get_property::>( account_id, Collection::Mailbox, document_id, @@ -307,7 +313,12 @@ impl MailboxFnc for Server { ) .await? { - found_names.push((obj.name, obj.parent_id, document_id + 1)); + let obj = obj.unarchive()?; + found_names.push(( + obj.name.to_string(), + u32::from(obj.parent_id), + document_id + 1, + )); } else { return Ok(None); } @@ -349,7 +360,7 @@ impl MailboxFnc for Server { .filter( account_id, Collection::Mailbox, - vec![Filter::eq(Property::Role, role.to_string())], + vec![Filter::eq(Property::Role, role.serialize())], ) .await .caused_by(trc::location!()) diff --git a/crates/email/src/mailbox/mod.rs b/crates/email/src/mailbox/mod.rs index 4aced5b0..42d87671 100644 --- a/crates/email/src/mailbox/mod.rs +++ b/crates/email/src/mailbox/mod.rs @@ -7,6 +7,7 @@ use common::config::jmap::settings::SpecialUse; use jmap_proto::types::value::AclGrant; +pub mod destroy; pub mod index; pub mod manage; pub mod serialize; @@ -19,7 +20,7 @@ pub const SENT_ID: u32 = 4; pub const ARCHIVE_ID: u32 = 5; pub const TOMBSTONE_ID: u32 = u32::MAX - 1; -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] pub struct Mailbox { pub name: String, pub role: SpecialUse, @@ -93,6 +94,12 @@ impl Mailbox { } } +impl ArchivedMailbox { + pub fn is_subscribed(&self, subscriber: u32) -> bool { + self.subscribers.iter().any(|x| u32::from(x) == subscriber) + } +} + impl PartialEq for UidMailbox { fn eq(&self, other: &Self) -> bool { self.mailbox_id == other.mailbox_id diff --git a/crates/email/src/mailbox/serialize.rs b/crates/email/src/mailbox/serialize.rs index 01bfe64c..2496f02f 100644 --- a/crates/email/src/mailbox/serialize.rs +++ b/crates/email/src/mailbox/serialize.rs @@ -4,44 +4,16 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ - use std::slice::Iter; +use std::slice::Iter; use store::{ - Deserialize, Serialize, U32_LEN, - write::{ - BitmapClass, DeserializeFrom, MaybeDynamicId, Operation, SerializeInto, TagValue, ToBitmaps, - }, + Serialize, U32_LEN, + write::{DeserializeFrom, SerializeInto}, }; use utils::codec::leb128::{Leb128Iterator, Leb128Vec}; use super::{Mailbox, UidMailbox}; -impl Serialize for Mailbox { - fn serialize(self) -> Vec { - let todo = 1; - todo!() - } -} - -impl Deserialize for Mailbox { - fn deserialize(bytes: &[u8]) -> trc::Result { - let todo = 1; - todo!() - } -} - -impl ToBitmaps for UidMailbox { - fn to_bitmaps(&self, ops: &mut Vec, field: u8, set: bool) { - ops.push(Operation::Bitmap { - class: BitmapClass::Tag { - field, - value: TagValue::Id(MaybeDynamicId::Static(self.mailbox_id)), - }, - set, - }); - } -} - impl SerializeInto for UidMailbox { fn serialize_into(&self, buf: &mut Vec) { buf.push_leb128(self.mailbox_id); @@ -59,9 +31,17 @@ impl DeserializeFrom for UidMailbox { } impl Serialize for UidMailbox { - fn serialize(self) -> Vec { + fn serialize(&self) -> trc::Result> { let mut buf = Vec::with_capacity(U32_LEN * 2); self.serialize_into(&mut buf); - buf + Ok(buf) + } +} + +impl Serialize for Mailbox { + fn serialize(&self) -> trc::Result> { + rkyv::to_bytes::(self) + .map(|r| r.into_vec()) + .map_err(Into::into) } } diff --git a/crates/email/src/message/copy.rs b/crates/email/src/message/copy.rs new file mode 100644 index 00000000..77d59be9 --- /dev/null +++ b/crates/email/src/message/copy.rs @@ -0,0 +1,235 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use common::{Server, auth::ResourceToken}; +use jmap_proto::{ + error::set::SetError, + types::{ + blob::BlobId, collection::Collection, date::UTCDate, id::Id, keyword::Keyword, + property::Property, + }, +}; +use mail_parser::{HeaderName, HeaderValue, parsers::fields::thread::thread_name}; +use store::{ + BlobClass, Serialize, SerializeInfallible, + write::{ + BatchBuilder, Bincode, MaybeDynamicId, TagValue, TaskQueueClass, ValueClass, + log::{Changes, LogInsert}, + }, +}; +use trc::AddContext; + +use crate::mailbox::UidMailbox; + +use super::{ + index::{EmailIndexBuilder, MAX_ID_LENGTH, MAX_SORT_FIELD_LENGTH, TrimTextValue, VisitValues}, + ingest::{EmailIngest, IngestedEmail, LogEmailInsert}, + metadata::MessageMetadata, +}; + +pub trait EmailCopy: Sync + Send { + #[allow(clippy::too_many_arguments)] + fn copy_message( + &self, + from_account_id: u32, + from_message_id: u32, + resource_token: &ResourceToken, + mailboxes: Vec, + keywords: Vec, + received_at: Option, + session_id: u64, + ) -> impl Future>> + Send; +} + +impl EmailCopy for Server { + #[allow(clippy::too_many_arguments)] + async fn copy_message( + &self, + from_account_id: u32, + from_message_id: u32, + resource_token: &ResourceToken, + mailboxes: Vec, + keywords: Vec, + received_at: Option, + session_id: u64, + ) -> trc::Result> { + // Obtain metadata + let account_id = resource_token.account_id; + let mut metadata = if let Some(metadata) = self + .get_property::>( + from_account_id, + Collection::Email, + from_message_id, + Property::BodyStructure, + ) + .await? + { + metadata.inner + } else { + return Ok(Err(SetError::not_found().with_description(format!( + "Message not found not found in account {}.", + Id::from(from_account_id) + )))); + }; + + // Check quota + match self + .has_available_quota(resource_token, metadata.size as u64) + .await + { + Ok(_) => (), + Err(err) => { + if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) + || err.matches(trc::EventType::Limit(trc::LimitEvent::TenantQuota)) + { + trc::error!(err.account_id(account_id).span_id(session_id)); + return Ok(Err(SetError::over_quota())); + } else { + return Err(err); + } + } + } + + // Set receivedAt + if let Some(received_at) = received_at { + metadata.received_at = received_at.timestamp() as u64; + } + + // Obtain threadId + let mut references = Vec::with_capacity(5); + let mut subject = ""; + for header in &metadata.contents.parts[0].headers { + match &header.name { + HeaderName::MessageId + | HeaderName::InReplyTo + | HeaderName::References + | HeaderName::ResentMessageId => { + header.value.visit_text(|id| { + if !id.is_empty() && id.len() < MAX_ID_LENGTH { + references.push(id); + } + }); + } + HeaderName::Subject if subject.is_empty() => { + subject = thread_name(match &header.value { + HeaderValue::Text(text) => text.as_ref(), + HeaderValue::TextList(list) if !list.is_empty() => { + list.first().unwrap().as_ref() + } + _ => "", + }) + .trim_text(MAX_SORT_FIELD_LENGTH); + } + _ => (), + } + } + + let thread_id = if !references.is_empty() { + self.find_or_merge_thread(account_id, subject, &references) + .await + .caused_by(trc::location!())? + } else { + None + }; + + // Assign id + let mut email = IngestedEmail { + size: metadata.size, + ..Default::default() + }; + let blob_hash = metadata.blob_hash.clone(); + + // Assign IMAP UIDs + let mut mailbox_ids = Vec::with_capacity(mailboxes.len()); + email.imap_uids = Vec::with_capacity(mailboxes.len()); + for mailbox_id in &mailboxes { + let uid = self + .assign_imap_uid(account_id, *mailbox_id) + .await + .caused_by(trc::location!())?; + mailbox_ids.push(UidMailbox::new(*mailbox_id, uid)); + email.imap_uids.push(uid); + } + + // Prepare batch + let change_id = self.assign_change_id(account_id)?; + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .with_change_id(change_id) + .with_collection(Collection::Thread); + if let Some(thread_id) = thread_id { + batch.log(Changes::update([thread_id])); + } else { + batch.create_document().log(LogInsert()); + }; + + // Build batch + let maybe_thread_id = thread_id + .map(MaybeDynamicId::Static) + .unwrap_or(MaybeDynamicId::Dynamic(0)); + batch + .with_collection(Collection::Mailbox) + .log(Changes::child_update(mailboxes.iter().copied())) + .with_collection(Collection::Email) + .create_document() + .log(LogEmailInsert::new(thread_id)) + .set(Property::ThreadId, maybe_thread_id) + .tag(Property::ThreadId, TagValue::Id(maybe_thread_id)) + .set( + Property::MailboxIds, + mailbox_ids.serialize().caused_by(trc::location!())?, + ) + .set( + Property::Keywords, + keywords.serialize().caused_by(trc::location!())?, + ) + .tag_many(Property::MailboxIds, mailbox_ids.iter()) + .tag_many(Property::Keywords, keywords.into_iter()) + .set(Property::Cid, change_id.serialize()) + .set( + ValueClass::TaskQueue(TaskQueueClass::IndexEmail { + seq: self.generate_snowflake_id()?, + hash: metadata.blob_hash.clone(), + }), + vec![], + ); + EmailIndexBuilder::set(metadata) + .build(&mut batch, account_id, resource_token.tenant.map(|t| t.id)) + .caused_by(trc::location!())?; + + // Insert and obtain ids + let ids = self + .core + .storage + .data + .write(batch.build()) + .await + .caused_by(trc::location!())?; + let thread_id = match thread_id { + Some(thread_id) => thread_id, + None => ids.first_document_id().caused_by(trc::location!())?, + }; + let document_id = ids.last_document_id().caused_by(trc::location!())?; + + // Request FTS index + self.notify_task_queue(); + + // Update response + email.id = Id::from_parts(thread_id, document_id); + email.change_id = change_id; + email.blob_id = BlobId::new( + blob_hash, + BlobClass::Linked { + account_id, + collection: Collection::Email.into(), + document_id, + }, + ); + + Ok(Ok(email)) + } +} diff --git a/crates/email/src/message/crypto.rs b/crates/email/src/message/crypto.rs index 5c77c83c..761280df 100644 --- a/crates/email/src/message/crypto.rs +++ b/crates/email/src/message/crypto.rs @@ -26,10 +26,7 @@ use rasn_cms::{ }; use rsa::{Pkcs1v15Encrypt, RsaPublicKey, pkcs1::DecodeRsaPublicKey}; use sequoia_openpgp as openpgp; -use store::{ - Deserialize, Serialize, - write::{Bincode, ToBitmaps}, -}; +use store::{Deserialize, Serialize, write::Bincode}; const P: openpgp::policy::StandardPolicy<'static> = openpgp::policy::StandardPolicy::new(); @@ -615,13 +612,17 @@ fn try_parse_pem( Ok(method.map(|method| (method, certs))) } -impl Serialize for &EncryptionParams { - fn serialize(self) -> Vec { +impl Serialize for EncryptionParams { + fn serialize(&self) -> trc::Result> { let len = bincode::serialized_size(&self).unwrap_or_default(); let mut buf = Vec::with_capacity(len as usize + 1); buf.push(1); - let _ = bincode::serialize_into(&mut buf, &self); - buf + bincode::serialize_into(&mut buf, &self).map_err(|err| { + trc::EventType::Store(trc::StoreEvent::DeserializeError) + .reason(err) + .caused_by(trc::location!()) + })?; + Ok(buf) } } @@ -645,12 +646,6 @@ impl Deserialize for EncryptionParams { } } -impl ToBitmaps for &EncryptionParams { - fn to_bitmaps(&self, _: &mut Vec, _: u8, _: bool) { - unreachable!() - } -} - impl Display for EncryptionMethod { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { diff --git a/crates/email/src/message/delete.rs b/crates/email/src/message/delete.rs index c981f3ea..ebdc9788 100644 --- a/crates/email/src/message/delete.rs +++ b/crates/email/src/message/delete.rs @@ -16,8 +16,8 @@ use store::{ ahash::AHashMap, roaring::RoaringBitmap, write::{ - BatchBuilder, Bincode, BitmapClass, F_BITMAP, F_CLEAR, F_VALUE, MaybeDynamicId, TagValue, - ValueClass, log::ChangeLogBuilder, + BatchBuilder, Bincode, BitmapClass, MaybeDynamicId, TagValue, ValueClass, + log::ChangeLogBuilder, }, }; use trc::{AddContext, StoreEvent}; @@ -155,11 +155,9 @@ impl EmailDeletion for Server { changes.log_child_update(Collection::Mailbox, mailbox_id.mailbox_id); } - batch.value( - Property::MailboxIds, - delete_properties.mailboxes, - F_VALUE | F_BITMAP | F_CLEAR, - ); + batch + .untag_many(Property::MailboxIds, delete_properties.mailboxes.iter()) + .clear(Property::MailboxIds); } else { trc::event!( Store(StoreEvent::NotFound), @@ -170,7 +168,9 @@ impl EmailDeletion for Server { ); } if let Some(thread_id) = delete_properties.thread_id { - batch.value(Property::ThreadId, thread_id, F_VALUE | F_BITMAP | F_CLEAR); + batch + .untag(Property::ThreadId, thread_id) + .clear(Property::ThreadId); // Log message deletion changes.log_delete(Collection::Email, Id::from_parts(thread_id, document_id)); @@ -191,7 +191,6 @@ impl EmailDeletion for Server { batch.tag( Property::MailboxIds, TagValue::Id(MaybeDynamicId::Static(TOMBSTONE_ID)), - 0, ); document_ids.remove(document_id); @@ -432,10 +431,9 @@ impl EmailDeletion for Server { .with_collection(Collection::Email) .delete_document(document_id) .clear(Property::Cid) - .tag( + .untag( Property::MailboxIds, TagValue::Id(MaybeDynamicId::Static(TOMBSTONE_ID)), - F_CLEAR, ); // Remove keywords @@ -451,7 +449,9 @@ impl EmailDeletion for Server { }) .await? { - batch.value(Property::Keywords, keywords, F_VALUE | F_BITMAP | F_CLEAR); + batch + .untag_many(Property::Keywords, keywords.into_iter()) + .clear(Property::Keywords); } else { trc::event!( Purge(trc::PurgeEvent::Error), @@ -491,7 +491,9 @@ impl EmailDeletion for Server { // SPDX-SnippetEnd // Delete message - EmailIndexBuilder::clear(metadata.inner).build(&mut batch, account_id, tenant_id); + EmailIndexBuilder::clear(metadata.inner) + .build(&mut batch, account_id, tenant_id) + .caused_by(trc::location!())?; // Commit batch self.core.storage.data.write(batch.build()).await?; diff --git a/crates/email/src/message/index.rs b/crates/email/src/message/index.rs index b370f9ec..20b7a109 100644 --- a/crates/email/src/message/index.rs +++ b/crates/email/src/message/index.rs @@ -8,17 +8,19 @@ use std::borrow::Cow; use jmap_proto::types::{keyword::Keyword, property::Property}; use mail_parser::{ - decoders::html::html_to_text, - parsers::{fields::thread::thread_name, preview::preview_text}, Addr, Address, GetHeader, Group, Header, HeaderName, HeaderValue, Message, MessagePart, PartType, + decoders::html::html_to_text, + parsers::{fields::thread::thread_name, preview::preview_text}, }; use nlp::language::Language; use store::{ + Serialize, SerializeInfallible, backend::MAX_TOKEN_LENGTH, - fts::{index::FtsDocument, Field}, - write::{BatchBuilder, Bincode, BlobOp, DirectoryClass, F_BITMAP, F_CLEAR, F_INDEX, F_VALUE}, + fts::{Field, index::FtsDocument}, + write::{BatchBuilder, Bincode, BlobOp, DirectoryClass}, }; +use trc::AddContext; use utils::BlobHash; use crate::mailbox::UidMailbox; @@ -48,9 +50,9 @@ pub(super) trait IndexMessage { keywords: Vec, mailbox_ids: Vec, received_at: u64, - ) -> &mut Self; + ) -> trc::Result<&mut Self>; - fn index_headers(&mut self, headers: &[Header<'_>], options: u32); + fn index_headers(&mut self, headers: &[Header<'_>], set: bool); } pub trait IndexMessageText<'x>: Sized { @@ -67,19 +69,30 @@ impl IndexMessage for BatchBuilder { keywords: Vec, mailbox_ids: Vec, received_at: u64, - ) -> &mut Self { + ) -> trc::Result<&mut Self> { // Index keywords - self.value(Property::Keywords, keywords, F_VALUE | F_BITMAP); + self.set( + Property::Keywords, + keywords.serialize().caused_by(trc::location!())?, + ) + .tag_many(Property::Keywords, keywords.into_iter()); // Index mailboxIds - self.value(Property::MailboxIds, mailbox_ids, F_VALUE | F_BITMAP); + self.set( + Property::MailboxIds, + mailbox_ids.serialize().caused_by(trc::location!())?, + ) + .tag_many(Property::MailboxIds, mailbox_ids.iter()); // Index size - self.value(Property::Size, message.raw_message.len() as u32, F_INDEX) - .add( - DirectoryClass::UsedQuota(account_id), - message.raw_message.len() as i64, - ); + self.index( + Property::Size, + (message.raw_message.len() as u32).serialize(), + ) + .add( + DirectoryClass::UsedQuota(account_id), + message.raw_message.len() as i64, + ); if let Some(tenant_id) = tenant_id { self.add( DirectoryClass::UsedQuota(tenant_id), @@ -88,7 +101,7 @@ impl IndexMessage for BatchBuilder { } // Index receivedAt - self.value(Property::ReceivedAt, received_at, F_INDEX); + self.index(Property::ReceivedAt, received_at.serialize()); let mut has_attachments = false; let mut preview = None; @@ -101,7 +114,7 @@ impl IndexMessage for BatchBuilder { for (part_id, part) in message.parts.iter().take(MAX_MESSAGE_PARTS).enumerate() { if part_id == 0 { - self.index_headers(&part.headers, 0); + self.index_headers(&part.headers, true); } match &part.body { @@ -139,7 +152,7 @@ impl IndexMessage for BatchBuilder { // Store and index hasAttachment property if has_attachments { - self.tag(Property::HasAttachment, (), 0); + self.tag(Property::HasAttachment, ()); } // Link blob @@ -152,7 +165,7 @@ impl IndexMessage for BatchBuilder { // Store message metadata let root_part = message.root_part(); - self.value( + self.set( Property::BodyStructure, Bincode::new(MessageMetadata { preview: preview.unwrap_or_default().into_owned(), @@ -167,14 +180,15 @@ impl IndexMessage for BatchBuilder { received_at, has_attachments, blob_hash, - }), - F_VALUE, + }) + .serialize() + .caused_by(trc::location!())?, ); - self + Ok(self) } - fn index_headers(&mut self, headers: &[Header<'_>], options: u32) { + fn index_headers(&mut self, headers: &[Header<'_>], set: bool) { let mut seen_headers = [false; 40]; for header in headers.iter().rev() { if matches!(header.name, HeaderName::Other(_)) { @@ -186,8 +200,13 @@ impl IndexMessage for BatchBuilder { header.value.visit_text(|id| { // Add ids to inverted index if id.len() < MAX_ID_LENGTH { - self.value(Property::MessageId, id, F_INDEX | options); - self.value(Property::References, id, F_INDEX | options); + if set { + self.index(Property::MessageId, id.serialize()) + .index(Property::References, id.serialize()); + } else { + self.unindex(Property::MessageId, id.serialize()) + .unindex(Property::References, id.serialize()); + } } }); } @@ -195,7 +214,11 @@ impl IndexMessage for BatchBuilder { header.value.visit_text(|id| { // Add ids to inverted index if id.len() < MAX_ID_LENGTH { - self.value(Property::References, id, F_INDEX | options); + if set { + self.index(Property::References, id.serialize()); + } else { + self.unindex(Property::References, id.serialize()); + } } }); } @@ -221,18 +244,23 @@ impl IndexMessage for BatchBuilder { }); // Add address to inverted index - self.value(u8::from(&property), sort_text.build(), F_INDEX | options); + if set { + self.index(u8::from(&property), sort_text.build()); + } else { + self.unindex(u8::from(&property), sort_text.build()); + } seen_headers[header.name.id() as usize] = true; } } HeaderName::Date => { if !seen_headers[header.name.id() as usize] { if let HeaderValue::DateTime(datetime) = &header.value { - self.value( - Property::SentAt, - datetime.to_timestamp() as u64, - F_INDEX | options, - ); + let value = (datetime.to_timestamp() as u64).serialize(); + if set { + self.index(Property::SentAt, value); + } else { + self.unindex(Property::SentAt, value); + } } seen_headers[header.name.id() as usize] = true; } @@ -250,15 +278,18 @@ impl IndexMessage for BatchBuilder { // Index thread name let thread_name = thread_name(&subject); - self.value( - Property::Subject, - if !thread_name.is_empty() { - thread_name.trim_text(MAX_SORT_FIELD_LENGTH) - } else { - "!" - }, - F_INDEX | options, - ); + let thread_name = if !thread_name.is_empty() { + thread_name.trim_text(MAX_SORT_FIELD_LENGTH) + } else { + "!" + } + .serialize(); + + if set { + self.index(Property::Subject, thread_name); + } else { + self.unindex(Property::Subject, thread_name); + } seen_headers[header.name.id() as usize] = true; } @@ -270,7 +301,11 @@ impl IndexMessage for BatchBuilder { // Add subject to index if missing if !seen_headers[HeaderName::Subject.id() as usize] { - self.value(Property::Subject, "!", F_INDEX | options); + if set { + self.index(Property::Subject, "!".serialize()); + } else { + self.unindex(Property::Subject, "!".serialize()); + } } } } @@ -407,17 +442,26 @@ impl<'x> EmailIndexBuilder<'x> { } impl EmailIndexBuilder<'_> { - pub fn build(self, batch: &mut BatchBuilder, account_id: u32, tenant_id: Option) { - let options = if self.set { + pub fn build( + self, + batch: &mut BatchBuilder, + account_id: u32, + tenant_id: Option, + ) -> trc::Result<()> { + let metadata = &self.inner.inner; + if self.set { // Serialize metadata - batch.value(Property::BodyStructure, &self.inner, F_VALUE); - 0 + batch + .set(Property::BodyStructure, (self.inner).serialize()?) + .index(Property::Size, (metadata.size as u32).serialize()) + .index(Property::ReceivedAt, (metadata.received_at).serialize()); } else { // Delete metadata - batch.value(Property::BodyStructure, (), F_VALUE | F_CLEAR); - F_CLEAR - }; - let metadata = &self.inner.inner; + batch + .clear(Property::BodyStructure) + .unindex(Property::Size, (metadata.size as u32).serialize()) + .unindex(Property::ReceivedAt, (metadata.received_at).serialize()); + } // Index properties let quota = if self.set { @@ -425,24 +469,21 @@ impl EmailIndexBuilder<'_> { } else { -(metadata.size as i64) }; - batch - .value(Property::Size, metadata.size as u32, F_INDEX | options) - .add(DirectoryClass::UsedQuota(account_id), quota); + batch.add(DirectoryClass::UsedQuota(account_id), quota); if let Some(tenant_id) = tenant_id { batch.add(DirectoryClass::UsedQuota(tenant_id), quota); } - batch.value( - Property::ReceivedAt, - metadata.received_at, - F_INDEX | options, - ); if metadata.has_attachments { - batch.tag(Property::HasAttachment, (), options); + if self.set { + batch.tag(Property::HasAttachment, ()); + } else { + batch.untag(Property::HasAttachment, ()); + } } // Index headers - batch.index_headers(&metadata.contents.parts[0].headers, options); + batch.index_headers(&metadata.contents.parts[0].headers, self.set); // Link blob if self.set { @@ -457,6 +498,7 @@ impl EmailIndexBuilder<'_> { hash: metadata.blob_hash.clone(), }); } + Ok(()) } } diff --git a/crates/email/src/message/ingest.rs b/crates/email/src/message/ingest.rs index 41ef73f5..a9d68a98 100644 --- a/crates/email/src/message/ingest.rs +++ b/crates/email/src/message/ingest.rs @@ -32,18 +32,18 @@ use spam_filter::{ SpamFilterInput, analysis::init::SpamFilterInit, modules::bayes::BayesClassifier, }; use std::future::Future; -use store::rand::Rng; use store::{ - BitmapKey, BlobClass, Serialize, + BitmapKey, BlobClass, ahash::AHashSet, query::Filter, write::{ - AssignedIds, BatchBuilder, BitmapClass, F_BITMAP, F_CLEAR, F_VALUE, MaybeDynamicId, - MaybeDynamicValue, SerializeWithId, TagValue, TaskQueueClass, ValueClass, + AssignedIds, BatchBuilder, BitmapClass, MaybeDynamicId, MaybeDynamicValue, SerializeWithId, + TagValue, TaskQueueClass, ValueClass, log::{ChangeLogBuilder, Changes, LogInsert}, now, }, }; +use store::{SerializeInfallible, rand::Rng}; use trc::{AddContext, MessageIngestEvent}; use utils::map::vec_map::VecMap; @@ -291,7 +291,7 @@ impl EmailIngest for Server { account_id, Collection::Email, vec![ - Filter::eq(Property::MessageId, &message_id), + Filter::eq(Property::MessageId, message_id.as_str().serialize()), Filter::is_in_bitmap( Property::MailboxIds, params.mailbox_ids.first().copied().unwrap_or(INBOX_ID), @@ -497,9 +497,10 @@ impl EmailIngest for Server { mailbox_ids, params.received_at.unwrap_or_else(now), ) - .value(Property::Cid, change_id, F_VALUE) + .caused_by(trc::location!())? + .set(Property::Cid, change_id.serialize()) .set(Property::ThreadId, maybe_thread_id) - .tag(Property::ThreadId, TagValue::Id(maybe_thread_id), 0) + .tag(Property::ThreadId, TagValue::Id(maybe_thread_id)) .set( ValueClass::TaskQueue(TaskQueueClass::IndexEmail { seq: self.generate_snowflake_id().caused_by(trc::location!())?, @@ -584,21 +585,20 @@ impl EmailIngest for Server { references: &[&str], ) -> trc::Result> { let mut try_count = 0; + let thread_name = if !thread_name.is_empty() { + thread_name + } else { + "!" + } + .serialize(); loop { // Find messages with matching references let mut filters = Vec::with_capacity(references.len() + 3); - filters.push(Filter::eq( - Property::Subject, - if !thread_name.is_empty() { - thread_name - } else { - "!" - }, - )); + filters.push(Filter::eq(Property::Subject, thread_name.clone())); filters.push(Filter::Or); for reference in references { - filters.push(Filter::eq(Property::References, *reference)); + filters.push(Filter::eq(Property::References, reference.serialize())); } filters.push(Filter::End); let results = self @@ -690,8 +690,9 @@ impl EmailIngest for Server { batch .update_document(document_id) .assert_value(Property::ThreadId, old_thread_id) - .value(Property::ThreadId, old_thread_id, F_BITMAP | F_CLEAR) - .value(Property::ThreadId, thread_id, F_VALUE | F_BITMAP); + .untag(Property::ThreadId, old_thread_id) + .tag(Property::ThreadId, thread_id) + .set(Property::ThreadId, thread_id.serialize()); changes.log_move( Collection::Email, Id::from_parts(old_thread_id, document_id), @@ -700,7 +701,7 @@ impl EmailIngest for Server { } } } - batch.custom(changes); + batch.custom(changes).caused_by(trc::location!())?; match self.core.storage.data.write(batch.build()).await { Ok(_) => return Ok(Some(thread_id)), diff --git a/crates/email/src/message/mod.rs b/crates/email/src/message/mod.rs index 8024560e..e0ee70c5 100644 --- a/crates/email/src/message/mod.rs +++ b/crates/email/src/message/mod.rs @@ -5,6 +5,7 @@ */ pub mod bayes; +pub mod copy; pub mod crypto; pub mod delete; pub mod delivery; diff --git a/crates/email/src/push/mod.rs b/crates/email/src/push/mod.rs index ad9efb9b..06cceec4 100644 --- a/crates/email/src/push/mod.rs +++ b/crates/email/src/push/mod.rs @@ -4,12 +4,13 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -pub mod serialize; - use jmap_proto::types::type_state::DataType; +use store::Serialize; use utils::map::bitmap::Bitmap; -#[derive(Debug, Default, Clone, PartialEq, Eq)] +#[derive( + rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Default, Debug, Clone, PartialEq, Eq, +)] pub struct PushSubscription { pub url: String, pub device_client_id: String, @@ -20,8 +21,16 @@ pub struct PushSubscription { pub keys: Option, } -#[derive(Debug, Default, Clone, PartialEq, Eq)] +#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] pub struct Keys { pub p256dh: Vec, pub auth: Vec, } + +impl Serialize for PushSubscription { + fn serialize(&self) -> trc::Result> { + rkyv::to_bytes::(self) + .map(|r| r.into_vec()) + .map_err(Into::into) + } +} diff --git a/crates/email/src/push/serialize.rs b/crates/email/src/push/serialize.rs deleted file mode 100644 index 23cefb6c..00000000 --- a/crates/email/src/push/serialize.rs +++ /dev/null @@ -1,23 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use store::{Deserialize, Serialize}; - -use super::PushSubscription; - -impl Serialize for PushSubscription { - fn serialize(self) -> Vec { - let todo = 1; - todo!() - } -} - -impl Deserialize for PushSubscription { - fn deserialize(bytes: &[u8]) -> trc::Result { - let todo = 1; - todo!() - } -} diff --git a/crates/email/src/sieve/activate.rs b/crates/email/src/sieve/activate.rs new file mode 100644 index 00000000..04cf759c --- /dev/null +++ b/crates/email/src/sieve/activate.rs @@ -0,0 +1,127 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use common::{Server, storage::index::ObjectIndexBuilder}; +use jmap_proto::types::{collection::Collection, property::Property}; +use store::{ + SerializeInfallible, + query::Filter, + write::{ArchivedValue, BatchBuilder, assert::HashedValue}, +}; +use trc::AddContext; + +use super::ArchivedSieveScript; + +pub trait SieveScriptActivate: Sync + Send { + fn sieve_activate_script( + &self, + account_id: u32, + activate_id: Option, + ) -> impl Future>> + Send; +} + +impl SieveScriptActivate for Server { + async fn sieve_activate_script( + &self, + account_id: u32, + mut activate_id: Option, + ) -> trc::Result> { + let mut changed_ids = Vec::new(); + // Find the currently active script + let mut active_ids = self + .store() + .filter( + account_id, + Collection::SieveScript, + vec![Filter::eq(Property::IsActive, 1u32.serialize())], + ) + .await? + .results; + + // Check if script is already active + if activate_id.is_some_and(|id| active_ids.remove(id)) { + if active_ids.is_empty() { + return Ok(changed_ids); + } else { + activate_id = None; + } + } + + // Prepare batch + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .with_collection(Collection::SieveScript); + + // Deactivate scripts + for document_id in active_ids { + if let Some(sieve) = self + .get_property::>>( + account_id, + Collection::SieveScript, + document_id, + Property::Value, + ) + .await? + { + let sieve = sieve.into_deserialized().caused_by(trc::location!())?; + let mut new_sieve = sieve.inner.clone(); + new_sieve.is_active = false; + batch + .update_document(document_id) + .clear(Property::EmailIds) + .custom( + ObjectIndexBuilder::new() + .with_changes(new_sieve) + .with_current(sieve), + ) + .caused_by(trc::location!())?; + changed_ids.push((document_id, false)); + } + } + + // Activate script + if let Some(document_id) = activate_id { + if let Some(sieve) = self + .get_property::>>( + account_id, + Collection::SieveScript, + document_id, + Property::Value, + ) + .await? + { + let sieve = sieve.into_deserialized().caused_by(trc::location!())?; + let mut new_sieve = sieve.inner.clone(); + new_sieve.is_active = true; + batch + .update_document(document_id) + .custom( + ObjectIndexBuilder::new() + .with_changes(new_sieve) + .with_current(sieve), + ) + .caused_by(trc::location!())?; + changed_ids.push((document_id, true)); + } + } + + // Write changes + if !changed_ids.is_empty() { + match self.core.storage.data.write(batch.build()).await { + Ok(_) => (), + Err(err) if err.is_assertion_failure() => { + return Ok(vec![]); + } + Err(err) => { + return Err(err.caused_by(trc::location!())); + } + } + } + + Ok(changed_ids) + } +} diff --git a/crates/email/src/sieve/delete.rs b/crates/email/src/sieve/delete.rs new file mode 100644 index 00000000..1bbead79 --- /dev/null +++ b/crates/email/src/sieve/delete.rs @@ -0,0 +1,81 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use common::{Server, auth::ResourceToken, storage::index::ObjectIndexBuilder}; +use jmap_proto::types::{collection::Collection, property::Property}; +use store::write::{ArchivedValue, BatchBuilder, BlobOp, assert::HashedValue}; +use trc::AddContext; + +use super::ArchivedSieveScript; + +pub trait SieveScriptDelete: Sync + Send { + fn sieve_script_delete( + &self, + resource_token: &ResourceToken, + document_id: u32, + fail_if_active: bool, + ) -> impl Future> + Send; +} + +impl SieveScriptDelete for Server { + async fn sieve_script_delete( + &self, + resource_token: &ResourceToken, + document_id: u32, + fail_if_active: bool, + ) -> trc::Result { + // Fetch record + let account_id = resource_token.account_id; + let obj = self + .get_property::>>( + account_id, + Collection::SieveScript, + document_id, + Property::Value, + ) + .await? + .ok_or_else(|| { + trc::StoreEvent::NotFound + .into_err() + .caused_by(trc::location!()) + .document_id(document_id) + })? + .into_deserialized() + .caused_by(trc::location!())?; + + // Make sure the script is not active + if fail_if_active && obj.inner.is_active { + return Ok(false); + } + + let blob_hash = obj.inner.blob_hash.clone(); + let mut builder = ObjectIndexBuilder::new().with_current(obj); + // Update tenant quota + #[cfg(feature = "enterprise")] + if self.core.is_enterprise_edition() { + if let Some(tenant) = resource_token.tenant { + builder.set_tenant_id(tenant.id); + } + } + + // Delete record + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .with_collection(Collection::SieveScript) + .delete_document(document_id) + .clear(Property::EmailIds) + .clear(BlobOp::Link { hash: blob_hash }) + .custom(builder) + .caused_by(trc::location!())?; + + self.store() + .write(batch) + .await + .caused_by(trc::location!())?; + Ok(true) + } +} diff --git a/crates/email/src/sieve/index.rs b/crates/email/src/sieve/index.rs index 49215942..7156ae0d 100644 --- a/crates/email/src/sieve/index.rs +++ b/crates/email/src/sieve/index.rs @@ -4,10 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use jmap_proto::{ - object::index::{IndexValue, IndexableObject}, - types::property::Property, -}; +use common::storage::index::{IndexValue, IndexableObject}; +use jmap_proto::types::property::Property; use super::SieveScript; @@ -16,17 +14,13 @@ impl IndexableObject for SieveScript { [ IndexValue::Text { field: Property::Name.into(), - value: self.name.as_str(), - tokenize: true, - index: true, + value: self.name.to_lowercase().into(), }, IndexValue::U32 { field: Property::IsActive.into(), value: Some(self.is_active as u32), }, - IndexValue::Quota { - used: self.blob_id.section.as_ref().map_or(0, |b| b.size as u32), - }, + IndexValue::Quota { used: self.size }, ] .into_iter() } diff --git a/crates/email/src/sieve/ingest.rs b/crates/email/src/sieve/ingest.rs index af4d9429..5a8ec1ce 100644 --- a/crates/email/src/sieve/ingest.rs +++ b/crates/email/src/sieve/ingest.rs @@ -21,17 +21,17 @@ use jmap_proto::types::{collection::Collection, id::Id, keyword::Keyword, proper use mail_parser::MessageParser; use sieve::{Envelope, Event, Input, Mailbox, Recipient, Sieve}; use store::{ - Deserialize, Serialize, + Deserialize, Serialize, SerializeInfallible, ahash::AHashSet, query::Filter, - write::{BatchBuilder, Bincode, BlobOp, F_VALUE, assert::HashedValue, now}, + write::{ArchivedValue, BatchBuilder, Bincode, BlobOp, assert::HashedValue, now}, }; use trc::{AddContext, SieveEvent}; use utils::config::utils::ParseValue; use std::future::Future; -use super::{ActiveScript, SeenIdHash, SeenIds, SieveScript}; +use super::{ActiveScript, ArchivedSieveScript, SeenIdHash, SeenIds}; struct SieveMessage<'x> { pub raw_message: Cow<'x, [u8]>, @@ -67,7 +67,7 @@ pub trait SieveScriptIngest: Sync + Send { &self, account_id: u32, document_id: u32, - ) -> impl Future> + Send; + ) -> impl Future> + Send; } impl SieveScriptIngest for Server { @@ -529,10 +529,11 @@ impl SieveScriptIngest for Server { .with_account_id(account_id) .with_collection(Collection::SieveScript) .update_document(active_script.document_id) - .value( + .set( Property::EmailIds, - Bincode::new(active_script.seen_ids), - F_VALUE, + Bincode::new(active_script.seen_ids) + .serialize() + .caused_by(trc::location!())?, ); if let Err(err) = self.store().write(batch).await.caused_by(trc::location!()) { trc::error!(err.details("Failed to save Sieve seen ids changes.")); @@ -561,19 +562,18 @@ impl SieveScriptIngest for Server { .filter( account_id, Collection::SieveScript, - vec![Filter::eq(Property::IsActive, 1u32)], + vec![Filter::eq(Property::IsActive, 1u32.serialize())], ) .await .caused_by(trc::location!())? .results .min() { - let (script, script_object) = - self.sieve_script_compile(account_id, document_id).await?; + let (script, script_name) = self.sieve_script_compile(account_id, document_id).await?; Ok(Some(ActiveScript { document_id, script: Arc::new(script), - script_name: script_object.name, + script_name, seen_ids: self .get_property::>( account_id, @@ -601,7 +601,7 @@ impl SieveScriptIngest for Server { .filter( account_id, Collection::SieveScript, - vec![Filter::eq(Property::Name, name)], + vec![Filter::eq(Property::Name, name.serialize())], ) .await .caused_by(trc::location!())? @@ -621,10 +621,10 @@ impl SieveScriptIngest for Server { &self, account_id: u32, document_id: u32, - ) -> trc::Result<(Sieve, SieveScript)> { + ) -> trc::Result<(Sieve, String)> { // Obtain script object let script_object = self - .get_property::>( + .get_property::>>( account_id, Collection::SieveScript, document_id, @@ -639,24 +639,18 @@ impl SieveScriptIngest for Server { })?; // Obtain the sieve script length - let blob_id = &script_object.inner.blob_id; - let script_offset = blob_id - .section - .as_ref() - .ok_or_else(|| { - trc::StoreEvent::NotFound - .into_err() - .caused_by(trc::location!()) - .document_id(document_id) - })? - .size; + let unarchived_script = script_object + .inner + .unarchive() + .caused_by(trc::location!())?; + let script_offset = u32::from(unarchived_script.size) as usize; // Obtain the sieve script blob let script_bytes = self .core .storage .blob - .get_blob(blob_id.hash.as_ref(), 0..usize::MAX) + .get_blob(unarchived_script.blob_hash.0.as_ref(), 0..usize::MAX) .await .caused_by(trc::location!())? .ok_or_else(|| { @@ -671,7 +665,7 @@ impl SieveScriptIngest for Server { .get(script_offset..) .and_then(|bytes| Bincode::::deserialize(bytes).ok()) { - Ok((sieve.inner, script_object.inner)) + Ok((sieve.inner, unarchived_script.name.to_string())) } else { // Deserialization failed, probably because the script compiler version changed match self.core.sieve.untrusted_compiler.compile( @@ -685,20 +679,21 @@ impl SieveScriptIngest for Server { Ok(sieve) => { // Store updated compiled sieve script let sieve = Bincode::new(sieve); - let compiled_bytes = (&sieve).serialize(); + let compiled_bytes = sieve.serialize().caused_by(trc::location!())?; let mut updated_sieve_bytes = Vec::with_capacity(script_offset + compiled_bytes.len()); updated_sieve_bytes.extend_from_slice(&script_bytes[0..script_offset]); updated_sieve_bytes.extend_from_slice(&compiled_bytes); // Store updated blob - let mut new_blob_id = blob_id.clone(); - new_blob_id.hash = self + let new_blob_hash = self .put_blob(account_id, &updated_sieve_bytes, false) .await? .hash; - let mut new_script_object = script_object.inner.clone(); - new_script_object.blob_id = new_blob_id.clone(); + let mut new_script_object = + rkyv::deserialize(unarchived_script).caused_by(trc::location!())?; + let blob_hash = + std::mem::replace(&mut new_script_object.blob_hash, new_blob_hash.clone()); // Update script object let mut batch = BatchBuilder::new(); @@ -707,13 +702,14 @@ impl SieveScriptIngest for Server { .with_collection(Collection::SieveScript) .update_document(document_id) .assert_value(Property::Value, &script_object) - .set(Property::Value, (&new_script_object).serialize()) - .clear(BlobOp::Link { - hash: blob_id.hash.clone(), - }) + .set( + Property::Value, + new_script_object.serialize().caused_by(trc::location!())?, + ) + .clear(BlobOp::Link { hash: blob_hash }) .set( BlobOp::Link { - hash: new_blob_id.hash, + hash: new_blob_hash, }, Vec::new(), ); @@ -722,7 +718,7 @@ impl SieveScriptIngest for Server { .await .caused_by(trc::location!())?; - Ok((sieve.inner, new_script_object)) + Ok((sieve.inner, new_script_object.name)) } Err(error) => Err(trc::StoreEvent::UnexpectedError .caused_by(trc::location!()) @@ -732,20 +728,3 @@ impl SieveScriptIngest for Server { } } } - -/* -#[inline(always)] -pub fn is_valid_role(role: &str) -> bool { - [ - "inbox", - "trash", - "spam", - "junk", - "drafts", - "archive", - "sent", - "important", - ] - .contains(&role) -} -*/ diff --git a/crates/email/src/sieve/mod.rs b/crates/email/src/sieve/mod.rs index 1b046020..ac40003c 100644 --- a/crates/email/src/sieve/mod.rs +++ b/crates/email/src/sieve/mod.rs @@ -4,16 +4,19 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::sync::Arc; +use std::{collections::HashSet, sync::Arc}; -use jmap_proto::types::blob::BlobId; use sieve::Sieve; -use store::{ahash::AHashSet, blake3}; +use store::{ahash::RandomState, blake3}; +use utils::BlobHash; +pub mod activate; +pub mod delete; pub mod index; pub mod ingest; pub mod serialize; +#[derive(Debug, Clone, PartialEq, Eq)] pub struct ActiveScript { pub document_id: u32, pub script_name: String, @@ -21,27 +24,34 @@ pub struct ActiveScript { pub seen_ids: SeenIds, } -#[derive(Debug, Clone)] +#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone)] pub struct SeenIdHash { hash: [u8; 32], expiry: u64, } -#[derive(Debug, Clone, Default)] +#[derive( + rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Default, Debug, Clone, PartialEq, Eq, +)] pub struct SeenIds { - pub ids: AHashSet, + pub ids: HashSet, pub has_changes: bool, } -#[derive(Debug, Default, Clone, PartialEq, Eq)] +#[derive( + rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, +)] pub struct SieveScript { pub name: String, pub is_active: bool, - pub blob_id: BlobId, + pub blob_hash: BlobHash, + pub size: u32, pub vacation_response: Option, } -#[derive(Debug, Default, Clone, PartialEq, Eq)] +#[derive( + rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, +)] pub struct VacationResponse { pub from_date: Option, pub to_date: Option, @@ -51,12 +61,13 @@ pub struct VacationResponse { } impl SieveScript { - pub fn new(name: impl Into, blob_id: BlobId) -> Self { + pub fn new(name: impl Into, blob_hash: BlobHash) -> Self { SieveScript { name: name.into(), is_active: false, - blob_id, + blob_hash, vacation_response: None, + size: 0, } } @@ -65,8 +76,8 @@ impl SieveScript { self } - pub fn with_blob_id(mut self, blob_id: BlobId) -> Self { - self.blob_id = blob_id; + pub fn with_blob_hash(mut self, blob_hash: BlobHash) -> Self { + self.blob_hash = blob_hash; self } @@ -75,6 +86,11 @@ impl SieveScript { self } + pub fn with_size(mut self, size: u32) -> Self { + self.size = size; + self + } + pub fn set_is_active(&mut self, is_active: bool) { self.is_active = is_active; } @@ -116,3 +132,17 @@ impl PartialEq for SeenIdHash { } impl Eq for SeenIdHash {} + +impl std::hash::Hash for ArchivedSeenIdHash { + fn hash(&self, state: &mut H) { + self.hash.hash(state); + } +} + +impl PartialEq for ArchivedSeenIdHash { + fn eq(&self, other: &Self) -> bool { + self.hash == other.hash + } +} + +impl Eq for ArchivedSeenIdHash {} diff --git a/crates/email/src/sieve/serialize.rs b/crates/email/src/sieve/serialize.rs index 044b2161..a1b4d75a 100644 --- a/crates/email/src/sieve/serialize.rs +++ b/crates/email/src/sieve/serialize.rs @@ -4,11 +4,21 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use std::collections::HashSet; + use serde::ser::SerializeSeq; -use store::{Deserialize, Serialize, ahash::AHashSet, write::now}; +use store::{Serialize, ahash::RandomState, write::now}; use super::{SeenIdHash, SeenIds, SieveScript}; +impl Serialize for SieveScript { + fn serialize(&self) -> trc::Result> { + rkyv::to_bytes::(self) + .map(|r| r.into_vec()) + .map_err(Into::into) + } +} + // SeenIds serializer impl serde::Serialize for SeenIds { fn serialize(&self, serializer: S) -> Result @@ -49,7 +59,7 @@ impl<'de> serde::de::Visitor<'de> for SeenIdsVisitor { { let num_entries = seq.size_hint().unwrap_or(0) / 2; let mut seen_ids = SeenIds { - ids: AHashSet::with_capacity(num_entries), + ids: HashSet::with_capacity_and_hasher(num_entries, RandomState::new()), has_changes: false, }; let now = now(); @@ -75,21 +85,3 @@ impl<'de> serde::de::Visitor<'de> for SeenIdsVisitor { Ok(seen_ids) } } - -impl Serialize for SieveScript { - fn serialize(self) -> Vec { - todo!() - } -} - -impl Serialize for &SieveScript { - fn serialize(self) -> Vec { - todo!() - } -} - -impl Deserialize for SieveScript { - fn deserialize(bytes: &[u8]) -> trc::Result { - todo!() - } -} diff --git a/crates/email/src/submission/index.rs b/crates/email/src/submission/index.rs index c8c98417..df356427 100644 --- a/crates/email/src/submission/index.rs +++ b/crates/email/src/submission/index.rs @@ -4,10 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ - use jmap_proto::{ - object::index::{IndexValue, IndexableObject}, - types::property::Property, -}; +use common::storage::index::{IndexValue, IndexableObject}; +use jmap_proto::types::property::Property; use super::EmailSubmission; @@ -16,9 +14,7 @@ impl IndexableObject for EmailSubmission { [ IndexValue::Text { field: Property::UndoStatus.into(), - value: self.undo_status.as_index(), - tokenize: false, - index: true, + value: self.undo_status.as_index().into(), }, IndexValue::U32 { field: Property::EmailId.into(), diff --git a/crates/email/src/submission/mod.rs b/crates/email/src/submission/mod.rs index 278537e4..847d5c05 100644 --- a/crates/email/src/submission/mod.rs +++ b/crates/email/src/submission/mod.rs @@ -4,12 +4,14 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use store::Serialize; use utils::map::vec_map::VecMap; pub mod index; -pub mod serialize; -#[derive(Debug, Default, Clone, PartialEq, Eq)] +#[derive( + rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, +)] pub struct EmailSubmission { pub email_id: u32, pub thread_id: u32, @@ -21,26 +23,34 @@ pub struct EmailSubmission { pub delivery_status: VecMap, } -#[derive(Debug, Default, Clone, PartialEq, Eq)] +#[derive( + rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, +)] pub struct Envelope { pub mail_from: Address, pub rcpt_to: Vec
, } -#[derive(Debug, Default, Clone, PartialEq, Eq)] +#[derive( + rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, +)] pub struct Address { pub email: String, pub parameters: Option>>, } -#[derive(Debug, Default, Clone, PartialEq, Eq)] +#[derive( + rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, +)] pub struct DeliveryStatus { pub smtp_reply: String, pub delivered: Delivered, pub displayed: bool, } -#[derive(Debug, Default, Clone, PartialEq, Eq)] +#[derive( + rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, +)] pub enum Delivered { Queued, Yes, @@ -49,7 +59,9 @@ pub enum Delivered { Unknown, } -#[derive(Debug, Default, Clone, PartialEq, Eq)] +#[derive( + rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, +)] pub enum UndoStatus { #[default] Pending, @@ -84,6 +96,39 @@ impl UndoStatus { } } +impl ArchivedUndoStatus { + pub fn as_str(&self) -> &'static str { + match self { + ArchivedUndoStatus::Pending => "pending", + ArchivedUndoStatus::Final => "final", + ArchivedUndoStatus::Canceled => "canceled", + } + } + + pub fn as_index(&self) -> &'static str { + match self { + ArchivedUndoStatus::Pending => "p", + ArchivedUndoStatus::Final => "f", + ArchivedUndoStatus::Canceled => "c", + } + } +} + +impl From<&ArchivedDeliveryStatus> for DeliveryStatus { + fn from(value: &ArchivedDeliveryStatus) -> Self { + DeliveryStatus { + smtp_reply: value.smtp_reply.to_string(), + delivered: match value.delivered { + ArchivedDelivered::Queued => Delivered::Queued, + ArchivedDelivered::Yes => Delivered::Yes, + ArchivedDelivered::No => Delivered::No, + ArchivedDelivered::Unknown => Delivered::Unknown, + }, + displayed: value.displayed, + } + } +} + impl Delivered { pub fn as_str(&self) -> &'static str { match self { @@ -94,3 +139,22 @@ impl Delivered { } } } + +impl ArchivedDelivered { + pub fn as_str(&self) -> &'static str { + match self { + ArchivedDelivered::Queued => "queued", + ArchivedDelivered::Yes => "yes", + ArchivedDelivered::No => "no", + ArchivedDelivered::Unknown => "unknown", + } + } +} + +impl Serialize for EmailSubmission { + fn serialize(&self) -> trc::Result> { + rkyv::to_bytes::(self) + .map(|r| r.into_vec()) + .map_err(Into::into) + } +} diff --git a/crates/email/src/submission/serialize.rs b/crates/email/src/submission/serialize.rs deleted file mode 100644 index bf3114c2..00000000 --- a/crates/email/src/submission/serialize.rs +++ /dev/null @@ -1,23 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use store::{Deserialize, Serialize}; - -use super::EmailSubmission; - -impl Serialize for EmailSubmission { - fn serialize(self) -> Vec { - let todo = 1; - todo!() - } -} - -impl Deserialize for EmailSubmission { - fn deserialize(bytes: &[u8]) -> trc::Result { - let todo = 1; - todo!() - } -} diff --git a/crates/groupware/Cargo.toml b/crates/groupware/Cargo.toml new file mode 100644 index 00000000..1f8eb0b9 --- /dev/null +++ b/crates/groupware/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "groupware" +version = "0.11.5" +edition = "2024" +resolver = "2" + +[dependencies] +utils = { path = "../utils" } +common = { path = "../common" } +jmap_proto = { path = "../jmap-proto" } +directory = { path = "../directory" } +calcard = { path = "/Users/me/code/calcard" } +hashify = "0.2" + +[features] +test_mode = [] +enterprise = [] + +[dev-dependencies] +tokio = { version = "1.23", features = ["full"] } diff --git a/crates/groupware/src/calendar/mod.rs b/crates/groupware/src/calendar/mod.rs new file mode 100644 index 00000000..15fa31d5 --- /dev/null +++ b/crates/groupware/src/calendar/mod.rs @@ -0,0 +1,91 @@ +use calcard::icalendar::ICalendar; +use jmap_proto::types::{acl::Acl, value::AclGrant}; +use utils::map::vec_map::VecMap; + +pub struct Calendar { + pub preferences: VecMap, + pub acls: Vec, +} + +pub struct CalendarPreferences { + pub name: String, + pub description: Option, + pub sort_order: u32, + pub color: Option, + pub is_subscribed: bool, + pub is_default: bool, + pub is_visible: bool, + pub include_in_availability: IncludeInAvailability, + pub default_alerts_with_time: VecMap, + pub default_alerts_without_time: VecMap, + pub time_zone: Timezone, +} + +pub struct CalendarEvent { + pub name: Option, + pub event: ICalendar, + pub calendar_ids: Vec, + pub user_properties: VecMap, + pub created: u64, + pub updated: u64, + pub may_invite_self: bool, + pub may_invite_others: bool, + pub hide_attendees: bool, + pub is_draft: bool, +} + +pub enum Timezone { + IANA(String), + Custom(ICalendar), + Default, +} + +pub enum IncludeInAvailability { + All, + Attending, + None, +} + +pub enum CalendarRight { + ReadFreeBusy, + ReadItems, + WriteAll, + WriteOwn, + UpdatePrivate, + RSVP, + Share, + Delete, +} + +impl TryFrom for CalendarRight { + type Error = Acl; + + fn try_from(value: Acl) -> Result { + match value { + Acl::ReadFreeBusy => Ok(CalendarRight::ReadFreeBusy), + Acl::ReadItems => Ok(CalendarRight::ReadItems), + Acl::Modify => Ok(CalendarRight::WriteAll), + Acl::ModifyItemsOwn => Ok(CalendarRight::WriteOwn), + Acl::ModifyPrivateProperties => Ok(CalendarRight::UpdatePrivate), + Acl::RSVP => Ok(CalendarRight::RSVP), + Acl::Share => Ok(CalendarRight::Share), + Acl::Delete => Ok(CalendarRight::Delete), + _ => Err(value), + } + } +} + +impl From for Acl { + fn from(value: CalendarRight) -> Self { + match value { + CalendarRight::ReadFreeBusy => Acl::ReadFreeBusy, + CalendarRight::ReadItems => Acl::ReadItems, + CalendarRight::WriteAll => Acl::Modify, + CalendarRight::WriteOwn => Acl::ModifyItemsOwn, + CalendarRight::UpdatePrivate => Acl::ModifyPrivateProperties, + CalendarRight::RSVP => Acl::RSVP, + CalendarRight::Share => Acl::Share, + CalendarRight::Delete => Acl::Delete, + } + } +} diff --git a/crates/groupware/src/contact/mod.rs b/crates/groupware/src/contact/mod.rs new file mode 100644 index 00000000..15a57f2c --- /dev/null +++ b/crates/groupware/src/contact/mod.rs @@ -0,0 +1,51 @@ +use calcard::vcard::VCard; +use jmap_proto::types::{acl::Acl, value::AclGrant}; + +pub struct AddressBook { + pub name: String, + pub description: Option, + pub sort_order: u32, + pub is_default: bool, + pub subscribers: Vec, + pub acls: Vec, +} + +pub enum AddressBookRight { + Read, + Write, + Share, + Delete, +} + +pub struct ContactCard { + pub name: Option, + pub addressbook_ids: Vec, + pub card: VCard, + pub created: u64, + pub updated: u64, +} + +impl TryFrom for AddressBookRight { + type Error = Acl; + + fn try_from(value: Acl) -> Result { + match value { + Acl::Read => Ok(AddressBookRight::Read), + Acl::Modify => Ok(AddressBookRight::Write), + Acl::Share => Ok(AddressBookRight::Share), + Acl::Delete => Ok(AddressBookRight::Delete), + _ => Err(value), + } + } +} + +impl From for Acl { + fn from(value: AddressBookRight) -> Self { + match value { + AddressBookRight::Read => Acl::Read, + AddressBookRight::Write => Acl::Modify, + AddressBookRight::Share => Acl::Share, + AddressBookRight::Delete => Acl::Delete, + } + } +} diff --git a/crates/groupware/src/file/mod.rs b/crates/groupware/src/file/mod.rs new file mode 100644 index 00000000..64ceff91 --- /dev/null +++ b/crates/groupware/src/file/mod.rs @@ -0,0 +1,13 @@ +use jmap_proto::types::{blob::BlobId, value::AclGrant}; + +pub struct FileNode { + pub parent_id: Option, + pub blob_id: Option, + pub size: Option, + pub name: String, + pub media_type: Option, + pub executable: bool, + pub created: u64, + pub modified: u64, + pub acls: Vec, +} diff --git a/crates/groupware/src/lib.rs b/crates/groupware/src/lib.rs new file mode 100644 index 00000000..7f48a0ca --- /dev/null +++ b/crates/groupware/src/lib.rs @@ -0,0 +1,3 @@ +pub mod calendar; +pub mod contact; +pub mod file; diff --git a/crates/imap-proto/src/protocol/capability.rs b/crates/imap-proto/src/protocol/capability.rs index e1873f48..4c8ef088 100644 --- a/crates/imap-proto/src/protocol/capability.rs +++ b/crates/imap-proto/src/protocol/capability.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{authenticate::Mechanism, ImapResponse}; +use super::{ImapResponse, authenticate::Mechanism}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Response { @@ -201,8 +201,8 @@ impl ImapResponse for Response { #[cfg(test)] mod tests { use crate::protocol::{ - capability::{Capability, Response}, ImapResponse, + capability::{Capability, Response}, }; #[test] diff --git a/crates/imap-proto/src/protocol/expunge.rs b/crates/imap-proto/src/protocol/expunge.rs index 301f1c45..3eca99cd 100644 --- a/crates/imap-proto/src/protocol/expunge.rs +++ b/crates/imap-proto/src/protocol/expunge.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{serialize_sequence, ImapResponse}; +use super::{ImapResponse, serialize_sequence}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Response { diff --git a/crates/imap-proto/src/protocol/select.rs b/crates/imap-proto/src/protocol/select.rs index 90b9c35a..9a91abad 100644 --- a/crates/imap-proto/src/protocol/select.rs +++ b/crates/imap-proto/src/protocol/select.rs @@ -6,7 +6,7 @@ use crate::{ResponseCode, StatusResponse}; -use super::{list::ListItem, ImapResponse, Sequence}; +use super::{ImapResponse, Sequence, list::ListItem}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { @@ -129,7 +129,7 @@ impl Exists { #[cfg(test)] mod tests { - use crate::protocol::{list::ListItem, ImapResponse}; + use crate::protocol::{ImapResponse, list::ListItem}; use super::HighestModSeq; diff --git a/crates/imap/Cargo.toml b/crates/imap/Cargo.toml index 5a99796d..a8a34977 100644 --- a/crates/imap/Cargo.toml +++ b/crates/imap/Cargo.toml @@ -6,7 +6,6 @@ resolver = "2" [dependencies] imap_proto = { path = "../imap-proto" } -jmap = { path = "../jmap" } jmap_proto = { path = "../jmap-proto" } directory = { path = "../directory" } trc = { path = "../trc" } diff --git a/crates/imap/src/core/mailbox.rs b/crates/imap/src/core/mailbox.rs index b02b4b5e..faad4f71 100644 --- a/crates/imap/src/core/mailbox.rs +++ b/crates/imap/src/core/mailbox.rs @@ -7,19 +7,19 @@ use ahash::AHashMap; use common::{ AccountId, Mailbox, auth::AccessToken, - config::jmap::settings::SpecialUse, + config::jmap::settings::{ArchivedSpecialUse, SpecialUse}, listener::{SessionStream, limiter::InFlight}, + sharing::EffectiveAcl, }; use directory::{QueryBy, backend::internal::PrincipalField}; -use email::mailbox::{INBOX_ID, manage::MailboxFnc}; +use email::mailbox::{ArchivedMailbox, INBOX_ID, manage::MailboxFnc}; use imap_proto::protocol::list::Attribute; -use jmap::{ - auth::acl::{AclMethods, EffectiveAcl}, - changes::get::ChangesLookup, -}; use jmap_proto::types::{acl::Acl, collection::Collection, id::Id, property::Property}; use parking_lot::Mutex; -use store::query::log::{Change, Query}; +use store::{ + query::log::{Change, Query}, + write::ArchivedValue, +}; use trc::AddContext; use super::{Account, MailboxId, MailboxSync, Session, SessionData}; @@ -143,11 +143,19 @@ impl SessionData { }; // Fetch mailboxes + struct MailboxData { + mailbox_id: u32, + parent_id: u32, + role: SpecialUse, + name: String, + is_subscribed: bool, + } + let mut mailboxes = Vec::with_capacity(10); let mut special_uses = AHashMap::new(); - for (mailbox_id, mailbox) in self + for (mailbox_id, mailbox_) in self .server - .get_properties::( + .get_properties::, _, _>( account_id, Collection::Mailbox, &mailbox_ids, @@ -156,13 +164,24 @@ impl SessionData { .await .caused_by(trc::location!())? { + let mailbox = mailbox_.unarchive().caused_by(trc::location!())?; // Map special uses - if mailbox.role != SpecialUse::None { - special_uses.insert(mailbox.role, mailbox_id); + let role = SpecialUse::from(&mailbox.role); + if !matches!(mailbox.role, ArchivedSpecialUse::None) { + special_uses.insert(role, mailbox_id); } // Add mailbox id - mailboxes.push((mailbox_id, mailbox.parent_id, mailbox)); + mailboxes.push(MailboxData { + mailbox_id, + parent_id: u32::from(mailbox.parent_id), + role, + name: mailbox.name.to_string(), + is_subscribed: mailbox + .subscribers + .iter() + .any(|s| u32::from(s) == access_token.primary_id()), + }); } // Build tree @@ -197,27 +216,27 @@ impl SessionData { .map(|k| k.len() + std::mem::size_of::()) .sum::() + (account.mailbox_state.len() - * (std::mem::size_of::() + std::mem::size_of::()))) - as u64; + * (std::mem::size_of::>() + + std::mem::size_of::()))) as u64; loop { - while let Some((mailbox_id, mailbox_parent_id, mailbox)) = iter.next() { - if *mailbox_parent_id == parent_id { + while let Some(mailbox) = iter.next() { + if mailbox.parent_id == parent_id { let mut mailbox_path = path.clone(); - if *mailbox_id != INBOX_ID || account.prefix.is_some() { + if mailbox.mailbox_id != INBOX_ID || account.prefix.is_some() { mailbox_path.push(mailbox.name.clone()); } else { mailbox_path.push("INBOX".to_string()); } let has_children = mailboxes .iter() - .any(|(_, child_parent_id, _)| *child_parent_id == *mailbox_id + 1); + .any(|child| child.parent_id == mailbox.mailbox_id + 1); account.mailbox_state.insert( - *mailbox_id, + mailbox.mailbox_id, Mailbox { has_children, - is_subscribed: mailbox.is_subscribed(access_token.primary_id()), + is_subscribed: mailbox.is_subscribed, special_use: match mailbox.role { SpecialUse::Trash => Some(Attribute::Trash), SpecialUse::Junk => Some(Attribute::Junk), @@ -233,7 +252,7 @@ impl SessionData { account_id, Collection::Email, Property::MailboxIds, - *mailbox_id, + mailbox.mailbox_id, ) .await .caused_by(trc::location!())? @@ -242,7 +261,7 @@ impl SessionData { .into(), total_unseen: self .server - .mailbox_unread_tags(account_id, *mailbox_id, &message_ids) + .mailbox_unread_tags(account_id, mailbox.mailbox_id, &message_ids) .await .caused_by(trc::location!())? .map(|v| v.len()) @@ -253,7 +272,8 @@ impl SessionData { ); let mut mailbox_name = mailbox_path.join("/"); - if mailbox_name.eq_ignore_ascii_case("inbox") && *mailbox_id != INBOX_ID { + if mailbox_name.eq_ignore_ascii_case("inbox") && mailbox.mailbox_id != INBOX_ID + { // If there is another mailbox called Inbox, rename it to avoid conflicts mailbox_name = format!("{mailbox_name} 2"); } @@ -270,7 +290,7 @@ impl SessionData { }) .and_then(|f| special_uses.get(&f.special_use)) .copied() - .unwrap_or(*mailbox_id); + .unwrap_or(mailbox.mailbox_id); account .mailbox_names @@ -278,7 +298,7 @@ impl SessionData { if has_children && iter_stack.len() < 100 { iter_stack.push((iter, parent_id, path)); - parent_id = *mailbox_id + 1; + parent_id = mailbox.mailbox_id + 1; path = mailbox_path; iter = mailboxes.iter(); } @@ -398,7 +418,8 @@ impl SessionData { for (account_id, last_state) in account_states { let changelog = self .server - .changes_( + .store() + .changes( account_id, Collection::Mailbox, last_state.map(Query::Since).unwrap_or(Query::All), @@ -600,14 +621,26 @@ impl SessionData { Ok(access_token.is_member(account_id) || self .server - .get_property::( + .get_property::>( account_id, Collection::Mailbox, document_id, Property::Value, ) - .await? - .map(|mailbox| mailbox.acls.effective_acl(&access_token).contains(item)) + .await + .and_then(|mailbox| { + if let Some(mailbox) = mailbox { + Ok(Some( + mailbox + .unarchive()? + .acls + .effective_acl(&access_token) + .contains(item), + )) + } else { + Ok(None) + } + })? .ok_or_else(|| { trc::ImapEvent::Error .caused_by(trc::location!()) diff --git a/crates/imap/src/core/message.rs b/crates/imap/src/core/message.rs index 0bea7b3e..7a315e8e 100644 --- a/crates/imap/src/core/message.rs +++ b/crates/imap/src/core/message.rs @@ -8,10 +8,10 @@ use std::{collections::BTreeMap, sync::Arc}; use ahash::AHashMap; use common::{NextMailboxState, listener::SessionStream}; -use email::mailbox::UidMailbox; +use email::mailbox::{ArchivedMailbox, UidMailbox}; use imap_proto::protocol::{Sequence, expunge, select::Exists}; use jmap_proto::types::{collection::Collection, property::Property}; -use store::write::assert::HashedValue; +use store::write::{ArchivedValue, assert::HashedValue}; use trc::AddContext; use crate::core::ImapId; @@ -225,7 +225,7 @@ impl SessionData { pub async fn get_uid_validity(&self, mailbox: &MailboxId) -> trc::Result { self.server - .get_property::( + .get_property::>( mailbox.account_id, Collection::Mailbox, mailbox.mailbox_id, @@ -240,7 +240,7 @@ impl SessionData { .collection(Collection::Mailbox) .document_id(mailbox.mailbox_id) }) - .map(|m| m.uid_validity) + .and_then(|m| m.unarchive().map(|m| u32::from(m.uid_validity))) } pub async fn get_uid_next(&self, mailbox: &MailboxId) -> trc::Result { diff --git a/crates/imap/src/op/acl.rs b/crates/imap/src/op/acl.rs index f758ffcf..6325934c 100644 --- a/crates/imap/src/op/acl.rs +++ b/crates/imap/src/op/acl.rs @@ -6,11 +6,15 @@ use std::{sync::Arc, time::Instant}; -use common::{MailboxId, auth::AccessToken, listener::SessionStream}; +use common::{ + MailboxId, auth::AccessToken, listener::SessionStream, sharing::EffectiveAcl, + storage::index::ObjectIndexBuilder, +}; use directory::{ Permission, QueryBy, Type, backend::internal::{PrincipalField, manage::ChangedPrincipals}, }; +use email::mailbox::ArchivedMailbox; use imap_proto::{ Command, ResponseCode, StatusResponse, protocol::acl::{ @@ -19,15 +23,11 @@ use imap_proto::{ receiver::Request, }; -use jmap::auth::acl::EffectiveAcl; -use jmap_proto::{ - object::index::ObjectIndexBuilder, - types::{ - acl::Acl, collection::Collection, property::Property, state::StateChange, - type_state::DataType, value::AclGrant, - }, +use jmap_proto::types::{ + acl::Acl, collection::Collection, property::Property, state::StateChange, type_state::DataType, + value::AclGrant, }; -use store::write::{BatchBuilder, assert::HashedValue, log::ChangeLogBuilder}; +use store::write::{ArchivedValue, BatchBuilder, assert::HashedValue, log::ChangeLogBuilder}; use trc::AddContext; use utils::map::bitmap::Bitmap; @@ -48,26 +48,29 @@ impl Session { let data = self.state.session_data(); spawn_op!(data, { - let (mailbox_id, mailbox, _) = data + let (mailbox_id, mailbox_, _) = data .get_acl_mailbox(&arguments, true) .await .imap_ctx(&arguments.tag, trc::location!())?; let mut permissions = Vec::new(); + let mailbox = mailbox_ + .to_unarchived() + .imap_ctx(&arguments.tag, trc::location!())?; - for item in mailbox.inner.acls { + for item in mailbox.inner.acls.iter() { if let Some(account_name) = data .server .core .storage .directory - .query(QueryBy::Id(item.account_id), false) + .query(QueryBy::Id(item.account_id.into()), false) .await .imap_ctx(&arguments.tag, trc::location!())? .and_then(|mut p| p.take_str(PrincipalField::Name)) { let mut rights = Vec::new(); - for acl in item.grants { + for acl in Bitmap::from(&item.grants) { match acl { Acl::Read => { rights.push(Rights::Lookup); @@ -101,7 +104,7 @@ impl Session { Acl::Submit => { rights.push(Rights::Post); } - Acl::None => (), + _ => (), } } @@ -144,12 +147,15 @@ impl Session { let is_rev2 = self.version.is_rev2(); spawn_op!(data, { - let (mailbox, values, access_token) = data + let (mailbox_id, mailbox_, access_token) = data .get_acl_mailbox(&arguments, false) .await .imap_ctx(&arguments.tag, trc::location!())?; - let rights = if access_token.is_shared(mailbox.account_id) { - let acl = values.inner.acls.effective_acl(&access_token); + let mailbox = mailbox_ + .to_unarchived() + .imap_ctx(&arguments.tag, trc::location!())?; + let rights = if access_token.is_shared(mailbox_id.account_id) { + let acl = mailbox.inner.acls.effective_acl(&access_token); let mut rights = Vec::with_capacity(5); if acl.contains(Acl::ReadItems) { rights.push(Rights::Read); @@ -195,8 +201,8 @@ impl Session { Imap(trc::ImapEvent::MyRights), SpanId = data.session_id, MailboxName = arguments.mailbox_name.clone(), - AccountId = mailbox.account_id, - MailboxId = mailbox.mailbox_id, + AccountId = mailbox_id.account_id, + MailboxId = mailbox_id.mailbox_id, Details = rights .iter() .map(|r| trc::Value::String(r.to_string())) @@ -234,6 +240,9 @@ impl Session { .get_acl_mailbox(&arguments, false) .await .imap_ctx(&arguments.tag, trc::location!())?; + let current_mailbox = current_mailbox + .into_deserialized() + .imap_ctx(&arguments.tag, trc::location!())?; // Obtain principal id let acl_account_id = data @@ -322,7 +331,8 @@ impl Session { ObjectIndexBuilder::new() .with_changes(mailbox) .with_current(current_mailbox), - ); + ) + .imap_ctx(&arguments.tag, trc::location!())?; if !batch.is_empty() { data.server .store() @@ -428,13 +438,13 @@ impl SessionData { validate: bool, ) -> trc::Result<( MailboxId, - HashedValue, + HashedValue>, Arc, )> { if let Some(mailbox) = self.get_mailbox_by_name(&arguments.mailbox_name) { if let Some(values) = self .server - .get_property::>( + .get_property::>>( mailbox.account_id, Collection::Mailbox, mailbox.mailbox_id, @@ -448,6 +458,8 @@ impl SessionData { || access_token.is_member(mailbox.account_id) || values .inner + .unarchive() + .caused_by(trc::location!())? .acls .effective_acl(&access_token) .contains(Acl::Administer) diff --git a/crates/imap/src/op/copy_move.rs b/crates/imap/src/op/copy_move.rs index 262bfc56..b0b56c71 100644 --- a/crates/imap/src/op/copy_move.rs +++ b/crates/imap/src/op/copy_move.rs @@ -9,7 +9,7 @@ use std::{sync::Arc, time::Instant}; use directory::Permission; use email::{ mailbox::{JUNK_ID, UidMailbox}, - message::{bayes::EmailBayesTrain, ingest::EmailIngest}, + message::{bayes::EmailBayesTrain, copy::EmailCopy, ingest::EmailIngest}, }; use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, protocol::copy_move::Arguments, @@ -20,8 +20,7 @@ use crate::{ core::{SelectedMailbox, Session, SessionData}, spawn_op, }; -use common::{MailboxId, listener::SessionStream}; -use jmap::email::{copy::EmailCopy, set::TagManager}; +use common::{MailboxId, listener::SessionStream, storage::tag::TagManager}; use jmap_proto::{ error::set::SetErrorType, types::{ @@ -30,8 +29,9 @@ use jmap_proto::{ }, }; use store::{ + SerializeInfallible, roaring::RoaringBitmap, - write::{BatchBuilder, F_VALUE, ValueClass, assert::HashedValue, log::ChangeLogBuilder}, + write::{BatchBuilder, ValueClass, assert::HashedValue, log::ChangeLogBuilder}, }; use super::ImapContext; @@ -233,14 +233,16 @@ impl SessionData { .with_account_id(account_id) .with_collection(Collection::Email) .update_document(id); - mailboxes.update_batch(&mut batch, Property::MailboxIds); + mailboxes + .update_batch(&mut batch, Property::MailboxIds) + .imap_ctx(&arguments.tag, trc::location!())?; if changelog.change_id == u64::MAX { changelog.change_id = self .server .assign_change_id(account_id) .imap_ctx(&arguments.tag, trc::location!())?; } - batch.value(Property::Cid, changelog.change_id, F_VALUE); + batch.set(Property::Cid, changelog.change_id.serialize()); // Add bayes train task if can_spam_train { diff --git a/crates/imap/src/op/create.rs b/crates/imap/src/op/create.rs index ba2e7d52..c9a570c2 100644 --- a/crates/imap/src/op/create.rs +++ b/crates/imap/src/op/create.rs @@ -11,20 +11,19 @@ use crate::{ op::ImapContext, spawn_op, }; -use common::{Account, Mailbox, config::jmap::settings::SpecialUse, listener::SessionStream}; +use common::{ + Account, Mailbox, config::jmap::settings::SpecialUse, listener::SessionStream, + storage::index::ObjectIndexBuilder, +}; use directory::Permission; use imap_proto::{ Command, ResponseCode, StatusResponse, protocol::{create::Arguments, list::Attribute}, receiver::Request, }; -use jmap::JmapMethods; -use jmap_proto::{ - object::index::ObjectIndexBuilder, - types::{ - acl::Acl, collection::Collection, id::Id, property::Property, state::StateChange, - type_state::DataType, - }, +use jmap_proto::types::{ + acl::Acl, collection::Collection, id::Id, property::Property, state::StateChange, + type_state::DataType, }; use store::{query::Filter, write::BatchBuilder}; use trc::AddContext; @@ -94,7 +93,8 @@ impl SessionData { .with_account_id(params.account_id) .with_collection(Collection::Mailbox) .create_document() - .custom(ObjectIndexBuilder::new().with_changes(mailbox)); + .custom(ObjectIndexBuilder::new().with_changes(mailbox)) + .imap_ctx(&arguments.tag, trc::location!())?; let mailbox_id = self .server .store() @@ -112,7 +112,8 @@ impl SessionData { batch .with_account_id(params.account_id) .with_collection(Collection::Mailbox) - .custom(changes); + .custom(changes) + .imap_ctx(&arguments.tag, trc::location!())?; self.server .store() .write(batch) @@ -389,10 +390,11 @@ impl SessionData { let role_name = attr_to_role(mailbox_role).as_str().unwrap_or_default(); if !self .server + .store() .filter( account_id, Collection::Mailbox, - vec![Filter::eq(Property::Role, role_name)], + vec![Filter::eq(Property::Role, role_name.as_bytes().to_vec())], ) .await .caused_by(trc::location!())? diff --git a/crates/imap/src/op/delete.rs b/crates/imap/src/op/delete.rs index f208d9a7..5b92e471 100644 --- a/crates/imap/src/op/delete.rs +++ b/crates/imap/src/op/delete.rs @@ -12,10 +12,10 @@ use crate::{ }; use common::listener::SessionStream; use directory::Permission; +use email::mailbox::destroy::MailboxDestroy; use imap_proto::{ - protocol::delete::Arguments, receiver::Request, Command, ResponseCode, StatusResponse, + Command, ResponseCode, StatusResponse, protocol::delete::Arguments, receiver::Request, }; -use jmap::mailbox::set::MailboxSet; use jmap_proto::types::{state::StateChange, type_state::DataType}; use store::write::log::ChangeLogBuilder; diff --git a/crates/imap/src/op/expunge.rs b/crates/imap/src/op/expunge.rs index 8c8817fc..904091ae 100644 --- a/crates/imap/src/op/expunge.rs +++ b/crates/imap/src/op/expunge.rs @@ -17,15 +17,15 @@ use imap_proto::{ use trc::AddContext; use crate::core::{SavedSearch, SelectedMailbox, Session, SessionData}; -use common::{ImapId, listener::SessionStream}; -use jmap::email::set::TagManager; +use common::{ImapId, listener::SessionStream, storage::tag::TagManager}; use jmap_proto::types::{ acl::Acl, collection::Collection, id::Id, keyword::Keyword, property::Property, state::StateChange, type_state::DataType, }; use store::{ + SerializeInfallible, roaring::RoaringBitmap, - write::{BatchBuilder, F_VALUE, assert::HashedValue, log::ChangeLogBuilder}, + write::{BatchBuilder, assert::HashedValue, log::ChangeLogBuilder}, }; use super::{ImapContext, ToModSeq}; @@ -243,12 +243,16 @@ impl SessionData { .with_account_id(account_id) .with_collection(Collection::Email) .update_document(id); - mailboxes.update_batch(&mut batch, Property::MailboxIds); - keywords.update_batch(&mut batch, Property::Keywords); + mailboxes + .update_batch(&mut batch, Property::MailboxIds) + .caused_by(trc::location!())?; + keywords + .update_batch(&mut batch, Property::Keywords) + .caused_by(trc::location!())?; if changelog.change_id == u64::MAX { changelog.change_id = self.server.assign_change_id(account_id)? } - batch.value(Property::Cid, changelog.change_id, F_VALUE); + batch.set(Property::Cid, changelog.change_id.serialize()); match self .server .store() diff --git a/crates/imap/src/op/fetch.rs b/crates/imap/src/op/fetch.rs index ae402724..e49a9f2f 100644 --- a/crates/imap/src/op/fetch.rs +++ b/crates/imap/src/op/fetch.rs @@ -27,15 +27,15 @@ use imap_proto::{ }, receiver::Request, }; -use jmap::{blob::download::BlobDownload, changes::get::ChangesLookup}; use jmap_proto::types::{ acl::Acl, collection::Collection, id::Id, keyword::Keyword, property::Property, state::StateChange, type_state::DataType, }; use mail_parser::{Address, GetHeader, HeaderName, Message, PartType}; use store::{ + Serialize, SerializeInfallible, query::log::{Change, Query}, - write::{BatchBuilder, Bincode, F_BITMAP, F_VALUE, assert::HashedValue}, + write::{BatchBuilder, Bincode, assert::HashedValue}, }; use trc::AddContext; @@ -148,7 +148,8 @@ impl SessionData { // Obtain changes since the modseq. let changelog = self .server - .changes_( + .store() + .changes( account_id, Collection::Email, Query::from_modseq(changed_since), @@ -337,7 +338,8 @@ impl SessionData { // Retrieve raw message if needed match self .server - .get_blob(&email.blob_hash, 0..usize::MAX) + .blob_store() + .get_blob(email.blob_hash.as_slice(), 0..usize::MAX) .await .imap_ctx(&arguments.tag, trc::location!())? { @@ -555,9 +557,12 @@ impl SessionData { .with_collection(Collection::Email) .update_document(id.document_id()) .assert_value(Property::Keywords, &keywords) - .value(Property::Keywords, keywords.inner, F_VALUE) - .value(Property::Keywords, Keyword::Seen, F_BITMAP) - .value(Property::Cid, changelog.change_id, F_VALUE); + .set( + Property::Keywords, + keywords.inner.serialize().caused_by(trc::location!())?, + ) + .tag(Property::Keywords, Keyword::Seen) + .set(Property::Cid, changelog.change_id.serialize()); match self .server .store() diff --git a/crates/imap/src/op/idle.rs b/crates/imap/src/op/idle.rs index c395bf21..d206a0a8 100644 --- a/crates/imap/src/op/idle.rs +++ b/crates/imap/src/op/idle.rs @@ -9,18 +9,16 @@ use std::{sync::Arc, time::Instant}; use ahash::AHashSet; use directory::Permission; use imap_proto::{ + Command, StatusResponse, protocol::{ - fetch, + Sequence, fetch, list::{Attribute, ListItem}, status::Status, - Sequence, }, receiver::Request, - Command, StatusResponse, }; use common::listener::SessionStream; -use jmap::{changes::get::ChangesLookup, services::state::StateManager}; use jmap_proto::types::{collection::Collection, type_state::DataType}; use store::query::log::Query; use tokio::io::AsyncReadExt; @@ -204,7 +202,8 @@ impl SessionData { // Obtain changed messages let changelog = self .server - .changes_( + .store() + .changes( mailbox.id.account_id, Collection::Email, modseq.map(Query::Since).unwrap_or(Query::All), diff --git a/crates/imap/src/op/rename.rs b/crates/imap/src/op/rename.rs index e4aeb25c..eeda162e 100644 --- a/crates/imap/src/op/rename.rs +++ b/crates/imap/src/op/rename.rs @@ -10,20 +10,16 @@ use crate::{ core::{Session, SessionData}, spawn_op, }; -use common::listener::SessionStream; +use common::{listener::SessionStream, sharing::EffectiveAcl, storage::index::ObjectIndexBuilder}; use directory::Permission; +use email::mailbox::ArchivedMailbox; use imap_proto::{ Command, ResponseCode, StatusResponse, protocol::rename::Arguments, receiver::Request, }; -use jmap::auth::acl::EffectiveAcl; -use jmap_proto::{ - object::index::ObjectIndexBuilder, - types::{ - acl::Acl, collection::Collection, property::Property, state::StateChange, - type_state::DataType, - }, +use jmap_proto::types::{ + acl::Acl, collection::Collection, property::Property, state::StateChange, type_state::DataType, }; -use store::write::{BatchBuilder, assert::HashedValue}; +use store::write::{ArchivedValue, BatchBuilder, assert::HashedValue}; use trc::AddContext; use super::ImapContext; @@ -93,7 +89,7 @@ impl SessionData { // Obtain mailbox let mailbox = self .server - .get_property::>( + .get_property::>>( params.account_id, Collection::Mailbox, mailbox_id, @@ -108,7 +104,9 @@ impl SessionData { .caused_by(trc::location!()) .code(ResponseCode::NonExistent) .id(arguments.tag.clone()) - })?; + })? + .into_deserialized() + .imap_ctx(&arguments.tag, trc::location!())?; // Validate ACL let access_token = self @@ -148,7 +146,8 @@ impl SessionData { .create_document() .custom(ObjectIndexBuilder::new().with_changes( email::mailbox::Mailbox::new(path_item).with_parent_id(parent_id), - )); + )) + .imap_ctx(&arguments.tag, trc::location!())?; let mailbox_id = self .server @@ -175,11 +174,14 @@ impl SessionData { ObjectIndexBuilder::new() .with_current(mailbox) .with_changes(new_mailbox), - ); + ) + .imap_ctx(&arguments.tag, trc::location!())?; changes.log_update(Collection::Mailbox, mailbox_id); let change_id = changes.change_id; - batch.custom(changes); + batch + .custom(changes) + .imap_ctx(&arguments.tag, trc::location!())?; self.server .store() .write(batch) diff --git a/crates/imap/src/op/search.rs b/crates/imap/src/op/search.rs index bf8670ac..b88394ba 100644 --- a/crates/imap/src/op/search.rs +++ b/crates/imap/src/op/search.rs @@ -6,23 +6,23 @@ use std::{sync::Arc, time::Instant}; -use common::{listener::SessionStream, ImapId}; +use common::{ImapId, listener::SessionStream}; use directory::Permission; use imap_proto::{ + Command, StatusResponse, protocol::{ - search::{self, Arguments, Filter, Response, ResultOption}, Sequence, + search::{self, Arguments, Filter, Response, ResultOption}, }, receiver::Request, - Command, StatusResponse, }; -use jmap::{changes::get::ChangesLookup, JmapMethods}; use jmap_proto::types::{collection::Collection, id::Id, keyword::Keyword, property::Property}; use mail_parser::HeaderName; use nlp::language::Language; use store::{ + SerializeInfallible, fts::{Field, FilterGroup, FtsFilter, IntoFilterGroup}, - query::{self, log::Query, sort::Pagination, ResultSet}, + query::{self, ResultSet, log::Query, sort::Pagination}, roaring::RoaringBitmap, write::now, }; @@ -418,7 +418,8 @@ impl SessionData { filters.push(query::Filter::is_in_set( self.server - .fts_filter(mailbox.id.account_id, Collection::Email, fts_filters) + .fts_store() + .query(mailbox.id.account_id, Collection::Email, fts_filters) .await?, )); } @@ -457,7 +458,10 @@ impl SessionData { )); } search::Filter::Before(date) => { - filters.push(query::Filter::lt(Property::ReceivedAt, date as u64)); + filters.push(query::Filter::lt( + Property::ReceivedAt, + (date as u64).serialize(), + )); } search::Filter::Deleted => { filters.push(query::Filter::is_in_bitmap( @@ -484,14 +488,17 @@ impl SessionData { )); } search::Filter::Larger(size) => { - filters.push(query::Filter::gt(Property::Size, size)); + filters.push(query::Filter::gt(Property::Size, size.serialize())); } search::Filter::On(date) => { filters.push(query::Filter::And); - filters.push(query::Filter::ge(Property::ReceivedAt, date as u64)); + filters.push(query::Filter::ge( + Property::ReceivedAt, + (date as u64).serialize(), + )); filters.push(query::Filter::lt( Property::ReceivedAt, - (date + 86400) as u64, + ((date + 86400) as u64).serialize(), )); filters.push(query::Filter::End); } @@ -502,22 +509,37 @@ impl SessionData { )); } search::Filter::SentBefore(date) => { - filters.push(query::Filter::lt(Property::SentAt, date as u64)); + filters.push(query::Filter::lt( + Property::SentAt, + (date as u64).serialize(), + )); } search::Filter::SentOn(date) => { filters.push(query::Filter::And); - filters.push(query::Filter::ge(Property::SentAt, date as u64)); - filters.push(query::Filter::lt(Property::SentAt, (date + 86400) as u64)); + filters.push(query::Filter::ge( + Property::SentAt, + (date as u64).serialize(), + )); + filters.push(query::Filter::lt( + Property::SentAt, + ((date + 86400) as u64).serialize(), + )); filters.push(query::Filter::End); } search::Filter::SentSince(date) => { - filters.push(query::Filter::ge(Property::SentAt, date as u64)); + filters.push(query::Filter::ge( + Property::SentAt, + (date as u64).serialize(), + )); } search::Filter::Since(date) => { - filters.push(query::Filter::ge(Property::ReceivedAt, date as u64)); + filters.push(query::Filter::ge( + Property::ReceivedAt, + (date as u64).serialize(), + )); } search::Filter::Smaller(size) => { - filters.push(query::Filter::lt(Property::Size, size)); + filters.push(query::Filter::lt(Property::Size, size.serialize())); } search::Filter::Unanswered => { filters.push(query::Filter::Not); @@ -601,20 +623,21 @@ impl SessionData { search::Filter::Older(secs) => { filters.push(query::Filter::le( Property::ReceivedAt, - now().saturating_sub(secs as u64), + now().saturating_sub(secs as u64).serialize(), )); } search::Filter::Younger(secs) => { filters.push(query::Filter::ge( Property::ReceivedAt, - now().saturating_sub(secs as u64), + now().saturating_sub(secs as u64).serialize(), )); } search::Filter::ModSeq((modseq, _)) => { let mut set = RoaringBitmap::new(); for change in self .server - .changes_( + .store() + .changes( mailbox.id.account_id, Collection::Email, Query::from_modseq(modseq), @@ -660,6 +683,7 @@ impl SessionData { // Run query self.server + .store() .filter(mailbox.id.account_id, Collection::Email, filters) .await .map(|res| (res, include_highest_modseq)) diff --git a/crates/imap/src/op/status.rs b/crates/imap/src/op/status.rs index feb05533..5b8492dc 100644 --- a/crates/imap/src/op/status.rs +++ b/crates/imap/src/op/status.rs @@ -13,6 +13,7 @@ use crate::{ }; use common::{Mailbox, listener::SessionStream}; use directory::Permission; +use email::mailbox::ArchivedMailbox; use imap_proto::{ Command, ResponseCode, StatusResponse, parser::PushUnique, @@ -20,7 +21,7 @@ use imap_proto::{ receiver::Request, }; use jmap_proto::types::{collection::Collection, id::Id, keyword::Keyword, property::Property}; -use store::{Deserialize, U32_LEN}; +use store::{Deserialize, U32_LEN, write::ArchivedValue}; use store::{ IndexKeyPrefix, IterateParams, ValueKey, roaring::RoaringBitmap, @@ -248,50 +249,13 @@ impl SessionData { for item in items_update { let result = match item { Status::Messages => mailbox_message_ids.as_ref().map(|v| v.len()).unwrap_or(0), -<<<<<<< HEAD Status::UidNext => self .get_uid_next(&mailbox) .await .caused_by(trc::location!())? as u64, - Status::UidValidity => self - .server - .get_property::>( - mailbox.account_id, - Collection::Mailbox, - mailbox.mailbox_id, - &Property::Value, - ) - .await? - .and_then(|obj| obj.get(&Property::Cid).as_uint()) - .ok_or_else(|| { - trc::StoreEvent::UnexpectedError - .into_err() - .details("Mailbox unavailable") - .ctx(trc::Key::Reason, "Failed to obtain uid validity") - .caused_by(trc::location!()) - .account_id(mailbox.account_id) - .document_id(mailbox.mailbox_id) - })?, -======= - Status::UidNext => { - (self - .server - .core - .storage - .data - .get_counter(ValueKey { - account_id: mailbox.account_id, - collection: Collection::Mailbox.into(), - document_id: mailbox.mailbox_id, - class: ValueClass::Property(Property::EmailIds.into()), - }) - .await - .caused_by(trc::location!())? - + 1) as u64 - } - Status::UidValidity => { + Status::UidValidity => u32::from( self.server - .get_property::( + .get_property::>( mailbox.account_id, Collection::Mailbox, mailbox.mailbox_id, @@ -307,9 +271,10 @@ impl SessionData { .account_id(mailbox.account_id) .document_id(mailbox.mailbox_id) })? - .uid_validity as u64 - } ->>>>>>> b34a8804 (Improved object serialization) + .unarchive() + .caused_by(trc::location!())? + .uid_validity, + ) as u64, Status::Unseen => { if let (Some(message_ids), Some(mailbox_message_ids)) = (&message_ids, &mailbox_message_ids) diff --git a/crates/imap/src/op/store.rs b/crates/imap/src/op/store.rs index f16ccbba..5ccfc506 100644 --- a/crates/imap/src/op/store.rs +++ b/crates/imap/src/op/store.rs @@ -11,7 +11,7 @@ use crate::{ spawn_op, }; use ahash::AHashSet; -use common::listener::SessionStream; +use common::{listener::SessionStream, storage::tag::TagManager}; use directory::Permission; use email::{ mailbox::UidMailbox, @@ -26,14 +26,14 @@ use imap_proto::{ }, receiver::Request, }; -use jmap::{changes::get::ChangesLookup, email::set::TagManager}; use jmap_proto::types::{ acl::Acl, collection::Collection, id::Id, keyword::Keyword, property::Property, state::StateChange, type_state::DataType, }; use store::{ + SerializeInfallible, query::log::{Change, Query}, - write::{BatchBuilder, F_VALUE, ValueClass, assert::HashedValue, log::ChangeLogBuilder}, + write::{BatchBuilder, ValueClass, assert::HashedValue, log::ChangeLogBuilder}, }; use trc::AddContext; @@ -116,7 +116,8 @@ impl SessionData { // Obtain changes since the modseq. let changelog = self .server - .changes_( + .store() + .changes( account_id, Collection::Email, Query::from_modseq(unchanged_since), @@ -285,14 +286,16 @@ impl SessionData { .with_account_id(account_id) .with_collection(Collection::Email) .update_document(*id); - keywords.update_batch(&mut batch, Property::Keywords); + keywords + .update_batch(&mut batch, Property::Keywords) + .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?; if changelog.change_id == u64::MAX { changelog.change_id = self .server .assign_change_id(account_id) .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())? } - batch.value(Property::Cid, changelog.change_id, F_VALUE); + batch.set(Property::Cid, changelog.change_id.serialize()); // Add spam train task if let Some(learn_spam) = train_spam { diff --git a/crates/imap/src/op/subscribe.rs b/crates/imap/src/op/subscribe.rs index d91f0f93..e03c7127 100644 --- a/crates/imap/src/op/subscribe.rs +++ b/crates/imap/src/op/subscribe.rs @@ -10,14 +10,14 @@ use crate::{ core::{Session, SessionData}, spawn_op, }; -use common::listener::SessionStream; +use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; +use email::mailbox::ArchivedMailbox; use imap_proto::{Command, ResponseCode, StatusResponse, receiver::Request}; -use jmap_proto::{ - object::index::ObjectIndexBuilder, - types::{collection::Collection, property::Property, state::StateChange, type_state::DataType}, +use jmap_proto::types::{ + collection::Collection, property::Property, state::StateChange, type_state::DataType, }; -use store::write::{BatchBuilder, assert::HashedValue}; +use store::write::{ArchivedValue, BatchBuilder, assert::HashedValue}; use super::ImapContext; @@ -97,7 +97,7 @@ impl SessionData { // Obtain mailbox let mailbox = self .server - .get_property::>( + .get_property::>>( account_id, Collection::Mailbox, mailbox_id, @@ -112,9 +112,10 @@ impl SessionData { .code(ResponseCode::NonExistent) .id(tag.clone()) .caused_by(trc::location!()) - })?; + })? + .into_deserialized() + .imap_ctx(&tag, trc::location!())?; - // Subscribe/unsubscribe to mailbox if (subscribe && !mailbox.inner.is_subscribed(self.account_id)) || (!subscribe && mailbox.inner.is_subscribed(self.account_id)) { @@ -138,11 +139,12 @@ impl SessionData { ObjectIndexBuilder::new() .with_current(mailbox) .with_changes(new_mailbox), - ); + ) + .imap_ctx(&tag, trc::location!())?; changes.log_update(Collection::Mailbox, mailbox_id); let change_id = changes.change_id; - batch.custom(changes); + batch.custom(changes).imap_ctx(&tag, trc::location!())?; self.server .store() .write(batch) diff --git a/crates/jmap-proto/Cargo.toml b/crates/jmap-proto/Cargo.toml index f8cddeee..01951df5 100644 --- a/crates/jmap-proto/Cargo.toml +++ b/crates/jmap-proto/Cargo.toml @@ -13,7 +13,8 @@ fast-float = "0.2.0" serde = { version = "1.0", features = ["derive"]} ahash = { version = "0.8.2", features = ["serde"] } serde_json = { version = "1.0", features = ["raw_value"] } - +hashify = "0.2" +rkyv = { version = "0.8.10", features = ["little_endian"] } [dev-dependencies] tokio = { version = "1.23", features = ["full"] } diff --git a/crates/jmap-proto/src/object/mod.rs b/crates/jmap-proto/src/object/mod.rs index a899ce17..99c16b56 100644 --- a/crates/jmap-proto/src/object/mod.rs +++ b/crates/jmap-proto/src/object/mod.rs @@ -20,7 +20,6 @@ use crate::types::{ pub mod blob; pub mod email; pub mod email_submission; -pub mod index; pub mod mailbox; pub mod sieve; diff --git a/crates/jmap-proto/src/types/acl.rs b/crates/jmap-proto/src/types/acl.rs index f6a7efcd..28144215 100644 --- a/crates/jmap-proto/src/types/acl.rs +++ b/crates/jmap-proto/src/types/acl.rs @@ -8,9 +8,22 @@ use std::fmt::{self, Display}; use utils::map::bitmap::BitmapItem; -use crate::parser::{json::Parser, JsonObjectParser}; +use crate::parser::{JsonObjectParser, json::Parser}; -#[derive(Debug, Eq, PartialEq, PartialOrd, Ord, Hash, Clone, Copy)] +#[derive( + rkyv::Archive, + rkyv::Deserialize, + rkyv::Serialize, + Debug, + Clone, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Copy, +)] +#[rkyv(compare(PartialEq), derive(Debug))] #[repr(u8)] pub enum Acl { Read = 0, @@ -23,7 +36,12 @@ pub enum Acl { CreateChild = 7, Administer = 8, Submit = 9, - None = 10, + ReadFreeBusy = 10, + ModifyItemsOwn = 11, + ModifyPrivateProperties = 12, + RSVP = 13, + Share = 14, + None = 15, } impl JsonObjectParser for Acl { @@ -72,6 +90,11 @@ impl Acl { Acl::CreateChild => "createChild", Acl::Administer => "administer", Acl::Submit => "submit", + Acl::ReadFreeBusy => "readFreeBusy", + Acl::ModifyItemsOwn => "modifyItemsOwn", + Acl::ModifyPrivateProperties => "modifyPrivateProperties", + Acl::RSVP => "rsvp", + Acl::Share => "share", Acl::None => "", } } @@ -125,27 +148,3 @@ impl From for Acl { } } } - -/*impl SerializeInto for Acl { - fn serialize_into(&self, buf: &mut Vec) { - buf.push(*self as u8); - } -} - -impl DeserializeFrom for Acl { - fn deserialize_from(bytes: &mut std::slice::Iter<'_, u8>) -> Option { - match *bytes.next()? { - 0 => Some(Acl::Read), - 1 => Some(Acl::Modify), - 2 => Some(Acl::Delete), - 3 => Some(Acl::ReadItems), - 4 => Some(Acl::AddItems), - 5 => Some(Acl::ModifyItems), - 6 => Some(Acl::RemoveItems), - 7 => Some(Acl::CreateChild), - 8 => Some(Acl::Administer), - 9 => Some(Acl::Submit), - _ => None, - } - } -}*/ diff --git a/crates/jmap-proto/src/types/collection.rs b/crates/jmap-proto/src/types/collection.rs index 0fbd6af0..49261c08 100644 --- a/crates/jmap-proto/src/types/collection.rs +++ b/crates/jmap-proto/src/types/collection.rs @@ -11,7 +11,7 @@ use std::{ use utils::map::bitmap::BitmapItem; -use super::type_state::DataType; +use super::{property::Property, type_state::DataType}; #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] #[repr(u8)] @@ -24,7 +24,35 @@ pub enum Collection { SieveScript = 5, PushSubscription = 6, Principal = 7, - None = 8, + Calendar = 8, + CalendarEvent = 9, + CalendarEventNotification = 10, + AddressBook = 11, + ContactCard = 12, + FileNode = 13, + None = 14, +} + +impl Collection { + pub fn child_collection(&self) -> Option { + match self { + Collection::Mailbox => Some(Collection::Email), + Collection::Calendar => Some(Collection::CalendarEvent), + Collection::AddressBook => Some(Collection::ContactCard), + Collection::FileNode => Some(Collection::FileNode), + _ => None, + } + } + + pub fn parent_property(&self) -> Option { + match self { + Collection::Email => Some(Property::MailboxIds), + Collection::CalendarEvent => Some(Property::ParentId), + Collection::ContactCard => Some(Property::ParentId), + Collection::FileNode => Some(Property::ParentId), + _ => None, + } + } } impl From for Collection { @@ -38,6 +66,12 @@ impl From for Collection { 5 => Collection::SieveScript, 6 => Collection::PushSubscription, 7 => Collection::Principal, + 8 => Collection::Calendar, + 9 => Collection::CalendarEvent, + 10 => Collection::CalendarEventNotification, + 11 => Collection::AddressBook, + 12 => Collection::ContactCard, + 13 => Collection::FileNode, _ => Collection::None, } } @@ -54,6 +88,12 @@ impl From for Collection { 5 => Collection::SieveScript, 6 => Collection::PushSubscription, 7 => Collection::Principal, + 8 => Collection::Calendar, + 9 => Collection::CalendarEvent, + 10 => Collection::CalendarEventNotification, + 11 => Collection::AddressBook, + 12 => Collection::ContactCard, + 13 => Collection::FileNode, _ => Collection::None, } } @@ -105,6 +145,12 @@ impl Collection { Collection::EmailSubmission => "emailSubmission", Collection::SieveScript => "sieveScript", Collection::Principal => "principal", + Collection::Calendar => "calendar", + Collection::CalendarEvent => "calendarEvent", + Collection::CalendarEventNotification => "calendarEventNotification", + Collection::AddressBook => "addressBook", + Collection::ContactCard => "contactCard", + Collection::FileNode => "fileNode", Collection::None => "", } } @@ -114,17 +160,23 @@ impl FromStr for Collection { type Err = (); fn from_str(s: &str) -> Result { - match s { - "pushSubscription" => Ok(Collection::PushSubscription), - "email" => Ok(Collection::Email), - "mailbox" => Ok(Collection::Mailbox), - "thread" => Ok(Collection::Thread), - "identity" => Ok(Collection::Identity), - "emailSubmission" => Ok(Collection::EmailSubmission), - "sieveScript" => Ok(Collection::SieveScript), - "principal" => Ok(Collection::Principal), - _ => Err(()), - } + hashify::tiny_map!(s.as_bytes(), + "pushSubscription" => Collection::PushSubscription, + "email" => Collection::Email, + "mailbox" => Collection::Mailbox, + "thread" => Collection::Thread, + "identity" => Collection::Identity, + "emailSubmission" => Collection::EmailSubmission, + "sieveScript" => Collection::SieveScript, + "principal" => Collection::Principal, + "calendar" => Collection::Calendar, + "calendarEvent" => Collection::CalendarEvent, + "calendarEventNotification" => Collection::CalendarEventNotification, + "addressBook" => Collection::AddressBook, + "contactCard" => Collection::ContactCard, + "fileNode" => Collection::FileNode, + ) + .ok_or(()) } } diff --git a/crates/jmap-proto/src/types/date.rs b/crates/jmap-proto/src/types/date.rs index 81345674..9c1dd365 100644 --- a/crates/jmap-proto/src/types/date.rs +++ b/crates/jmap-proto/src/types/date.rs @@ -6,7 +6,7 @@ use std::fmt::Display; -use store::Serialize; +use store::SerializeInfallible; use crate::parser::{JsonObjectParser, json::Parser}; @@ -220,8 +220,8 @@ impl serde::Serialize for UTCDate { } } -impl Serialize for UTCDate { - fn serialize(self) -> Vec { +impl SerializeInfallible for UTCDate { + fn serialize(&self) -> Vec { (self.timestamp() as u64).serialize() } } diff --git a/crates/jmap-proto/src/types/keyword.rs b/crates/jmap-proto/src/types/keyword.rs index 61cec658..8ca997c2 100644 --- a/crates/jmap-proto/src/types/keyword.rs +++ b/crates/jmap-proto/src/types/keyword.rs @@ -7,14 +7,12 @@ use std::fmt::Display; use store::{ - write::{ - BitmapClass, DeserializeFrom, MaybeDynamicId, Operation, SerializeInto, TagValue, ToBitmaps, - }, Serialize, + write::{DeserializeFrom, MaybeDynamicId, SerializeInto, TagValue}, }; use utils::codec::leb128::{Leb128Iterator, Leb128Vec}; -use crate::parser::{json::Parser, JsonObjectParser}; +use crate::parser::{JsonObjectParser, json::Parser}; pub const SEEN: usize = 0; pub const DRAFT: usize = 1; @@ -170,41 +168,9 @@ impl Display for Keyword { } } -impl ToBitmaps for Keyword { - fn to_bitmaps(&self, ops: &mut Vec, field: u8, set: bool) { - ops.push(Operation::Bitmap { - class: BitmapClass::Tag { - field, - value: self.into(), - }, - set, - }); - } -} - impl Serialize for Keyword { - fn serialize(self) -> Vec { - match self { - Keyword::Seen => vec![SEEN as u8], - Keyword::Draft => vec![DRAFT as u8], - Keyword::Flagged => vec![FLAGGED as u8], - Keyword::Answered => vec![ANSWERED as u8], - Keyword::Recent => vec![RECENT as u8], - Keyword::Important => vec![IMPORTANT as u8], - Keyword::Phishing => vec![PHISHING as u8], - Keyword::Junk => vec![JUNK as u8], - Keyword::NotJunk => vec![NOTJUNK as u8], - Keyword::Deleted => vec![DELETED as u8], - Keyword::Forwarded => vec![FORWARDED as u8], - Keyword::MdnSent => vec![MDN_SENT as u8], - Keyword::Other(string) => string.into_bytes(), - } - } -} - -impl Serialize for &Keyword { - fn serialize(self) -> Vec { - match self { + fn serialize(&self) -> trc::Result> { + Ok(match self { Keyword::Seen => vec![SEEN as u8], Keyword::Draft => vec![DRAFT as u8], Keyword::Flagged => vec![FLAGGED as u8], @@ -218,7 +184,7 @@ impl Serialize for &Keyword { Keyword::Forwarded => vec![FORWARDED as u8], Keyword::MdnSent => vec![MDN_SENT as u8], Keyword::Other(string) => string.as_bytes().to_vec(), - } + }) } } diff --git a/crates/jmap-proto/src/types/property.rs b/crates/jmap-proto/src/types/property.rs index 99a05fda..ad32b329 100644 --- a/crates/jmap-proto/src/types/property.rs +++ b/crates/jmap-proto/src/types/property.rs @@ -8,7 +8,7 @@ use std::fmt::{Display, Formatter}; use mail_parser::HeaderName; use serde::Serialize; -use store::write::{DeserializeFrom, SerializeInto}; +use store::write::{DeserializeFrom, SerializeInto, ValueClass}; use crate::parser::{JsonObjectParser, json::Parser}; @@ -1459,3 +1459,9 @@ impl AsRef for Property { self } } + +impl From for ValueClass { + fn from(value: Property) -> Self { + ValueClass::Property(value.into()) + } +} diff --git a/crates/jmap-proto/src/types/type_state.rs b/crates/jmap-proto/src/types/type_state.rs index 563dc14d..19b3af7c 100644 --- a/crates/jmap-proto/src/types/type_state.rs +++ b/crates/jmap-proto/src/types/type_state.rs @@ -10,7 +10,7 @@ use serde::Serialize; use store::write::{DeserializeFrom, SerializeInto}; use utils::map::bitmap::BitmapItem; -use crate::parser::{json::Parser, JsonObjectParser}; +use crate::parser::{JsonObjectParser, json::Parser}; #[derive(Debug, Eq, PartialEq, Hash, Clone, Copy, Serialize)] #[repr(u8)] @@ -41,7 +41,19 @@ pub enum DataType { Quota = 11, #[serde(rename = "SieveScript")] SieveScript = 12, - None = 13, + #[serde(rename = "Calendar")] + Calendar = 13, + #[serde(rename = "CalendarEvent")] + CalendarEvent = 14, + #[serde(rename = "CalendarEventNotification")] + CalendarEventNotification = 15, + #[serde(rename = "AddressBook")] + AddressBook = 16, + #[serde(rename = "ContactCard")] + ContactCard = 17, + #[serde(rename = "FileNode")] + FileNode = 18, + None = 19, } impl BitmapItem for DataType { @@ -70,6 +82,12 @@ impl From for DataType { 10 => DataType::Mdn, 11 => DataType::Quota, 12 => DataType::SieveScript, + 13 => DataType::Calendar, + 14 => DataType::CalendarEvent, + 15 => DataType::CalendarEventNotification, + 16 => DataType::AddressBook, + 17 => DataType::ContactCard, + 18 => DataType::FileNode, _ => { debug_assert!(false, "Invalid type_state value: {}", value); DataType::None @@ -171,6 +189,12 @@ impl DataType { DataType::Mdn => "MDN", DataType::Quota => "Quota", DataType::SieveScript => "SieveScript", + DataType::Calendar => "Calendar", + DataType::CalendarEvent => "CalendarEvent", + DataType::CalendarEventNotification => "CalendarEventNotification", + DataType::AddressBook => "AddressBook", + DataType::ContactCard => "ContactCard", + DataType::FileNode => "FileNode", DataType::None => "", } } diff --git a/crates/jmap-proto/src/types/value.rs b/crates/jmap-proto/src/types/value.rs index 93e6fba3..5aa0d1a7 100644 --- a/crates/jmap-proto/src/types/value.rs +++ b/crates/jmap-proto/src/types/value.rs @@ -7,6 +7,7 @@ use std::{borrow::Cow, fmt::Display}; use mail_parser::{Addr, DateTime, Group}; +use rkyv::{option::ArchivedOption, string::ArchivedString}; use serde::Serialize; use utils::{ json::{JsonPointerItem, JsonQueryable}, @@ -49,7 +50,10 @@ pub enum Value { #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize)] pub struct Object(pub VecMap); -#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize)] +#[derive( + rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq, Serialize, +)] +#[rkyv(compare(PartialEq), derive(Debug))] pub struct AclGrant { pub account_id: u32, pub grants: Bitmap, @@ -455,6 +459,12 @@ impl From> for Value { } } +impl From<&ArchivedString> for Value { + fn from(value: &ArchivedString) -> Self { + Value::Text(value.to_string()) + } +} + impl> From> for Value { fn from(value: Vec) -> Self { Value::List(value.into_iter().map(|v| v.into()).collect()) @@ -470,6 +480,30 @@ impl> From> for Value { } } +impl From<&ArchivedOption> for Value { + fn from(value: &ArchivedOption) -> Self { + match value { + ArchivedOption::Some(value) => Value::Text(value.to_string()), + ArchivedOption::None => Value::Null, + } + } +} + +impl From<&ArchivedOption> for Value { + fn from(value: &ArchivedOption) -> Self { + match value { + ArchivedOption::Some(value) => Value::UnsignedInt(u32::from(value) as u64), + ArchivedOption::None => Value::Null, + } + } +} + +impl From<&rkyv::rend::u32_le> for Value { + fn from(value: &rkyv::rend::u32_le) -> Self { + Value::UnsignedInt(u32::from(value) as u64) + } +} + impl From> for Value { fn from(value: Addr<'_>) -> Self { Value::Object(Object( diff --git a/crates/jmap/src/api/event_source.rs b/crates/jmap/src/api/event_source.rs index 0a37544c..0762876b 100644 --- a/crates/jmap/src/api/event_source.rs +++ b/crates/jmap/src/api/event_source.rs @@ -9,16 +9,16 @@ use std::{ time::{Duration, Instant}, }; -use common::{auth::AccessToken, Server}; -use http_body_util::{combinators::BoxBody, StreamBody}; +use common::{Server, auth::AccessToken}; +use http_body_util::{StreamBody, combinators::BoxBody}; use hyper::{ - body::{Bytes, Frame}, StatusCode, + body::{Bytes, Frame}, }; use jmap_proto::types::type_state::DataType; use utils::map::bitmap::Bitmap; -use crate::{services::state::StateManager, LONG_SLUMBER}; +use crate::LONG_SLUMBER; use super::{HttpRequest, HttpResponse, HttpResponseBody, StateChangeResponse}; use std::future::Future; diff --git a/crates/jmap/src/api/form.rs b/crates/jmap/src/api/form.rs index f9bc27ad..ffd209be 100644 --- a/crates/jmap/src/api/form.rs +++ b/crates/jmap/src/api/form.rs @@ -25,7 +25,7 @@ use mail_builder::{ }; use serde_json::json; use store::{ - Serialize, + SerializeInfallible, write::{BatchBuilder, BlobOp, now}, }; use trc::AddContext; diff --git a/crates/jmap/src/api/management/stores.rs b/crates/jmap/src/api/management/stores.rs index 0c28e03f..99da0d23 100644 --- a/crates/jmap/src/api/management/stores.rs +++ b/crates/jmap/src/api/management/stores.rs @@ -9,20 +9,24 @@ use common::{ auth::AccessToken, ipc::{HousekeeperEvent, PurgeType}, manager::webadmin::Resource, + storage::index::ObjectIndexBuilder, *, }; use directory::{ Permission, backend::internal::manage::{self, ManageDirectory}, }; -use email::{mailbox::UidMailbox, message::ingest::EmailIngest}; -use hyper::Method; -use jmap_proto::{ - object::index::ObjectIndexBuilder, - types::{collection::Collection, property::Property}, +use email::{ + mailbox::{ArchivedMailbox, UidMailbox}, + message::ingest::EmailIngest, }; +use hyper::Method; +use jmap_proto::types::{collection::Collection, property::Property}; use serde_json::json; -use store::write::{BatchBuilder, F_VALUE, ValueClass, assert::HashedValue}; +use store::{ + Serialize, + write::{ArchivedValue, BatchBuilder, ValueClass, assert::HashedValue}, +}; use trc::AddContext; use utils::url_params::UrlParams; @@ -337,7 +341,7 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u .unwrap_or_default() { let mailbox = server - .get_property::>( + .get_property::>>( account_id, Collection::Mailbox, mailbox_id, @@ -345,7 +349,9 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u ) .await .caused_by(trc::location!())? - .ok_or_else(|| trc::ImapEvent::Error.into_err().caused_by(trc::location!()))?; + .ok_or_else(|| trc::ImapEvent::Error.into_err().caused_by(trc::location!()))? + .into_deserialized::() + .caused_by(trc::location!())?; let mut new_mailbox = mailbox.inner.clone(); new_mailbox.uid_validity = rand::random::(); let mut batch = BatchBuilder::new(); @@ -358,6 +364,7 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u .with_current(mailbox) .with_changes(new_mailbox), ) + .caused_by(trc::location!())? .clear(Property::EmailIds); server .store() @@ -403,7 +410,10 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u .with_collection(Collection::Email) .update_document(message_id) .assert_value(ValueClass::Property(Property::MailboxIds.into()), &uids) - .value(Property::MailboxIds, uids.inner, F_VALUE); + .set( + Property::MailboxIds, + uids.inner.serialize().caused_by(trc::location!())?, + ); server .store() .write(batch) diff --git a/crates/jmap/src/api/request.rs b/crates/jmap/src/api/request.rs index e9d6eef4..6f364976 100644 --- a/crates/jmap/src/api/request.rs +++ b/crates/jmap/src/api/request.rs @@ -6,13 +6,13 @@ use std::{sync::Arc, time::Instant}; -use common::{auth::AccessToken, Server}; +use common::{Server, auth::AccessToken}; use jmap_proto::{ method::{ get, query, set::{self}, }, - request::{method::MethodName, Call, Request, RequestMethod}, + request::{Call, Request, RequestMethod, method::MethodName}, response::{Response, ResponseMethod}, types::collection::Collection, }; @@ -22,8 +22,8 @@ use crate::{ blob::{copy::BlobCopy, get::BlobOperations, upload::BlobUpload}, changes::{get::ChangesLookup, query::QueryChanges}, email::{ - copy::EmailCopy, get::EmailGet, import::EmailImport, parse::EmailParse, query::EmailQuery, - set::EmailSet, snippet::EmailSearchSnippet, + copy::JmapEmailCopy, get::EmailGet, import::EmailImport, parse::EmailParse, + query::EmailQuery, set::EmailSet, snippet::EmailSearchSnippet, }, identity::{get::IdentityGet, set::IdentitySet}, mailbox::{get::MailboxGet, query::MailboxQuery, set::MailboxSet}, @@ -138,10 +138,12 @@ impl RequestHandler for Server { Err(error) => { let method_error = error.clone(); - trc::error!(error - .span_id(session.session_id) - .ctx_unique(trc::Key::AccountId, access_token.primary_id()) - .caused_by(method_name)); + trc::error!( + error + .span_id(session.session_id) + .ctx_unique(trc::Key::AccountId, access_token.primary_id()) + .caused_by(method_name) + ); response.push_error(call.id, method_error); } diff --git a/crates/jmap/src/api/session.rs b/crates/jmap/src/api/session.rs index 3c94a334..e21cd838 100644 --- a/crates/jmap/src/api/session.rs +++ b/crates/jmap/src/api/session.rs @@ -6,8 +6,8 @@ use std::sync::Arc; -use common::{auth::AccessToken, Server}; -use directory::{backend::internal::PrincipalField, QueryBy}; +use common::{Server, auth::AccessToken}; +use directory::{QueryBy, backend::internal::PrincipalField}; use jmap_proto::{ request::capability::{Capability, Session}, types::{acl::Acl, collection::Collection, id::Id}, @@ -15,8 +15,6 @@ use jmap_proto::{ use std::future::Future; use trc::AddContext; -use crate::auth::acl::AclMethods; - pub trait SessionHandler: Sync + Send { fn handle_session_resource( &self, diff --git a/crates/jmap/src/auth/mod.rs b/crates/jmap/src/auth/mod.rs index 884a3f6f..62df1495 100644 --- a/crates/jmap/src/auth/mod.rs +++ b/crates/jmap/src/auth/mod.rs @@ -4,7 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -pub mod acl; pub mod authenticate; pub mod oauth; pub mod rate_limit; diff --git a/crates/jmap/src/auth/oauth/auth.rs b/crates/jmap/src/auth/oauth/auth.rs index d28ec80e..b4ab3c02 100644 --- a/crates/jmap/src/auth/oauth/auth.rs +++ b/crates/jmap/src/auth/oauth/auth.rs @@ -22,6 +22,7 @@ use serde::Deserialize; use serde_json::json; use std::future::Future; use store::{Serialize, dispatch::lookup::KeyValue, write::Bincode}; +use trc::AddContext; use crate::{ api::{ @@ -113,7 +114,8 @@ impl OAuthApiHandler for Server { nonce, params: redirect_uri.unwrap_or_default(), }) - .serialize(); + .serialize() + .caused_by(trc::location!())?; // Insert client code self.core @@ -174,7 +176,7 @@ impl OAuthApiHandler for Server { KeyValue::with_prefix( KV_OAUTH, device_code.as_bytes(), - auth_code.serialize(), + auth_code.serialize().caused_by(trc::location!())?, ) .expires(self.core.oauth.oauth_expiry_auth_code), ) @@ -237,7 +239,8 @@ impl OAuthApiHandler for Server { nonce, params: device_code.clone(), }) - .serialize(); + .serialize() + .caused_by(trc::location!())?; // Insert device code self.core diff --git a/crates/jmap/src/blob/copy.rs b/crates/jmap/src/blob/copy.rs index c53f5714..b90c07a4 100644 --- a/crates/jmap/src/blob/copy.rs +++ b/crates/jmap/src/blob/copy.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{auth::AccessToken, Server}; +use common::{Server, auth::AccessToken}; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::copy::{CopyBlobRequest, CopyBlobResponse}, @@ -14,8 +14,8 @@ use trc::AddContext; use std::future::Future; use store::{ - write::{now, BatchBuilder, BlobOp}, - BlobClass, Serialize, + BlobClass, SerializeInfallible, + write::{BatchBuilder, BlobOp, now}, }; use utils::map::vec_map::VecMap; diff --git a/crates/jmap/src/blob/download.rs b/crates/jmap/src/blob/download.rs index ceac9561..451b4353 100644 --- a/crates/jmap/src/blob/download.rs +++ b/crates/jmap/src/blob/download.rs @@ -6,23 +6,13 @@ use std::ops::Range; -use common::{auth::AccessToken, Server}; -use jmap_proto::types::{ - acl::Acl, - blob::{BlobId, BlobSection}, - collection::Collection, -}; -use mail_parser::{ - decoders::{base64::base64_decode, quoted_printable::quoted_printable_decode}, - Encoding, -}; +use common::{Server, auth::AccessToken}; +use jmap_proto::types::{acl::Acl, blob::BlobId, collection::Collection}; use std::future::Future; use store::BlobClass; use trc::AddContext; use utils::BlobHash; -use crate::auth::acl::AclMethods; - pub trait BlobDownload: Sync + Send { fn blob_download( &self, @@ -30,12 +20,6 @@ pub trait BlobDownload: Sync + Send { access_token: &AccessToken, ) -> impl Future>>> + Send; - fn get_blob_section( - &self, - hash: &BlobHash, - section: &BlobSection, - ) -> impl Future>>> + Send; - fn get_blob( &self, hash: &BlobHash, @@ -76,7 +60,12 @@ impl BlobDownload for Server { } => { if Collection::from(*collection) == Collection::Email { match self - .shared_messages(access_token, *account_id, Acl::ReadItems) + .shared_document_children( + access_token, + *account_id, + Collection::Mailbox, + Acl::ReadItems, + ) .await { Ok(shared_messages) if shared_messages.contains(*document_id) => (), @@ -111,24 +100,6 @@ impl BlobDownload for Server { } } - async fn get_blob_section( - &self, - hash: &BlobHash, - section: &BlobSection, - ) -> trc::Result>> { - Ok(self - .get_blob( - hash, - (section.offset_start)..(section.offset_start.saturating_add(section.size)), - ) - .await? - .and_then(|bytes| match Encoding::from(section.encoding) { - Encoding::None => Some(bytes), - Encoding::Base64 => base64_decode(&bytes), - Encoding::QuotedPrintable => quoted_printable_decode(&bytes), - })) - } - #[inline(always)] async fn get_blob(&self, hash: &BlobHash, range: Range) -> trc::Result>> { self.core @@ -160,7 +131,12 @@ impl BlobDownload for Server { if Collection::from(*collection) == Collection::Email { access_token.is_member(*account_id) || self - .shared_messages(access_token, *account_id, Acl::ReadItems) + .shared_document_children( + access_token, + *account_id, + Collection::Mailbox, + Acl::ReadItems, + ) .await? .contains(*document_id) } else { diff --git a/crates/jmap/src/changes/get.rs b/crates/jmap/src/changes/get.rs index 3da6ba15..ac44ef08 100644 --- a/crates/jmap/src/changes/get.rs +++ b/crates/jmap/src/changes/get.rs @@ -4,14 +4,13 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{auth::AccessToken, Server}; +use common::{Server, auth::AccessToken}; use jmap_proto::{ method::changes::{ChangesRequest, ChangesResponse, RequestArguments}, types::{collection::Collection, property::Property, state::State}, }; use std::future::Future; -use store::query::log::{Change, Changes, Query}; -use trc::AddContext; +use store::query::log::{Change, Query}; pub trait ChangesLookup: Sync + Send { fn changes( @@ -19,13 +18,6 @@ pub trait ChangesLookup: Sync + Send { request: ChangesRequest, access_token: &AccessToken, ) -> impl Future> + Send; - - fn changes_( - &self, - account_id: u32, - collection: Collection, - query: Query, - ) -> impl Future> + Send; } impl ChangesLookup for Server { @@ -88,7 +80,10 @@ impl ChangesLookup for Server { let (items_sent, mut changelog) = match &request.since_state { State::Initial => { - let changelog = self.changes_(account_id, collection, Query::All).await?; + let changelog = self + .store() + .changes(account_id, collection, Query::All) + .await?; if changelog.changes.is_empty() && changelog.from_change_id == 0 { return Ok(response); } @@ -97,12 +92,14 @@ impl ChangesLookup for Server { } State::Exact(change_id) => ( 0, - self.changes_(account_id, collection, Query::Since(*change_id)) + self.store() + .changes(account_id, collection, Query::Since(*change_id)) .await?, ), State::Intermediate(intermediate_state) => { let mut changelog = self - .changes_( + .store() + .changes( account_id, collection, Query::RangeInclusive(intermediate_state.from_id, intermediate_state.to_id), @@ -111,12 +108,13 @@ impl ChangesLookup for Server { if intermediate_state.items_sent >= changelog.changes.len() { ( 0, - self.changes_( - account_id, - collection, - Query::Since(intermediate_state.to_id), - ) - .await?, + self.store() + .changes( + account_id, + collection, + Query::Since(intermediate_state.to_id), + ) + .await?, ) } else { changelog.changes.drain( @@ -173,18 +171,4 @@ impl ChangesLookup for Server { Ok(response) } - - async fn changes_( - &self, - account_id: u32, - collection: Collection, - query: Query, - ) -> trc::Result { - self.core - .storage - .data - .changes(account_id, collection, query) - .await - .caused_by(trc::location!()) - } } diff --git a/crates/jmap/src/email/copy.rs b/crates/jmap/src/email/copy.rs index 5bb1b064..9c18dcfa 100644 --- a/crates/jmap/src/email/copy.rs +++ b/crates/jmap/src/email/copy.rs @@ -4,21 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{ - Server, - auth::{AccessToken, ResourceToken}, -}; +use common::{Server, auth::AccessToken}; -use email::{ - mailbox::{UidMailbox, manage::MailboxFnc}, - message::{ - index::{ - EmailIndexBuilder, MAX_ID_LENGTH, MAX_SORT_FIELD_LENGTH, TrimTextValue, VisitValues, - }, - ingest::{EmailIngest, IngestedEmail, LogEmailInsert}, - metadata::MessageMetadata, - }, -}; +use email::{mailbox::manage::MailboxFnc, message::copy::EmailCopy}; use jmap_proto::{ error::set::SetError, method::{ @@ -33,33 +21,19 @@ use jmap_proto::{ response::references::EvalObjectReferences, types::{ acl::Acl, - blob::BlobId, collection::Collection, - date::UTCDate, - id::Id, - keyword::Keyword, property::Property, state::{State, StateChange}, type_state::DataType, value::{MaybePatchValue, Value}, }, }; -use mail_parser::{HeaderName, HeaderValue, parsers::fields::thread::thread_name}; -use store::{ - BlobClass, - write::{ - BatchBuilder, Bincode, F_BITMAP, F_VALUE, MaybeDynamicId, TagValue, TaskQueueClass, - ValueClass, - log::{Changes, LogInsert}, - }, -}; -use trc::AddContext; + +use crate::{api::http::HttpSessionData, changes::state::StateManager}; +use std::future::Future; use utils::map::vec_map::VecMap; -use crate::{api::http::HttpSessionData, auth::acl::AclMethods, changes::state::StateManager}; -use std::future::Future; - -pub trait EmailCopy: Sync + Send { +pub trait JmapEmailCopy: Sync + Send { fn email_copy( &self, request: CopyRequest, @@ -67,21 +41,9 @@ pub trait EmailCopy: Sync + Send { next_call: &mut Option>, session: &HttpSessionData, ) -> impl Future> + Send; - - #[allow(clippy::too_many_arguments)] - fn copy_message( - &self, - from_account_id: u32, - from_message_id: u32, - resource_token: &ResourceToken, - mailboxes: Vec, - keywords: Vec, - received_at: Option, - session_id: u64, - ) -> impl Future>> + Send; } -impl EmailCopy for Server { +impl JmapEmailCopy for Server { async fn email_copy( &self, request: CopyRequest, @@ -111,7 +73,12 @@ impl EmailCopy for Server { }; let from_message_ids = self - .owned_or_shared_messages(access_token, from_account_id, Acl::ReadItems) + .owned_or_shared_document_children( + access_token, + from_account_id, + Collection::Mailbox, + Acl::ReadItems, + ) .await?; let mailbox_ids = self.mailbox_get_or_create(account_id).await?; let can_add_mailbox_ids = if access_token.is_shared(account_id) { @@ -300,186 +267,4 @@ impl EmailCopy for Server { Ok(response) } - - #[allow(clippy::too_many_arguments)] - async fn copy_message( - &self, - from_account_id: u32, - from_message_id: u32, - resource_token: &ResourceToken, - mailboxes: Vec, - keywords: Vec, - received_at: Option, - session_id: u64, - ) -> trc::Result> { - // Obtain metadata - let account_id = resource_token.account_id; - let mut metadata = if let Some(metadata) = self - .get_property::>( - from_account_id, - Collection::Email, - from_message_id, - Property::BodyStructure, - ) - .await? - { - metadata.inner - } else { - return Ok(Err(SetError::not_found().with_description(format!( - "Message not found not found in account {}.", - Id::from(from_account_id) - )))); - }; - - // Check quota - match self - .has_available_quota(resource_token, metadata.size as u64) - .await - { - Ok(_) => (), - Err(err) => { - if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) - || err.matches(trc::EventType::Limit(trc::LimitEvent::TenantQuota)) - { - trc::error!(err.account_id(account_id).span_id(session_id)); - return Ok(Err(SetError::over_quota())); - } else { - return Err(err); - } - } - } - - // Set receivedAt - if let Some(received_at) = received_at { - metadata.received_at = received_at.timestamp() as u64; - } - - // Obtain threadId - let mut references = Vec::with_capacity(5); - let mut subject = ""; - for header in &metadata.contents.parts[0].headers { - match &header.name { - HeaderName::MessageId - | HeaderName::InReplyTo - | HeaderName::References - | HeaderName::ResentMessageId => { - header.value.visit_text(|id| { - if !id.is_empty() && id.len() < MAX_ID_LENGTH { - references.push(id); - } - }); - } - HeaderName::Subject if subject.is_empty() => { - subject = thread_name(match &header.value { - HeaderValue::Text(text) => text.as_ref(), - HeaderValue::TextList(list) if !list.is_empty() => { - list.first().unwrap().as_ref() - } - _ => "", - }) - .trim_text(MAX_SORT_FIELD_LENGTH); - } - _ => (), - } - } - - let thread_id = if !references.is_empty() { - self.find_or_merge_thread(account_id, subject, &references) - .await - .caused_by(trc::location!())? - } else { - None - }; - - // Assign id - let mut email = IngestedEmail { - size: metadata.size, - ..Default::default() - }; - let blob_hash = metadata.blob_hash.clone(); - - // Assign IMAP UIDs - let mut mailbox_ids = Vec::with_capacity(mailboxes.len()); - email.imap_uids = Vec::with_capacity(mailboxes.len()); - for mailbox_id in &mailboxes { - let uid = self - .assign_imap_uid(account_id, *mailbox_id) - .await - .caused_by(trc::location!())?; - mailbox_ids.push(UidMailbox::new(*mailbox_id, uid)); - email.imap_uids.push(uid); - } - - // Prepare batch - let change_id = self.assign_change_id(account_id)?; - let mut batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_change_id(change_id) - .with_collection(Collection::Thread); - if let Some(thread_id) = thread_id { - batch.log(Changes::update([thread_id])); - } else { - batch.create_document().log(LogInsert()); - }; - - // Build batch - let maybe_thread_id = thread_id - .map(MaybeDynamicId::Static) - .unwrap_or(MaybeDynamicId::Dynamic(0)); - batch - .with_collection(Collection::Mailbox) - .log(Changes::child_update(mailboxes.iter().copied())) - .with_collection(Collection::Email) - .create_document() - .log(LogEmailInsert::new(thread_id)) - .set(Property::ThreadId, maybe_thread_id) - .tag(Property::ThreadId, TagValue::Id(maybe_thread_id), 0) - .value(Property::MailboxIds, mailbox_ids, F_VALUE | F_BITMAP) - .value(Property::Keywords, keywords, F_VALUE | F_BITMAP) - .value(Property::Cid, change_id, F_VALUE) - .set( - ValueClass::TaskQueue(TaskQueueClass::IndexEmail { - seq: self.generate_snowflake_id()?, - hash: metadata.blob_hash.clone(), - }), - vec![], - ); - EmailIndexBuilder::set(metadata).build( - &mut batch, - account_id, - resource_token.tenant.map(|t| t.id), - ); - - // Insert and obtain ids - let ids = self - .core - .storage - .data - .write(batch.build()) - .await - .caused_by(trc::location!())?; - let thread_id = match thread_id { - Some(thread_id) => thread_id, - None => ids.first_document_id().caused_by(trc::location!())?, - }; - let document_id = ids.last_document_id().caused_by(trc::location!())?; - - // Request FTS index - self.notify_task_queue(); - - // Update response - email.id = Id::from_parts(thread_id, document_id); - email.change_id = change_id; - email.blob_id = BlobId::new( - blob_hash, - BlobClass::Linked { - account_id, - collection: Collection::Email.into(), - document_id, - }, - ); - - Ok(Ok(email)) - } } diff --git a/crates/jmap/src/email/crypto.rs b/crates/jmap/src/email/crypto.rs index a3840c1e..7db5853d 100644 --- a/crates/jmap/src/email/crypto.rs +++ b/crates/jmap/src/email/crypto.rs @@ -19,8 +19,9 @@ use mail_parser::MessageParser; use serde_json::json; use store::{ Serialize, - write::{BatchBuilder, Bincode, F_CLEAR, F_VALUE}, + write::{BatchBuilder, Bincode}, }; +use trc::AddContext; pub trait CryptoHandler: Sync + Send { fn handle_crypto_get( @@ -51,7 +52,11 @@ impl CryptoHandler for Server { let algo = params.algo; let mut certs = Vec::new(); certs.extend_from_slice(b"-----STALWART CERTIFICATE-----\r\n"); - let _ = base64_encode_mime(&Bincode::new(params).serialize(), &mut certs, false); + let _ = base64_encode_mime( + &Bincode::new(params).serialize().unwrap_or_default(), + &mut certs, + false, + ); certs.extend_from_slice(b"\r\n"); let certs = String::from_utf8(certs).unwrap_or_default(); @@ -86,7 +91,7 @@ impl CryptoHandler for Server { .with_account_id(access_token.primary_id()) .with_collection(Collection::Principal) .update_document(0) - .value(Property::Parameters, (), F_VALUE | F_CLEAR); + .clear(Property::Parameters); self.core.storage.data.write(batch.build()).await?; return Ok(JsonResponse::new(json!({ "data": (), @@ -130,7 +135,10 @@ impl CryptoHandler for Server { .with_account_id(access_token.primary_id()) .with_collection(Collection::Principal) .update_document(0) - .value(Property::Parameters, ¶ms, F_VALUE); + .set( + Property::Parameters, + params.serialize().caused_by(trc::location!())?, + ); self.core.storage.data.write(batch.build()).await?; Ok(JsonResponse::new(json!({ diff --git a/crates/jmap/src/email/get.rs b/crates/jmap/src/email/get.rs index efcc62d5..238a70bb 100644 --- a/crates/jmap/src/email/get.rs +++ b/crates/jmap/src/email/get.rs @@ -30,8 +30,7 @@ use store::{BlobClass, write::Bincode}; use trc::{AddContext, StoreEvent}; use crate::{ - auth::acl::AclMethods, blob::download::BlobDownload, changes::state::StateManager, - email::headers::HeaderToValue, + blob::download::BlobDownload, changes::state::StateManager, email::headers::HeaderToValue, }; use std::future::Future; @@ -102,7 +101,12 @@ impl EmailGet for Server { let account_id = request.account_id.document_id(); let message_ids = self - .owned_or_shared_messages(access_token, account_id, Acl::ReadItems) + .owned_or_shared_document_children( + access_token, + account_id, + Collection::Mailbox, + Acl::ReadItems, + ) .await?; let ids = if let Some(ids) = ids { ids diff --git a/crates/jmap/src/email/import.rs b/crates/jmap/src/email/import.rs index f59c2ea8..dc58c6a4 100644 --- a/crates/jmap/src/email/import.rs +++ b/crates/jmap/src/email/import.rs @@ -25,8 +25,7 @@ use mail_parser::MessageParser; use utils::map::vec_map::VecMap; use crate::{ - api::http::HttpSessionData, auth::acl::AclMethods, blob::download::BlobDownload, - changes::state::StateManager, + api::http::HttpSessionData, blob::download::BlobDownload, changes::state::StateManager, }; use std::future::Future; diff --git a/crates/jmap/src/email/query.rs b/crates/jmap/src/email/query.rs index 4f656d68..bd7199be 100644 --- a/crates/jmap/src/email/query.rs +++ b/crates/jmap/src/email/query.rs @@ -15,14 +15,14 @@ use mail_parser::HeaderName; use nlp::language::Language; use std::future::Future; use store::{ - ValueKey, + SerializeInfallible, ValueKey, fts::{Field, FilterGroup, FtsFilter, IntoFilterGroup}, query::{self}, roaring::RoaringBitmap, write::ValueClass, }; -use crate::{JmapMethods, auth::acl::AclMethods}; +use crate::JmapMethods; pub trait EmailQuery: Sync + Send { fn email_query( @@ -203,16 +203,16 @@ impl EmailQuery for Server { filters.push(query::Filter::End); } Filter::Before(date) => { - filters.push(query::Filter::lt(Property::ReceivedAt, date)) + filters.push(query::Filter::lt(Property::ReceivedAt, date.serialize())) } Filter::After(date) => { - filters.push(query::Filter::gt(Property::ReceivedAt, date)) + filters.push(query::Filter::gt(Property::ReceivedAt, date.serialize())) } Filter::MinSize(size) => { - filters.push(query::Filter::ge(Property::Size, size)) + filters.push(query::Filter::ge(Property::Size, size.serialize())) } Filter::MaxSize(size) => { - filters.push(query::Filter::lt(Property::Size, size)) + filters.push(query::Filter::lt(Property::Size, size.serialize())) } Filter::AllInThreadHaveKeyword(keyword) => { filters.push(query::Filter::is_in_set( @@ -258,10 +258,10 @@ impl EmailQuery for Server { filters.push(query::Filter::is_in_set(set)); } Filter::SentBefore(date) => { - filters.push(query::Filter::lt(Property::SentAt, date)) + filters.push(query::Filter::lt(Property::SentAt, date.serialize())) } Filter::SentAfter(date) => { - filters.push(query::Filter::gt(Property::SentAt, date)) + filters.push(query::Filter::gt(Property::SentAt, date.serialize())) } Filter::InThread(id) => filters.push(query::Filter::is_in_bitmap( Property::ThreadId, @@ -284,8 +284,13 @@ impl EmailQuery for Server { let mut result_set = self.filter(account_id, Collection::Email, filters).await?; if access_token.is_shared(account_id) { result_set.apply_mask( - self.shared_messages(access_token, account_id, Acl::ReadItems) - .await?, + self.shared_document_children( + access_token, + account_id, + Collection::Mailbox, + Acl::ReadItems, + ) + .await?, ); } let (response, paginate) = self.build_query_response(&result_set, &request).await?; diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index 469113d5..081dc6dc 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -4,9 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{borrow::Cow, collections::HashMap, slice::IterMut}; +use std::{borrow::Cow, collections::HashMap}; -use common::{Server, auth::AccessToken}; +use common::{Server, auth::AccessToken, storage::tag::TagManager}; use email::{ mailbox::{UidMailbox, manage::MailboxFnc}, message::{ @@ -38,18 +38,15 @@ use mail_builder::{ }; use mail_parser::MessageParser; use store::{ - Serialize, + SerializeInfallible, ahash::AHashSet, roaring::RoaringBitmap, - write::{ - BatchBuilder, DeserializeFrom, F_BITMAP, F_CLEAR, F_VALUE, SerializeInto, ToBitmaps, - ValueClass, assert::HashedValue, log::ChangeLogBuilder, - }, + write::{BatchBuilder, assert::HashedValue, log::ChangeLogBuilder}, }; use trc::AddContext; use crate::{ - JmapMethods, api::http::HttpSessionData, auth::acl::AclMethods, blob::download::BlobDownload, + JmapMethods, api::http::HttpSessionData, blob::download::BlobDownload, changes::state::StateManager, }; use std::future::Future; @@ -96,9 +93,14 @@ impl EmailSet for Server { ) .await? .into(), - self.shared_messages(access_token, account_id, Acl::ModifyItems) - .await? - .into(), + self.shared_document_children( + access_token, + account_id, + Collection::Mailbox, + Acl::ModifyItems, + ) + .await? + .into(), ) } else { (None, None, None) @@ -878,13 +880,15 @@ impl EmailSet for Server { } // Update keywords property - keywords.update_batch(&mut batch, Property::Keywords); + keywords + .update_batch(&mut batch, Property::Keywords) + .caused_by(trc::location!())?; // Update last change id if changes.change_id == u64::MAX { changes.change_id = self.assign_change_id(account_id)?; } - batch.value(Property::Cid, changes.change_id, F_VALUE); + batch.set(Property::Cid, changes.change_id.serialize()); } // Process mailboxes @@ -960,7 +964,9 @@ impl EmailSet for Server { } // Update mailboxIds property - mailboxes.update_batch(&mut batch, Property::MailboxIds); + mailboxes + .update_batch(&mut batch, Property::MailboxIds) + .caused_by(trc::location!())?; } // Log mailbox changes @@ -997,9 +1003,14 @@ impl EmailSet for Server { .await? .unwrap_or_default(); let can_destroy_message_ids = if access_token.is_shared(account_id) { - self.shared_messages(access_token, account_id, Acl::RemoveItems) - .await? - .into() + self.shared_document_children( + access_token, + account_id, + Collection::Mailbox, + Acl::RemoveItems, + ) + .await? + .into() } else { None }; @@ -1074,109 +1085,3 @@ impl EmailSet for Server { Ok(response) } } -pub struct TagManager< - T: PartialEq + Clone + ToBitmaps + SerializeInto + Serialize + DeserializeFrom + Sync + Send, -> { - current: HashedValue>, - added: Vec, - removed: Vec, - last: LastTag, -} - -enum LastTag { - Set, - Update, - None, -} - -impl - TagManager -{ - pub fn new(current: HashedValue>) -> Self { - Self { - current, - added: Vec::new(), - removed: Vec::new(), - last: LastTag::None, - } - } - - pub fn set(&mut self, tags: Vec) { - if matches!(self.last, LastTag::None) { - self.added.clear(); - self.removed.clear(); - - for tag in &tags { - if !self.current.inner.contains(tag) { - self.added.push(tag.clone()); - } - } - - for tag in &self.current.inner { - if !tags.contains(tag) { - self.removed.push(tag.clone()); - } - } - - self.current.inner = tags; - self.last = LastTag::Set; - } - } - - pub fn update(&mut self, tag: T, add: bool) { - if matches!(self.last, LastTag::None | LastTag::Update) { - if add { - if !self.current.inner.contains(&tag) { - self.added.push(tag.clone()); - self.current.inner.push(tag); - } - } else if let Some(index) = self.current.inner.iter().position(|t| t == &tag) { - self.current.inner.swap_remove(index); - self.removed.push(tag); - } - self.last = LastTag::Update; - } - } - - pub fn added(&self) -> &[T] { - &self.added - } - - pub fn removed(&self) -> &[T] { - &self.removed - } - - pub fn current(&self) -> &[T] { - &self.current.inner - } - - pub fn changed_tags(&self) -> impl Iterator { - self.added.iter().chain(self.removed.iter()) - } - - pub fn inner_tags_mut(&mut self) -> IterMut<'_, T> { - self.current.inner.iter_mut() - } - - pub fn has_tags(&self) -> bool { - !self.current.inner.is_empty() - } - - pub fn has_changes(&self) -> bool { - !self.added.is_empty() || !self.removed.is_empty() - } - - pub fn update_batch(self, batch: &mut BatchBuilder, property: Property) { - let property = u8::from(property); - - batch - .assert_value(ValueClass::Property(property), &self.current) - .value(property, self.current.inner, F_VALUE); - for added in self.added { - batch.value(property, added, F_BITMAP); - } - for removed in self.removed { - batch.value(property, removed, F_BITMAP | F_CLEAR); - } - } -} diff --git a/crates/jmap/src/email/snippet.rs b/crates/jmap/src/email/snippet.rs index 5cd6b007..4cdff507 100644 --- a/crates/jmap/src/email/snippet.rs +++ b/crates/jmap/src/email/snippet.rs @@ -17,7 +17,7 @@ use mail_parser::{GetHeader, HeaderName, PartType, decoders::html::html_to_text} use nlp::language::{Language, search_snippet::generate_snippet, stemmer::Stemmer}; use store::{backend::MAX_TOKEN_LENGTH, write::Bincode}; -use crate::{auth::acl::AclMethods, blob::download::BlobDownload}; +use crate::blob::download::BlobDownload; use std::future::Future; @@ -82,7 +82,12 @@ impl EmailSearchSnippet for Server { } let account_id = request.account_id.document_id(); let document_ids = self - .owned_or_shared_messages(access_token, account_id, Acl::ReadItems) + .owned_or_shared_document_children( + access_token, + account_id, + Collection::Mailbox, + Acl::ReadItems, + ) .await?; let email_ids = request.email_ids.unwrap(); let mut response = GetSearchSnippetResponse { diff --git a/crates/jmap/src/identity/get.rs b/crates/jmap/src/identity/get.rs index 4734c2bd..81e17555 100644 --- a/crates/jmap/src/identity/get.rs +++ b/crates/jmap/src/identity/get.rs @@ -6,7 +6,7 @@ use common::Server; use directory::{QueryBy, backend::internal::PrincipalField}; -use email::identity::{EmailAddress, Identity}; +use email::identity::{ArchivedEmailAddress, ArchivedIdentity, Identity}; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, types::{ @@ -15,7 +15,12 @@ use jmap_proto::{ value::{Object, Value}, }, }; -use store::{Serialize, roaring::RoaringBitmap, write::BatchBuilder}; +use store::{ + Serialize, + rkyv::{option::ArchivedOption, vec::ArchivedVec}, + roaring::RoaringBitmap, + write::{ArchivedValue, BatchBuilder}, +}; use trc::AddContext; use utils::sanitize_email; @@ -79,8 +84,8 @@ impl IdentityGet for Server { response.not_found.push(id.into()); continue; } - let mut identity = if let Some(identity) = self - .get_property::( + let _identity = if let Some(identity) = self + .get_property::>( account_id, Collection::Identity, document_id, @@ -93,6 +98,7 @@ impl IdentityGet for Server { response.not_found.push(id.into()); continue; }; + let identity = _identity.unarchive().caused_by(trc::location!())?; let mut result = Object::with_capacity(properties.len()); for property in &properties { match property { @@ -103,28 +109,22 @@ impl IdentityGet for Server { result.append(Property::MayDelete, Value::Bool(true)); } Property::Name => { - result.append(Property::Name, std::mem::take(&mut identity.name)); + result.append(Property::Name, identity.name.to_string()); } Property::Email => { - result.append(Property::Email, std::mem::take(&mut identity.email)); + result.append(Property::Email, identity.email.to_string()); } Property::TextSignature => { - result.append( - Property::TextSignature, - std::mem::take(&mut identity.text_signature), - ); + result.append(Property::TextSignature, identity.text_signature.to_string()); } Property::HtmlSignature => { - result.append( - Property::HtmlSignature, - std::mem::take(&mut identity.html_signature), - ); + result.append(Property::HtmlSignature, identity.html_signature.to_string()); } Property::Bcc => { - result.append(Property::Bcc, email_to_value(identity.bcc.take())); + result.append(Property::Bcc, email_to_value(&identity.bcc)); } Property::ReplyTo => { - result.append(Property::ReplyTo, email_to_value(identity.reply_to.take())); + result.append(Property::ReplyTo, email_to_value(&identity.reply_to)); } property => { result.append(property.clone(), Value::Null); @@ -192,7 +192,8 @@ impl IdentityGet for Server { email, ..Default::default() } - .serialize(), + .serialize() + .caused_by(trc::location!())?, ); identity_ids.insert(document_id); } @@ -207,16 +208,16 @@ impl IdentityGet for Server { } } -fn email_to_value(email: Option>) -> Value { - if let Some(email) = email { +fn email_to_value(email: &ArchivedOption>) -> Value { + if let ArchivedOption::Some(email) = email { Value::List( email - .into_iter() + .iter() .map(|email| { Value::Object( Object::with_capacity(2) - .with_property(Property::Name, email.name) - .with_property(Property::Email, email.email), + .with_property(Property::Name, &email.name) + .with_property(Property::Email, &email.email), ) }) .collect(), diff --git a/crates/jmap/src/identity/set.rs b/crates/jmap/src/identity/set.rs index 64d158d7..e9e57c10 100644 --- a/crates/jmap/src/identity/set.rs +++ b/crates/jmap/src/identity/set.rs @@ -6,7 +6,7 @@ use common::Server; use directory::{QueryBy, backend::internal::PrincipalField}; -use email::identity::{EmailAddress, Identity}; +use email::identity::{ArchivedIdentity, EmailAddress, Identity}; use jmap_proto::{ error::set::SetError, method::set::{RequestArguments, SetRequest, SetResponse}, @@ -18,8 +18,8 @@ use jmap_proto::{ }, }; use std::future::Future; -use store::Serialize; -use store::write::{BatchBuilder, F_CLEAR, F_VALUE, log::ChangeLogBuilder}; +use store::write::{BatchBuilder, log::ChangeLogBuilder}; +use store::{Serialize, write::ArchivedValue}; use trc::AddContext; use utils::sanitize_email; @@ -94,7 +94,10 @@ impl IdentitySet for Server { .with_account_id(account_id) .with_collection(Collection::Identity) .create_document() - .set(Property::Value, identity.serialize()); + .set( + Property::Value, + identity.serialize().caused_by(trc::location!())?, + ); let document_id = self .store() .write_expect_id(batch) @@ -116,7 +119,7 @@ impl IdentitySet for Server { // Obtain identity let document_id = id.document_id(); let mut identity = if let Some(identity) = self - .get_property::( + .get_property::>( account_id, Collection::Identity, document_id, @@ -124,7 +127,7 @@ impl IdentitySet for Server { ) .await? { - identity + identity.deserialize().caused_by(trc::location!())? } else { response.not_updated.append(id, SetError::not_found()); continue 'update; @@ -145,7 +148,10 @@ impl IdentitySet for Server { .with_account_id(account_id) .with_collection(Collection::Identity) .update_document(document_id) - .set(Property::Value, identity.serialize()); + .set( + Property::Value, + identity.serialize().caused_by(trc::location!())?, + ); self.store() .write(batch) .await @@ -164,7 +170,7 @@ impl IdentitySet for Server { .with_account_id(account_id) .with_collection(Collection::Identity) .delete_document(document_id) - .value(Property::Value, (), F_VALUE | F_CLEAR); + .clear(Property::Value); self.store() .write(batch) .await diff --git a/crates/jmap/src/mailbox/get.rs b/crates/jmap/src/mailbox/get.rs index 8b5de409..59dc713d 100644 --- a/crates/jmap/src/mailbox/get.rs +++ b/crates/jmap/src/mailbox/get.rs @@ -4,8 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{Server, auth::AccessToken}; -use email::mailbox::{Mailbox, manage::MailboxFnc}; +use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; +use email::mailbox::{ArchivedMailbox, manage::MailboxFnc}; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, types::{ @@ -15,11 +15,10 @@ use jmap_proto::{ value::{Object, Value}, }, }; +use store::write::ArchivedValue; +use trc::AddContext; -use crate::{ - auth::acl::{AclMethods, EffectiveAcl}, - changes::state::StateManager, -}; +use crate::changes::state::StateManager; use std::future::Future; @@ -97,9 +96,9 @@ impl MailboxGet for Server { continue; } - let mut values = if fetch_properties { + let archived_mailbox_ = if fetch_properties { match self - .get_property::( + .get_property::>( account_id, Collection::Mailbox, document_id, @@ -117,34 +116,33 @@ impl MailboxGet for Server { } else { None }; + let archived_mailbox = if let Some(archived_mailbox) = &archived_mailbox_ { + archived_mailbox + .unarchive() + .caused_by(trc::location!())? + .into() + } else { + None + }; let mut mailbox = Object::with_capacity(properties.len()); for property in &properties { let value = match property { Property::Id => Value::Id(id), - Property::Name => { - Value::Text(std::mem::take(&mut values.as_mut().unwrap().name)) - } + Property::Name => Value::Text(archived_mailbox.unwrap().name.to_string()), Property::Role => { - if let Some(role) = values.as_ref().unwrap().role.as_str() { + if let Some(role) = archived_mailbox.unwrap().role.as_str() { Value::Text(role.to_string()) } else { Value::Null } } - Property::SortOrder => Value::UnsignedInt( - values - .as_ref() - .unwrap() - .sort_order - .unwrap_or_default() - .into(), - ), + Property::SortOrder => Value::from(&archived_mailbox.unwrap().sort_order), Property::ParentId => { - let parent_id = values.as_ref().unwrap().parent_id; + let parent_id = archived_mailbox.as_ref().unwrap().parent_id; if parent_id > 0 { - Value::Id((parent_id - 1).into()) + Value::Id((u32::from(parent_id) - 1).into()) } else { Value::Null } @@ -189,7 +187,7 @@ impl MailboxGet for Server { ), Property::MyRights => { if access_token.is_shared(account_id) { - let acl = values.as_ref().unwrap().acls.effective_acl(access_token); + let acl = archived_mailbox.unwrap().acls.effective_acl(access_token); Object::with_capacity(9) .with_property(Property::MayReadItems, acl.contains(Acl::ReadItems)) .with_property(Property::MayAddItems, acl.contains(Acl::AddItems)) @@ -225,11 +223,11 @@ impl MailboxGet for Server { } } Property::IsSubscribed => { - if values - .as_ref() + if archived_mailbox .unwrap() .subscribers - .contains(&access_token.primary_id()) + .iter() + .any(|s| u32::from(s) == access_token.primary_id()) { Value::Bool(true) } else { @@ -237,7 +235,7 @@ impl MailboxGet for Server { } } Property::Acl => { - self.acl_get(&values.as_ref().unwrap().acls, access_token, account_id) + self.acl_get(&archived_mailbox.unwrap().acls, access_token, account_id) .await } diff --git a/crates/jmap/src/mailbox/query.rs b/crates/jmap/src/mailbox/query.rs index 22e6645c..d9fdea7a 100644 --- a/crates/jmap/src/mailbox/query.rs +++ b/crates/jmap/src/mailbox/query.rs @@ -5,19 +5,21 @@ */ use common::{Server, auth::AccessToken}; -use email::mailbox::{Mailbox, manage::MailboxFnc}; +use email::mailbox::{ArchivedMailbox, manage::MailboxFnc}; use jmap_proto::{ method::query::{Comparator, Filter, QueryRequest, QueryResponse, SortProperty}, object::mailbox::QueryArguments, types::{acl::Acl, collection::Collection, property::Property}, }; use store::{ + Serialize, SerializeInfallible, ahash::{AHashMap, AHashSet}, query::{self, sort::Pagination}, roaring::RoaringBitmap, + write::ArchivedValue, }; -use crate::{JmapMethods, UpdateResults, auth::acl::AclMethods}; +use crate::{JmapMethods, UpdateResults}; use std::future::Future; pub trait MailboxQuery: Sync + Send { @@ -44,7 +46,10 @@ impl MailboxQuery for Server { match cond { Filter::ParentId(parent_id) => filters.push(query::Filter::eq( Property::ParentId, - parent_id.map(|id| id.document_id() + 1).unwrap_or(0), + parent_id + .map(|id| id.document_id() + 1) + .unwrap_or(0) + .serialize(), )), Filter::Name(name) => { #[cfg(feature = "test_mode")] @@ -54,11 +59,14 @@ impl MailboxQuery for Server { tokio::time::sleep(std::time::Duration::from_secs(1)).await; } } - filters.push(query::Filter::has_text(Property::Name, &name)); + filters.push(query::Filter::contains( + Property::Name, + name.to_lowercase().into_bytes(), + )); } Filter::Role(role) => { if let Some(role) = role { - filters.push(query::Filter::eq(Property::Role, role)); + filters.push(query::Filter::eq(Property::Role, role.into_bytes())); } else { filters.push(query::Filter::Not); filters.push(query::Filter::is_in_bitmap(Property::Role, ())); @@ -80,7 +88,7 @@ impl MailboxQuery for Server { } filters.push(query::Filter::eq( Property::IsSubscribed, - access_token.primary_id, + access_token.primary_id.serialize(), )); if !is_subscribed { filters.push(query::Filter::End); @@ -117,7 +125,7 @@ impl MailboxQuery for Server { || (response.total.is_some_and(|total| total > 0) && filter_as_tree)) { for (document_id, value) in self - .get_properties::( + .get_properties::, _, _>( account_id, Collection::Mailbox, &mailbox_ids, @@ -125,8 +133,11 @@ impl MailboxQuery for Server { ) .await? { - hierarchy.insert(document_id + 1, value.parent_id); - tree.entry(value.parent_id) + let todo = "use index"; + let mailbox = value.unarchive()?; + let parent_id = u32::from(mailbox.parent_id); + hierarchy.insert(document_id + 1, parent_id); + tree.entry(parent_id) .or_insert_with(AHashSet::default) .insert(document_id + 1); } diff --git a/crates/jmap/src/mailbox/set.rs b/crates/jmap/src/mailbox/set.rs index ba3bb73b..4e3fa009 100644 --- a/crates/jmap/src/mailbox/set.rs +++ b/crates/jmap/src/mailbox/set.rs @@ -4,16 +4,16 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{Server, auth::AccessToken, config::jmap::settings::SpecialUse}; -use directory::Permission; -use email::{ - mailbox::{Mailbox, manage::MailboxFnc}, - message::delete::EmailDeletion, +use common::{ + Server, auth::AccessToken, config::jmap::settings::SpecialUse, sharing::EffectiveAcl, + storage::index::ObjectIndexBuilder, }; + +use email::mailbox::{ArchivedMailbox, Mailbox, destroy::MailboxDestroy, manage::MailboxFnc}; use jmap_proto::{ - error::set::{SetError, SetErrorType}, + error::set::SetError, method::set::{SetRequest, SetResponse}, - object::{index::ObjectIndexBuilder, mailbox::SetArguments}, + object::mailbox::SetArguments, response::references::EvalObjectReferences, types::{ acl::Acl, @@ -26,20 +26,19 @@ use jmap_proto::{ }, }; use store::{ + SerializeInfallible, query::Filter, roaring::RoaringBitmap, write::{ - BatchBuilder, F_BITMAP, F_CLEAR, F_VALUE, + ArchivedValue, BatchBuilder, assert::{AssertValue, HashedValue}, log::ChangeLogBuilder, }, }; +use trc::AddContext; use utils::config::utils::ParseValue; -use crate::{ - JmapMethods, - auth::acl::{AclMethods, EffectiveAcl}, -}; +use crate::JmapMethods; #[allow(unused_imports)] use email::mailbox::{INBOX_ID, JUNK_ID, TRASH_ID, UidMailbox}; @@ -61,15 +60,6 @@ pub trait MailboxSet: Sync + Send { access_token: &AccessToken, ) -> impl Future> + Send; - fn mailbox_destroy( - &self, - account_id: u32, - document_id: u32, - changes: &mut ChangeLogBuilder, - access_token: &AccessToken, - remove_emails: bool, - ) -> impl Future>> + Send; - fn mailbox_set_item( &self, changes_: Object, @@ -116,7 +106,10 @@ impl MailboxSet for Server { .assert_value(Property::Value, AssertValue::Some); } - batch.create_document().custom(builder); + batch + .create_document() + .custom(builder) + .caused_by(trc::location!())?; match self .core @@ -165,7 +158,7 @@ impl MailboxSet for Server { // Obtain mailbox let document_id = id.document_id(); if let Some(mailbox) = self - .get_property::>( + .get_property::>>( account_id, Collection::Mailbox, document_id, @@ -174,6 +167,7 @@ impl MailboxSet for Server { .await? { // Validate ACL + let mailbox = mailbox.into_deserialized().caused_by(trc::location!())?; if ctx.is_shared { let acl = mailbox.inner.acls.effective_acl(access_token); if !acl.contains(Acl::Modify) { @@ -213,7 +207,10 @@ impl MailboxSet for Server { .assert_value(Property::Value, AssertValue::Some); } - batch.update_document(document_id).custom(builder); + batch + .update_document(document_id) + .custom(builder) + .caused_by(trc::location!())?; if !batch.is_empty() { match self.core.storage.data.write(batch.build()).await { @@ -284,197 +281,6 @@ impl MailboxSet for Server { Ok(ctx.response) } - async fn mailbox_destroy( - &self, - account_id: u32, - document_id: u32, - changes: &mut ChangeLogBuilder, - access_token: &AccessToken, - remove_emails: bool, - ) -> trc::Result> { - // Internal folders cannot be deleted - #[cfg(feature = "test_mode")] - if [INBOX_ID, TRASH_ID].contains(&document_id) - && !access_token.has_permission(Permission::DeleteSystemFolders) - { - return Ok(Err(SetError::forbidden().with_description( - "You are not allowed to delete Inbox, Junk or Trash folders.", - ))); - } - - #[cfg(not(feature = "test_mode"))] - if [INBOX_ID, TRASH_ID, JUNK_ID].contains(&document_id) - && !access_token.has_permission(Permission::DeleteSystemFolders) - { - return Ok(Err(SetError::forbidden().with_description( - "You are not allowed to delete Inbox, Junk or Trash folders.", - ))); - } - - // Verify that this mailbox does not have sub-mailboxes - if !self - .filter( - account_id, - Collection::Mailbox, - vec![Filter::eq(Property::ParentId, document_id + 1)], - ) - .await? - .results - .is_empty() - { - return Ok(Err(SetError::new(SetErrorType::MailboxHasChild) - .with_description("Mailbox has at least one children."))); - } - - // Verify that the mailbox is empty - let mut did_remove_emails = false; - if let Some(message_ids) = self - .get_tag( - account_id, - Collection::Email, - Property::MailboxIds, - document_id, - ) - .await? - { - if remove_emails { - // Flag removal for state change notification - did_remove_emails = true; - - // If the message is in multiple mailboxes, untag it from the current mailbox, - // otherwise delete it. - let mut destroy_ids = RoaringBitmap::new(); - for (message_id, mut mailbox_ids) in self - .get_properties::>, _, _>( - account_id, - Collection::Email, - &message_ids, - Property::MailboxIds, - ) - .await? - { - // Remove mailbox from list - let orig_len = mailbox_ids.inner.len(); - mailbox_ids.inner.retain(|id| id.mailbox_id != document_id); - if mailbox_ids.inner.len() == orig_len { - continue; - } - - if !mailbox_ids.inner.is_empty() { - // Obtain threadId - if let Some(thread_id) = self - .get_property::( - account_id, - Collection::Email, - message_id, - Property::ThreadId, - ) - .await? - { - // Untag message from mailbox - let mut batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_collection(Collection::Email) - .update_document(message_id) - .assert_value(Property::MailboxIds, &mailbox_ids) - .value(Property::MailboxIds, mailbox_ids.inner, F_VALUE) - .value(Property::MailboxIds, document_id, F_BITMAP | F_CLEAR); - match self.core.storage.data.write(batch.build()).await { - Ok(_) => changes.log_update( - Collection::Email, - Id::from_parts(thread_id, message_id), - ), - Err(err) if err.is_assertion_failure() => { - return Ok(Err(SetError::forbidden().with_description( - concat!( - "Another process modified a message in this mailbox ", - "while deleting it, please try again." - ), - ))); - } - Err(err) => { - return Err(err.caused_by(trc::location!())); - } - } - } else { - trc::event!( - Store(trc::StoreEvent::NotFound), - AccountId = account_id, - MessageId = message_id, - MailboxId = document_id, - Details = "Message does not have a threadId.", - CausedBy = trc::location!(), - ); - } - } else { - // Delete message - destroy_ids.insert(message_id); - } - } - - // Bulk delete messages - if !destroy_ids.is_empty() { - let (mut change, _) = self.emails_tombstone(account_id, destroy_ids).await?; - change.changes.remove(&(Collection::Mailbox as u8)); - changes.merge(change); - } - } else { - return Ok(Err(SetError::new(SetErrorType::MailboxHasEmail) - .with_description("Mailbox is not empty."))); - } - } - - // Obtain mailbox - if let Some(mailbox) = self - .get_property::>( - account_id, - Collection::Mailbox, - document_id, - Property::Value, - ) - .await? - { - // Validate ACLs - if access_token.is_shared(account_id) { - let acl = mailbox.inner.acls.effective_acl(access_token); - if !acl.contains(Acl::Administer) { - if !acl.contains(Acl::Delete) { - return Ok(Err(SetError::forbidden() - .with_description("You are not allowed to delete this mailbox."))); - } else if remove_emails && !acl.contains(Acl::RemoveItems) { - return Ok(Err(SetError::forbidden().with_description( - "You are not allowed to delete emails from this mailbox.", - ))); - } - } - } - - let mut batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_collection(Collection::Mailbox) - .delete_document(document_id) - .value(Property::EmailIds, (), F_VALUE | F_CLEAR) - .custom(ObjectIndexBuilder::new().with_current(mailbox)); - - match self.core.storage.data.write(batch.build()).await { - Ok(_) => { - changes.log_delete(Collection::Mailbox, document_id); - Ok(Ok(did_remove_emails)) - } - Err(err) if err.is_assertion_failure() => Ok(Err(SetError::forbidden() - .with_description(concat!( - "Another process modified this mailbox ", - "while deleting it, please try again." - )))), - Err(err) => Err(err.caused_by(trc::location!())), - } - } else { - Ok(Err(SetError::not_found())) - } - } - #[allow(clippy::blocks_in_conditions)] async fn mailbox_set_item( &self, @@ -599,8 +405,8 @@ impl MailboxSet for Server { } let parent_document_id = mailbox_parent_id - 1; - if let Some(fields) = self - .get_property::( + if let Some(mailbox_) = self + .get_property::>( ctx.account_id, Collection::Mailbox, parent_document_id, @@ -608,9 +414,10 @@ impl MailboxSet for Server { ) .await? { + let mailbox = mailbox_.unarchive().caused_by(trc::location!())?; if depth == 0 && ctx.is_shared - && !fields + && !mailbox .acls .effective_acl(ctx.access_token) .contains_any([Acl::CreateChild, Acl::Administer].into_iter()) @@ -620,7 +427,7 @@ impl MailboxSet for Server { ))); } - mailbox_parent_id = fields.parent_id; + mailbox_parent_id = mailbox.parent_id.into(); } else if ctx.mailbox_ids.contains(parent_document_id) { // Parent mailbox is probably created within the same request success = true; @@ -652,7 +459,12 @@ impl MailboxSet for Server { Collection::Mailbox, vec![Filter::eq( Property::Role, - changes.role.as_str().unwrap_or_default(), + changes + .role + .as_str() + .unwrap_or_default() + .as_bytes() + .to_vec(), )], ) .await? @@ -690,8 +502,8 @@ impl MailboxSet for Server { ctx.account_id, Collection::Mailbox, vec![ - Filter::eq(Property::Name, changes.name.as_str()), - Filter::eq(Property::ParentId, changes.parent_id), + Filter::eq(Property::Name, changes.name.as_bytes().to_vec()), + Filter::eq(Property::ParentId, changes.parent_id.serialize()), ], ) .await? diff --git a/crates/jmap/src/push/get.rs b/crates/jmap/src/push/get.rs index 76b6aa69..99d636f0 100644 --- a/crates/jmap/src/push/get.rs +++ b/crates/jmap/src/push/get.rs @@ -9,6 +9,7 @@ use common::{ auth::AccessToken, ipc::{StateEvent, UpdateSubscription}, }; +use email::push::ArchivedPushSubscription; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, types::{ @@ -20,8 +21,10 @@ use jmap_proto::{ }; use store::{ BitmapKey, ValueKey, - write::{ValueClass, now}, + write::{ArchivedValue, ValueClass, now}, }; +use trc::AddContext; +use utils::map::bitmap::Bitmap; use super::{EncryptionKeys, PushSubscription}; use std::future::Future; @@ -81,8 +84,8 @@ impl PushSubscriptionFetch for Server { response.not_found.push(id.into()); continue; } - let mut push = if let Some(push) = self - .get_property::( + let push_ = if let Some(push) = self + .get_property::>( account_id, Collection::PushSubscription, document_id, @@ -95,6 +98,7 @@ impl PushSubscriptionFetch for Server { response.not_found.push(id.into()); continue; }; + let push = push_.unarchive().caused_by(trc::location!())?; let mut result = Object::with_capacity(properties.len()); for property in &properties { match property { @@ -109,12 +113,12 @@ impl PushSubscriptionFetch for Server { Property::DeviceClientId => { result.append( Property::DeviceClientId, - std::mem::take(&mut push.device_client_id), + Value::from(&push.device_client_id), ); } Property::Types => { let mut types = Vec::new(); - for typ in push.types.into_iter() { + for typ in Bitmap::from(&push.types).into_iter() { types.push(Value::Text(typ.to_string())); } result.append(Property::Types, Value::List(types)); @@ -123,7 +127,9 @@ impl PushSubscriptionFetch for Server { if push.expires > 0 { result.append( Property::Expires, - Value::Date(UTCDate::from_timestamp(push.expires as i64)), + Value::Date( + UTCDate::from_timestamp(u64::from(push.expires) as i64), + ), ); } else { result.append(Property::Expires, Value::Null); @@ -160,7 +166,7 @@ impl PushSubscriptionFetch for Server { .core .storage .data - .get_value::(ValueKey { + .get_value::>(ValueKey { account_id, collection: Collection::PushSubscription.into(), document_id, @@ -172,7 +178,9 @@ impl PushSubscriptionFetch for Server { .into_err() .caused_by(trc::location!()) .document_id(document_id) - })?; + })? + .deserialize() + .caused_by(trc::location!())?; if subscription.expires > current_time { if subscription.verified { diff --git a/crates/jmap/src/push/set.rs b/crates/jmap/src/push/set.rs index 2358bb3b..57a44a53 100644 --- a/crates/jmap/src/push/set.rs +++ b/crates/jmap/src/push/set.rs @@ -6,7 +6,7 @@ use base64::{Engine, engine::general_purpose}; use common::{Server, auth::AccessToken}; -use email::push::{Keys, PushSubscription}; +use email::push::{ArchivedPushSubscription, Keys, PushSubscription}; use jmap_proto::{ error::set::SetError, method::set::{RequestArguments, SetRequest, SetResponse}, @@ -24,7 +24,7 @@ use std::future::Future; use store::{ Serialize, rand::{Rng, rng}, - write::{BatchBuilder, F_CLEAR, F_VALUE, now}, + write::{ArchivedValue, BatchBuilder, now}, }; use trc::AddContext; use utils::map::bitmap::Bitmap; @@ -106,7 +106,10 @@ impl PushSubscriptionSet for Server { .with_account_id(account_id) .with_collection(Collection::PushSubscription) .create_document() - .set(Property::Value, push.serialize()); + .set( + Property::Value, + push.serialize().caused_by(trc::location!())?, + ); let document_id = self .store() .write_expect_id(batch) @@ -133,7 +136,7 @@ impl PushSubscriptionSet for Server { // Obtain push subscription let document_id = id.document_id(); let mut push = if let Some(push) = self - .get_property::( + .get_property::>( account_id, Collection::PushSubscription, document_id, @@ -141,7 +144,7 @@ impl PushSubscriptionSet for Server { ) .await? { - push + push.deserialize().caused_by(trc::location!())? } else { response.not_updated.append(id, SetError::not_found()); continue 'update; @@ -163,7 +166,10 @@ impl PushSubscriptionSet for Server { .with_account_id(account_id) .with_collection(Collection::PushSubscription) .update_document(document_id) - .set(Property::Value, push.serialize()); + .set( + Property::Value, + push.serialize().caused_by(trc::location!())?, + ); self.store() .write(batch) .await @@ -181,7 +187,7 @@ impl PushSubscriptionSet for Server { .with_account_id(account_id) .with_collection(Collection::PushSubscription) .delete_document(document_id) - .value(Property::Value, (), F_VALUE | F_CLEAR); + .clear(Property::Value); self.store() .write(batch) .await diff --git a/crates/jmap/src/services/index.rs b/crates/jmap/src/services/index.rs index c0075b70..073d86c8 100644 --- a/crates/jmap/src/services/index.rs +++ b/crates/jmap/src/services/index.rs @@ -14,7 +14,7 @@ use directory::{ use email::message::{bayes::EmailBayesTrain, index::IndexMessageText, metadata::MessageMetadata}; use jmap_proto::types::{collection::Collection, property::Property}; use store::{ - IterateParams, Serialize, U32_LEN, U64_LEN, ValueKey, + IterateParams, SerializeInfallible, U32_LEN, U64_LEN, ValueKey, ahash::AHashMap, fts::index::FtsDocument, roaring::RoaringBitmap, diff --git a/crates/jmap/src/services/state.rs b/crates/jmap/src/services/state.rs index 0d1d4764..a25b28db 100644 --- a/crates/jmap/src/services/state.rs +++ b/crates/jmap/src/services/state.rs @@ -10,9 +10,9 @@ use std::{ }; use common::{ + Inner, Server, core::BuildServer, ipc::{PushSubscription, StateEvent, UpdateSubscription}, - Inner, Server, IPC_CHANNEL_BUFFER, }; use jmap_proto::types::{id::Id, state::StateChange, type_state::DataType}; use std::future::Future; @@ -85,9 +85,10 @@ pub fn spawn_state_manager(inner: Arc, mut change_rx: mpsc::Receiver result, Err(err) => { - trc::error!(err - .account_id(account_id) - .details("Failed to obtain access token.")); + trc::error!( + err.account_id(account_id) + .details("Failed to obtain access token.") + ); continue; } @@ -366,49 +367,18 @@ pub fn spawn_state_manager(inner: Arc, mut change_rx: mpsc::Receiver, - ) -> impl Future>> + Send; - fn update_push_subscriptions(&self, account_id: u32) -> impl Future + Send; } impl StateManager for Server { - async fn subscribe_state_manager( - &self, - account_id: u32, - types: Bitmap, - ) -> trc::Result> { - let (change_tx, change_rx) = mpsc::channel::(IPC_CHANNEL_BUFFER); - let state_tx = self.inner.ipc.state_tx.clone(); - - for event in [ - StateEvent::UpdateSharedAccounts { account_id }, - StateEvent::Subscribe { - account_id, - types, - tx: change_tx, - }, - ] { - state_tx.send(event).await.map_err(|err| { - trc::EventType::Server(trc::ServerEvent::ThreadError) - .reason(err) - .caused_by(trc::location!()) - })?; - } - - Ok(change_rx) - } - async fn update_push_subscriptions(&self, account_id: u32) -> bool { let push_subs = match self.fetch_push_subscriptions(account_id).await { Ok(push_subs) => push_subs, Err(err) => { - trc::error!(err - .account_id(account_id) - .details("Failed to fetch push subscriptions")); + trc::error!( + err.account_id(account_id) + .details("Failed to fetch push subscriptions") + ); return false; } }; diff --git a/crates/jmap/src/sieve/get.rs b/crates/jmap/src/sieve/get.rs index 54a3eaff..81693bd0 100644 --- a/crates/jmap/src/sieve/get.rs +++ b/crates/jmap/src/sieve/get.rs @@ -5,16 +5,18 @@ */ use common::Server; -use email::sieve::SieveScript; +use email::sieve::ArchivedSieveScript; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, types::{ + blob::{BlobId, BlobSection}, collection::Collection, property::Property, value::{Object, Value}, }, }; -use store::BlobClass; +use store::{BlobClass, write::ArchivedValue}; +use trc::AddContext; use crate::changes::state::StateManager; @@ -70,8 +72,8 @@ impl SieveScriptGet for Server { response.not_found.push(id.into()); continue; } - let mut sieve = if let Some(sieve) = self - .get_property::( + let sieve_ = if let Some(sieve) = self + .get_property::>( account_id, Collection::SieveScript, document_id, @@ -84,6 +86,7 @@ impl SieveScriptGet for Server { response.not_found.push(id.into()); continue; }; + let sieve = sieve_.unarchive().caused_by(trc::location!())?; let mut result = Object::with_capacity(properties.len()); for property in &properties { match property { @@ -91,18 +94,26 @@ impl SieveScriptGet for Server { result.append(Property::Id, Value::Id(id)); } Property::Name => { - result.append(Property::Name, Value::Text(std::mem::take(&mut sieve.name))); + result.append(Property::Name, Value::from(&sieve.name)); } Property::IsActive => { result.append(Property::IsActive, Value::Bool(sieve.is_active)); } Property::BlobId => { - let mut blob_id = sieve.blob_id.clone(); - blob_id.class = BlobClass::Linked { - account_id, - collection: Collection::SieveScript.into(), - document_id, + let blob_id = BlobId { + hash: (&sieve.blob_hash).into(), + class: BlobClass::Linked { + account_id, + collection: Collection::SieveScript.into(), + document_id, + }, + section: BlobSection { + size: u32::from(sieve.size) as usize, + ..Default::default() + } + .into(), }; + result.append(Property::BlobId, Value::BlobId(blob_id)); } property => { diff --git a/crates/jmap/src/sieve/query.rs b/crates/jmap/src/sieve/query.rs index 0e1c8cc9..a83dec72 100644 --- a/crates/jmap/src/sieve/query.rs +++ b/crates/jmap/src/sieve/query.rs @@ -12,7 +12,10 @@ use jmap_proto::{ types::{collection::Collection, property::Property}, }; use std::future::Future; -use store::query::{self}; +use store::{ + SerializeInfallible, + query::{self}, +}; use crate::JmapMethods; @@ -33,17 +36,21 @@ impl SieveScriptQuery for Server { for cond in std::mem::take(&mut request.filter) { match cond { - Filter::Name(name) => filters.push(query::Filter::has_text(Property::Name, &name)), - Filter::IsActive(is_active) => { - filters.push(query::Filter::eq(Property::IsActive, is_active as u32)) - } + Filter::Name(name) => filters.push(query::Filter::contains( + Property::Name, + name.to_lowercase().into_bytes(), + )), + Filter::IsActive(is_active) => filters.push(query::Filter::eq( + Property::IsActive, + (is_active as u32).serialize(), + )), Filter::And | Filter::Or | Filter::Not | Filter::Close => { filters.push(cond.into()); } other => { return Err(trc::JmapEvent::UnsupportedFilter .into_err() - .details(other.to_string())) + .details(other.to_string())); } } } @@ -72,7 +79,7 @@ impl SieveScriptQuery for Server { other => { return Err(trc::JmapEvent::UnsupportedSort .into_err() - .details(other.to_string())) + .details(other.to_string())); } }); } diff --git a/crates/jmap/src/sieve/set.rs b/crates/jmap/src/sieve/set.rs index 6942ffaa..fd5c9904 100644 --- a/crates/jmap/src/sieve/set.rs +++ b/crates/jmap/src/sieve/set.rs @@ -7,16 +7,19 @@ use common::{ Server, auth::{AccessToken, ResourceToken}, + storage::index::ObjectIndexBuilder, +}; +use email::sieve::{ + ArchivedSieveScript, SieveScript, activate::SieveScriptActivate, delete::SieveScriptDelete, }; -use email::sieve::SieveScript; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{SetRequest, SetResponse}, - object::{index::ObjectIndexBuilder, sieve::SetArguments}, + object::sieve::SetArguments, request::reference::MaybeReference, response::references::EvalObjectReferences, types::{ - blob::BlobId, + blob::{BlobId, BlobSection}, collection::Collection, id::Id, property::Property, @@ -29,7 +32,7 @@ use store::{ BlobClass, query::Filter, rand::{Rng, rng}, - write::{BatchBuilder, BlobOp, F_CLEAR, F_VALUE, assert::HashedValue, log::ChangeLogBuilder}, + write::{ArchivedValue, BatchBuilder, BlobOp, assert::HashedValue, log::ChangeLogBuilder}, }; use trc::AddContext; @@ -50,13 +53,6 @@ pub trait SieveScriptSet: Sync + Send { session: &HttpSessionData, ) -> impl Future> + Send; - fn sieve_script_delete( - &self, - resource_token: &ResourceToken, - document_id: u32, - fail_if_active: bool, - ) -> impl Future> + Send; - #[allow(clippy::type_complexity)] fn sieve_set_item( &self, @@ -67,12 +63,6 @@ pub trait SieveScriptSet: Sync + Send { ) -> impl Future< Output = trc::Result, Option>), SetError>>, > + Send; - - fn sieve_activate_script( - &self, - account_id: u32, - activate_id: Option, - ) -> impl Future>> + Send; } impl SieveScriptSet for Server { @@ -106,9 +96,10 @@ impl SieveScriptSet for Server { { Ok((mut builder, Some(blob))) => { // Store blob - let blob_id = &mut builder.changes_mut().unwrap().blob_id; - blob_id.hash = self.put_blob(account_id, &blob, false).await?.hash; - let mut blob_id = blob_id.clone(); + let sieve = &mut builder.changes_mut().unwrap(); + sieve.blob_hash = self.put_blob(account_id, &blob, false).await?.hash; + let blob_size = sieve.size as usize; + let blob_hash = sieve.blob_hash.clone(); // Increment tenant quota #[cfg(feature = "enterprise")] @@ -126,11 +117,12 @@ impl SieveScriptSet for Server { .create_document() .set( BlobOp::Link { - hash: blob_id.hash.clone(), + hash: blob_hash.clone(), }, Vec::new(), ) - .custom(builder); + .custom(builder) + .caused_by(trc::location!())?; let document_id = self .store() @@ -141,16 +133,26 @@ impl SieveScriptSet for Server { changes.log_insert(Collection::SieveScript, document_id); // Add result with updated blobId - blob_id.class = BlobClass::Linked { - account_id, - collection: Collection::SieveScript.into(), - document_id, - }; ctx.response.created.insert( id, Object::with_capacity(1) .with_property(Property::Id, Value::Id(document_id.into())) - .with_property(Property::BlobId, blob_id), + .with_property( + Property::BlobId, + BlobId { + hash: blob_hash, + class: BlobClass::Linked { + account_id, + collection: Collection::SieveScript.into(), + document_id, + }, + section: BlobSection { + size: blob_size, + ..Default::default() + } + .into(), + }, + ), ); } Err(err) => { @@ -182,7 +184,7 @@ impl SieveScriptSet for Server { // Obtain sieve script let document_id = id.document_id(); if let Some(sieve) = self - .get_property::>( + .get_property::>>( account_id, Collection::SieveScript, document_id, @@ -190,7 +192,8 @@ impl SieveScriptSet for Server { ) .await? { - let prev_blob_id = sieve.inner.blob_id.clone(); + let sieve = sieve.into_deserialized().caused_by(trc::location!())?; + let prev_blob_hash = sieve.inner.blob_hash.clone(); match self .sieve_set_item( @@ -211,9 +214,10 @@ impl SieveScriptSet for Server { let blob_id = if let Some(blob) = blob { // Store blob - let blob_id = &mut builder.changes_mut().unwrap().blob_id; - blob_id.hash = self.put_blob(account_id, &blob, false).await?.hash; - let blob_id = blob_id.clone(); + let sieve = &mut builder.changes_mut().unwrap(); + sieve.blob_hash = self.put_blob(account_id, &blob, false).await?.hash; + let blob_hash = sieve.blob_hash.clone(); + let blob_size = sieve.size as usize; // Update tenant quota #[cfg(feature = "enterprise")] @@ -226,22 +230,35 @@ impl SieveScriptSet for Server { // Update blobId batch .clear(BlobOp::Link { - hash: prev_blob_id.hash, + hash: prev_blob_hash, }) .set( BlobOp::Link { - hash: blob_id.hash.clone(), + hash: blob_hash.clone(), }, Vec::new(), ); - blob_id.into() + BlobId { + hash: blob_hash, + class: BlobClass::Linked { + account_id, + collection: Collection::SieveScript.into(), + document_id, + }, + section: BlobSection { + size: blob_size, + ..Default::default() + } + .into(), + } + .into() } else { None }; // Write record - batch.custom(builder); + batch.custom(builder).caused_by(trc::location!())?; if !batch.is_empty() { changes.log_update(Collection::SieveScript, document_id); @@ -342,61 +359,6 @@ impl SieveScriptSet for Server { Ok(ctx.response) } - async fn sieve_script_delete( - &self, - resource_token: &ResourceToken, - document_id: u32, - fail_if_active: bool, - ) -> trc::Result { - // Fetch record - let account_id = resource_token.account_id; - let obj = self - .get_property::>( - account_id, - Collection::SieveScript, - document_id, - Property::Value, - ) - .await? - .ok_or_else(|| { - trc::StoreEvent::NotFound - .into_err() - .caused_by(trc::location!()) - .document_id(document_id) - })?; - - // Make sure the script is not active - if fail_if_active && obj.inner.is_active { - return Ok(false); - } - - let blob_hash = obj.inner.blob_id.hash.clone(); - let mut builder = ObjectIndexBuilder::new().with_current(obj); - // Update tenant quota - #[cfg(feature = "enterprise")] - if self.core.is_enterprise_edition() { - if let Some(tenant) = resource_token.tenant { - builder.set_tenant_id(tenant.id); - } - } - - // Delete record - let mut batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_collection(Collection::SieveScript) - .delete_document(document_id) - .value(Property::EmailIds, (), F_VALUE | F_CLEAR) - .clear(BlobOp::Link { hash: blob_hash }) - .custom(builder); - - self.store() - .write(batch) - .await - .caused_by(trc::location!())?; - Ok(true) - } - #[allow(clippy::blocks_in_conditions)] async fn sieve_set_item( &self, @@ -449,7 +411,7 @@ impl SieveScriptSet for Server { .filter( ctx.resource_token.account_id, Collection::SieveScript, - vec![Filter::eq(Property::Name, &value)], + vec![Filter::eq(Property::Name, value.as_bytes().to_vec())], ) .await? .results @@ -522,7 +484,7 @@ impl SieveScriptSet for Server { // Compile script match self.core.sieve.untrusted_compiler.compile(&bytes) { Ok(script) => { - changes.blob_id = BlobId::default().with_section_size(bytes.len()); + changes.size = bytes.len() as u32; bytes.extend(bincode::serialize(&script).unwrap_or_default()); bytes.into() } @@ -561,98 +523,4 @@ impl SieveScriptSet for Server { blob_update, ))) } - - async fn sieve_activate_script( - &self, - account_id: u32, - mut activate_id: Option, - ) -> trc::Result> { - let mut changed_ids = Vec::new(); - // Find the currently active script - let mut active_ids = self - .filter( - account_id, - Collection::SieveScript, - vec![Filter::eq(Property::IsActive, 1u32)], - ) - .await? - .results; - - // Check if script is already active - if activate_id.is_some_and(|id| active_ids.remove(id)) { - if active_ids.is_empty() { - return Ok(changed_ids); - } else { - activate_id = None; - } - } - - // Prepare batch - let mut batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_collection(Collection::SieveScript); - - // Deactivate scripts - for document_id in active_ids { - if let Some(sieve) = self - .get_property::>( - account_id, - Collection::SieveScript, - document_id, - Property::Value, - ) - .await? - { - let mut new_sieve = sieve.inner.clone(); - new_sieve.is_active = false; - batch - .update_document(document_id) - .value(Property::EmailIds, (), F_VALUE | F_CLEAR) - .custom( - ObjectIndexBuilder::new() - .with_changes(new_sieve) - .with_current(sieve), - ); - changed_ids.push((document_id, false)); - } - } - - // Activate script - if let Some(document_id) = activate_id { - if let Some(sieve) = self - .get_property::>( - account_id, - Collection::SieveScript, - document_id, - Property::Value, - ) - .await? - { - let mut new_sieve = sieve.inner.clone(); - new_sieve.is_active = true; - batch.update_document(document_id).custom( - ObjectIndexBuilder::new() - .with_changes(new_sieve) - .with_current(sieve), - ); - changed_ids.push((document_id, true)); - } - } - - // Write changes - if !changed_ids.is_empty() { - match self.core.storage.data.write(batch.build()).await { - Ok(_) => (), - Err(err) if err.is_assertion_failure() => { - return Ok(vec![]); - } - Err(err) => { - return Err(err.caused_by(trc::location!())); - } - } - } - - Ok(changed_ids) - } } diff --git a/crates/jmap/src/submission/get.rs b/crates/jmap/src/submission/get.rs index 20345d5d..a55f590f 100644 --- a/crates/jmap/src/submission/get.rs +++ b/crates/jmap/src/submission/get.rs @@ -5,7 +5,10 @@ */ use common::Server; -use email::submission::{Address, Delivered, EmailSubmission, Envelope, UndoStatus}; +use email::submission::{ + ArchivedAddress, ArchivedEmailSubmission, ArchivedEnvelope, Delivered, DeliveryStatus, + UndoStatus, +}; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, types::{ @@ -18,6 +21,9 @@ use jmap_proto::{ }; use smtp::queue::{self, spool::SmtpSpool}; use std::future::Future; +use store::{rkyv::option::ArchivedOption, write::ArchivedValue}; +use trc::AddContext; +use utils::map::vec_map::VecMap; use crate::changes::state::StateManager; @@ -77,8 +83,8 @@ impl EmailSubmissionGet for Server { response.not_found.push(id.into()); continue; } - let mut submission = if let Some(submission) = self - .get_property::( + let submission_ = if let Some(submission) = self + .get_property::>( account_id, Collection::EmailSubmission, document_id, @@ -91,33 +97,40 @@ impl EmailSubmissionGet for Server { response.not_found.push(id.into()); continue; }; + let submission = submission_.unarchive().caused_by(trc::location!())?; // Obtain queueId - if let Some(queue_id) = submission.queue_id { + let mut delivery_status = submission + .delivery_status + .iter() + .map(|(k, v)| (k.to_string(), DeliveryStatus::from(v))) + .collect::>(); + let mut is_pending = false; + if let Some(queue_id) = submission.queue_id.as_ref().map(u64::from) { if let Some(mut queued_message) = self.read_message(queue_id).await { for rcpt in std::mem::take(&mut queued_message.recipients) { - let rcpt_status = submission - .delivery_status - .get_mut_or_insert(rcpt.address_lcase); - rcpt_status.delivered = match &rcpt.status { - queue::Status::Scheduled | queue::Status::TemporaryFailure(_) => { - Delivered::Queued - } - queue::Status::Completed(_) => Delivered::Yes, - queue::Status::PermanentFailure(_) => Delivered::No, - }; - rcpt_status.smtp_reply = match &rcpt.status { - queue::Status::Completed(reply) => { - reply.response.to_string().replace('\n', " ") - } - queue::Status::TemporaryFailure(reply) - | queue::Status::PermanentFailure(reply) => { - reply.response.to_string().replace('\n', " ") - } - queue::Status::Scheduled => "250 2.1.5 Queued".to_string(), + *delivery_status.get_mut_or_insert(rcpt.address_lcase) = DeliveryStatus { + smtp_reply: match &rcpt.status { + queue::Status::Completed(reply) => { + reply.response.to_string().replace('\n', " ") + } + queue::Status::TemporaryFailure(reply) + | queue::Status::PermanentFailure(reply) => { + reply.response.to_string().replace('\n', " ") + } + queue::Status::Scheduled => "250 2.1.5 Queued".to_string(), + }, + delivered: match &rcpt.status { + queue::Status::Scheduled | queue::Status::TemporaryFailure(_) => { + Delivered::Queued + } + queue::Status::Completed(_) => Delivered::Yes, + queue::Status::PermanentFailure(_) => Delivered::No, + }, + displayed: false, }; } - submission.undo_status = UndoStatus::Pending; + is_pending = true; } } @@ -126,11 +139,9 @@ impl EmailSubmissionGet for Server { let value = match property { Property::Id => Value::Id(id), Property::DeliveryStatus => { - let mut status = Object::with_capacity(submission.delivery_status.len()); + let mut status = Object::with_capacity(delivery_status.len()); - for (rcpt, delivery_status) in - std::mem::take(&mut submission.delivery_status) - { + for (rcpt, delivery_status) in std::mem::take(&mut delivery_status) { status.set( Property::_T(rcpt), Object::with_capacity(3) @@ -145,17 +156,25 @@ impl EmailSubmissionGet for Server { Value::Object(status) } - Property::UndoStatus => { - Value::Text(submission.undo_status.as_str().to_string()) - } - Property::EmailId => { - Value::Id(Id::from_parts(submission.thread_id, submission.email_id)) - } - Property::IdentityId => Value::Id(Id::from(submission.identity_id)), - Property::ThreadId => Value::Id(Id::from(submission.thread_id)), - Property::Envelope => build_envelope(std::mem::take(&mut submission.envelope)), + Property::UndoStatus => Value::Text( + { + if is_pending { + UndoStatus::Pending.as_str() + } else { + submission.undo_status.as_str() + } + } + .to_string(), + ), + Property::EmailId => Value::Id(Id::from_parts( + u32::from(submission.thread_id), + u32::from(submission.email_id), + )), + Property::IdentityId => Value::Id(Id::from(u32::from(submission.identity_id))), + Property::ThreadId => Value::Id(Id::from(u32::from(submission.thread_id))), + Property::Envelope => build_envelope(&submission.envelope), Property::SendAt => { - Value::Date(UTCDate::from_timestamp(submission.send_at as i64)) + Value::Date(UTCDate::from_timestamp(u64::from(submission.send_at) as i64)) } Property::MdnBlobIds | Property::DsnBlobIds => Value::List(vec![]), _ => Value::Null, @@ -170,26 +189,26 @@ impl EmailSubmissionGet for Server { } } -fn build_envelope(envelope: Envelope) -> Value { +fn build_envelope(envelope: &ArchivedEnvelope) -> Value { Object::with_capacity(2) - .with_property(Property::MailFrom, build_address(envelope.mail_from)) + .with_property(Property::MailFrom, build_address(&envelope.mail_from)) .with_property( Property::RcptTo, - Value::List(envelope.rcpt_to.into_iter().map(build_address).collect()), + Value::List(envelope.rcpt_to.iter().map(build_address).collect()), ) .into() } -fn build_address(envelope: Address) -> Value { +fn build_address(envelope: &ArchivedAddress) -> Value { Object::with_capacity(2) - .with_property(Property::Email, Value::Text(envelope.email)) + .with_property(Property::Email, Value::Text(envelope.email.to_string())) .with_property( Property::Parameters, - if let Some(params) = envelope.parameters { + if let ArchivedOption::Some(params) = &envelope.parameters { Value::Object(Object( params - .into_iter() - .map(|(k, v)| (Property::_T(k), v.into())) + .iter() + .map(|(k, v)| (Property::_T(k.to_string()), v.into())) .collect(), )) } else { diff --git a/crates/jmap/src/submission/query.rs b/crates/jmap/src/submission/query.rs index 9ce3a2cb..c059a873 100644 --- a/crates/jmap/src/submission/query.rs +++ b/crates/jmap/src/submission/query.rs @@ -13,7 +13,10 @@ use jmap_proto::{ types::{collection::Collection, property::Property}, }; use std::future::Future; -use store::query::{self}; +use store::{ + SerializeInfallible, + query::{self}, +}; use crate::JmapMethods; @@ -37,21 +40,27 @@ impl EmailSubmissionQuery for Server { Filter::IdentityIds(ids) => { filters.push(query::Filter::Or); for id in ids { - filters.push(query::Filter::eq(Property::IdentityId, id.document_id())); + filters.push(query::Filter::eq( + Property::IdentityId, + id.document_id().serialize(), + )); } filters.push(query::Filter::End); } Filter::EmailIds(ids) => { filters.push(query::Filter::Or); for id in ids { - filters.push(query::Filter::eq(Property::EmailId, id.id())); + filters.push(query::Filter::eq(Property::EmailId, id.id().serialize())); } filters.push(query::Filter::End); } Filter::ThreadIds(ids) => { filters.push(query::Filter::Or); for id in ids { - filters.push(query::Filter::eq(Property::ThreadId, id.document_id())); + filters.push(query::Filter::eq( + Property::ThreadId, + id.document_id().serialize(), + )); } filters.push(query::Filter::End); } @@ -59,15 +68,16 @@ impl EmailSubmissionQuery for Server { Property::UndoStatus, UndoStatus::parse(&undo_status) .unwrap_or(UndoStatus::Pending) - .as_index(), + .as_index() + .serialize(), )), Filter::Before(before) => filters.push(query::Filter::lt( Property::SendAt, - before.timestamp() as u64, + (before.timestamp() as u64).serialize(), )), Filter::After(after) => filters.push(query::Filter::gt( Property::SendAt, - after.timestamp() as u64, + (after.timestamp() as u64).serialize(), )), Filter::And | Filter::Or | Filter::Not | Filter::Close => { filters.push(cond.into()); diff --git a/crates/jmap/src/submission/set.rs b/crates/jmap/src/submission/set.rs index abbc978e..cbc48ae5 100644 --- a/crates/jmap/src/submission/set.rs +++ b/crates/jmap/src/submission/set.rs @@ -9,16 +9,19 @@ use std::{collections::HashMap, sync::Arc}; use common::{ Server, listener::{ServerInstance, stream::NullIo}, + storage::index::ObjectIndexBuilder, }; use email::{ - identity::Identity, + identity::ArchivedIdentity, message::metadata::MessageMetadata, - submission::{Address, Delivered, DeliveryStatus, EmailSubmission, UndoStatus}, + submission::{ + Address, ArchivedEmailSubmission, Delivered, DeliveryStatus, EmailSubmission, UndoStatus, + }, }; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{self, SetRequest, SetResponse}, - object::{email_submission::SetArguments, index::ObjectIndexBuilder}, + object::email_submission::SetArguments, request::{ Call, RequestMethod, method::{MethodFunction, MethodName, MethodObject}, @@ -38,7 +41,9 @@ use smtp::{ queue::spool::SmtpSpool, }; use smtp_proto::{MailFrom, RcptTo, request::parser::Rfc5321Parser}; -use store::write::{BatchBuilder, Bincode, assert::HashedValue, log::ChangeLogBuilder, now}; +use store::write::{ + ArchivedValue, BatchBuilder, Bincode, assert::HashedValue, log::ChangeLogBuilder, now, +}; use trc::AddContext; use utils::{map::vec_map::VecMap, sanitize_email}; @@ -94,7 +99,8 @@ impl EmailSubmissionSet for Server { .with_account_id(account_id) .with_collection(Collection::EmailSubmission) .create_document() - .custom(ObjectIndexBuilder::new().with_changes(submission)); + .custom(ObjectIndexBuilder::new().with_changes(submission)) + .caused_by(trc::location!())?; let document_id = self .store() .write_expect_id(batch) @@ -120,7 +126,7 @@ impl EmailSubmissionSet for Server { // Obtain submission let document_id = id.document_id(); let submission = if let Some(submission) = self - .get_property::>( + .get_property::>>( account_id, Collection::EmailSubmission, document_id, @@ -128,7 +134,7 @@ impl EmailSubmissionSet for Server { ) .await? { - submission + submission.into_deserialized().caused_by(trc::location!())? } else { response.not_updated.append(id, SetError::not_found()); continue 'update; @@ -183,7 +189,8 @@ impl EmailSubmissionSet for Server { ObjectIndexBuilder::new() .with_current(submission) .with_changes(new_submission), - ); + ) + .caused_by(trc::location!())?; self.store() .write(batch) .await @@ -221,7 +228,7 @@ impl EmailSubmissionSet for Server { for id in will_destroy { let document_id = id.document_id(); if let Some(submission) = self - .get_property::>( + .get_property::>>( account_id, Collection::EmailSubmission, document_id, @@ -235,7 +242,12 @@ impl EmailSubmissionSet for Server { .with_account_id(account_id) .with_collection(Collection::EmailSubmission) .delete_document(document_id) - .custom(ObjectIndexBuilder::new().with_current(submission)); + .custom( + ObjectIndexBuilder::new().with_current( + submission.into_deserialized().caused_by(trc::location!())?, + ), + ) + .caused_by(trc::location!())?; self.store() .write(batch) .await @@ -317,9 +329,12 @@ impl EmailSubmissionSet for Server { instance: &Arc, object: Object, ) -> trc::Result> { - let mut submission = EmailSubmission::default(); - submission.email_id = u32::MAX; - submission.identity_id = u32::MAX; + let mut submission = EmailSubmission { + email_id: u32::MAX, + identity_id: u32::MAX, + thread_id: u32::MAX, + ..Default::default() + }; let mut mail_from = None; let mut rcpt_to: Vec> = Vec::new(); @@ -444,7 +459,7 @@ impl EmailSubmissionSet for Server { // Fetch identity's mailFrom let identity_mail_from = if let Some(identity) = self - .get_property::( + .get_property::>( account_id, Collection::Identity, submission.identity_id, @@ -452,7 +467,11 @@ impl EmailSubmissionSet for Server { ) .await? { - identity.email + identity + .unarchive() + .caused_by(trc::location!())? + .email + .to_string() } else { return Ok(Err(SetError::invalid_properties() .with_property(Property::IdentityId) diff --git a/crates/jmap/src/vacation/get.rs b/crates/jmap/src/vacation/get.rs index 1ed860b7..bccecaf2 100644 --- a/crates/jmap/src/vacation/get.rs +++ b/crates/jmap/src/vacation/get.rs @@ -5,7 +5,7 @@ */ use common::Server; -use email::sieve::SieveScript; +use email::sieve::ArchivedSieveScript; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, request::reference::MaybeReference, @@ -19,7 +19,8 @@ use jmap_proto::{ }, }; use std::future::Future; -use store::query::Filter; +use store::{query::Filter, write::ArchivedValue}; +use trc::AddContext; use crate::{JmapMethods, changes::state::StateManager}; @@ -79,8 +80,8 @@ impl VacationResponseGet for Server { }; if do_get { if let Some(document_id) = self.get_vacation_sieve_script_id(account_id).await? { - if let Some(mut obj) = self - .get_property::( + if let Some(sieve_) = self + .get_property::>( account_id, Collection::SieveScript, document_id, @@ -88,6 +89,8 @@ impl VacationResponseGet for Server { ) .await? { + let sieve = sieve_.unarchive().caused_by(trc::location!())?; + let vacation = sieve.vacation_response.as_ref(); let mut result = Object::with_capacity(properties.len()); for property in &properties { match property { @@ -95,46 +98,48 @@ impl VacationResponseGet for Server { result.append(Property::Id, Value::Id(Id::singleton())); } Property::IsEnabled => { - result.append(Property::IsEnabled, obj.is_active); + result.append(Property::IsEnabled, sieve.is_active); } Property::FromDate => { result.append( Property::FromDate, - obj.vacation_response.as_mut().and_then(|r| { - r.from_date.take().map(UTCDate::from).map(Value::Date) + vacation.and_then(|r| { + r.from_date + .as_ref() + .map(u64::from) + .map(UTCDate::from) + .map(Value::Date) }), ); } Property::ToDate => { result.append( Property::ToDate, - obj.vacation_response.as_mut().and_then(|r| { - r.to_date.take().map(UTCDate::from).map(Value::Date) + vacation.and_then(|r| { + r.to_date + .as_ref() + .map(u64::from) + .map(UTCDate::from) + .map(Value::Date) }), ); } Property::Subject => { result.append( Property::Subject, - obj.vacation_response - .as_mut() - .and_then(|r| r.subject.take().map(Value::from)), + vacation.and_then(|r| r.subject.as_ref().map(Value::from)), ); } Property::TextBody => { result.append( Property::TextBody, - obj.vacation_response - .as_mut() - .and_then(|r| r.text_body.take().map(Value::from)), + vacation.and_then(|r| r.text_body.as_ref().map(Value::from)), ); } Property::HtmlBody => { result.append( Property::HtmlBody, - obj.vacation_response - .as_mut() - .and_then(|r| r.html_body.take().map(Value::from)), + vacation.and_then(|r| r.html_body.as_ref().map(Value::from)), ); } property => { @@ -158,7 +163,7 @@ impl VacationResponseGet for Server { self.filter( account_id, Collection::SieveScript, - vec![Filter::eq(Property::Name, "vacation")], + vec![Filter::eq(Property::Name, "vacation".as_bytes().to_vec())], ) .await .map(|r| r.results.min()) diff --git a/crates/jmap/src/vacation/set.rs b/crates/jmap/src/vacation/set.rs index b98aad10..55999acc 100644 --- a/crates/jmap/src/vacation/set.rs +++ b/crates/jmap/src/vacation/set.rs @@ -6,15 +6,16 @@ use std::borrow::Cow; -use common::{Server, auth::AccessToken}; -use email::sieve::{SieveScript, VacationResponse}; +use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; +use email::sieve::{ + ArchivedSieveScript, SieveScript, VacationResponse, activate::SieveScriptActivate, + delete::SieveScriptDelete, +}; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{RequestArguments, SetRequest, SetResponse}, - object::index::ObjectIndexBuilder, response::references::EvalObjectReferences, types::{ - blob::BlobId, collection::Collection, date::UTCDate, id::Id, @@ -26,13 +27,13 @@ use mail_builder::MessageBuilder; use mail_parser::decoders::html::html_to_text; use std::future::Future; use store::write::{ - BatchBuilder, BlobOp, F_CLEAR, F_VALUE, + ArchivedValue, BatchBuilder, BlobOp, assert::HashedValue, log::{Changes, LogInsert}, }; use trc::AddContext; -use crate::{JmapMethods, sieve::set::SieveScriptSet}; +use crate::JmapMethods; use super::get::VacationResponseGet; @@ -218,7 +219,7 @@ impl VacationResponseSet for Server { let mut obj = if let Some(document_id) = document_id { let prev_sieve = self - .get_property::>( + .get_property::>>( account_id, Collection::SieveScript, document_id, @@ -230,6 +231,7 @@ impl VacationResponseSet for Server { .into_err() .caused_by(trc::location!()) })?; + let prev_sieve = prev_sieve.into_deserialized().caused_by(trc::location!())?; was_active = prev_sieve.inner.is_active; let mut sieve = prev_sieve.inner.clone(); sieve.vacation_response = vacation.into(); @@ -242,7 +244,8 @@ impl VacationResponseSet for Server { ObjectIndexBuilder::new().with_changes(SieveScript { name: "vacation".into(), is_active, - blob_id: Default::default(), + blob_hash: Default::default(), + size: 0, vacation_response: vacation.into(), }) }; @@ -251,7 +254,7 @@ impl VacationResponseSet for Server { if let Some(document_id) = document_id { batch .update_document(document_id) - .value(Property::EmailIds, (), F_VALUE | F_CLEAR) + .clear(Property::EmailIds) .log(Changes::update([document_id])); } else { batch.create_document().log(LogInsert()); @@ -268,13 +271,13 @@ impl VacationResponseSet for Server { ) .await? .hash; - let blob_id = &mut obj.changes_mut().unwrap().blob_id; - blob_id.hash = hash; + let sieve = &mut obj.changes_mut().unwrap(); + sieve.blob_hash = hash; // Link blob batch.set( BlobOp::Link { - hash: blob_id.hash.clone(), + hash: sieve.blob_hash.clone(), }, Vec::new(), ); @@ -282,7 +285,7 @@ impl VacationResponseSet for Server { // Unlink previous blob if let Some(current) = obj.current() { batch.clear(BlobOp::Link { - hash: current.inner.blob_id.hash.clone(), + hash: current.inner.blob_hash.clone(), }); } @@ -296,7 +299,7 @@ impl VacationResponseSet for Server { }; // Write changes - batch.custom(obj); + batch.custom(obj).caused_by(trc::location!())?; let document_id = if !batch.is_empty() { let ids = self .store() @@ -458,7 +461,7 @@ impl VacationResponseSet for Server { match self.core.sieve.untrusted_compiler.compile(&script) { Ok(compiled_script) => { // Update blob length - obj.blob_id = BlobId::default().with_section_size(script.len()); + obj.size = script.len() as u32; // Serialize script script.extend(bincode::serialize(&compiled_script).unwrap_or_default()); diff --git a/crates/jmap/src/websocket/stream.rs b/crates/jmap/src/websocket/stream.rs index 1a8b79ea..c820586f 100644 --- a/crates/jmap/src/websocket/stream.rs +++ b/crates/jmap/src/websocket/stream.rs @@ -6,7 +6,7 @@ use std::{sync::Arc, time::Instant}; -use common::{auth::AccessToken, Server}; +use common::{Server, auth::AccessToken}; use futures_util::{SinkExt, StreamExt}; use hyper::upgrade::Upgraded; use hyper_util::rt::TokioIo; @@ -22,12 +22,9 @@ use trc::JmapEvent; use tungstenite::Message; use utils::map::bitmap::Bitmap; -use crate::{ - api::{ - http::{HttpSessionData, ToRequestError}, - request::RequestHandler, - }, - services::state::StateManager, +use crate::api::{ + http::{HttpSessionData, ToRequestError}, + request::RequestHandler, }; use std::future::Future; @@ -69,9 +66,10 @@ impl WebSocketHandler for Server { { Ok(change_rx) => change_rx, Err(err) => { - trc::error!(err - .details("Failed to subscribe to state manager") - .span_id(session.session_id)); + trc::error!( + err.details("Failed to subscribe to state manager") + .span_id(session.session_id) + ); let _ = stream .send(Message::Text( diff --git a/crates/main/Cargo.toml b/crates/main/Cargo.toml index a5a6160c..f53973c4 100644 --- a/crates/main/Cargo.toml +++ b/crates/main/Cargo.toml @@ -25,6 +25,7 @@ pop3 = { path = "../pop3" } spam-filter = { path = "../spam-filter" } managesieve = { path = "../managesieve" } common = { path = "../common" } +email = { path = "../email" } directory = { path = "../directory" } trc = { path = "../trc" } utils = { path = "../utils" } @@ -51,4 +52,5 @@ enterprise = [ "jmap/enterprise", "store/enterprise", "managesieve/enterprise", "directory/enterprise", + "email/enterprise", "spam-filter/enterprise" ] diff --git a/crates/managesieve/Cargo.toml b/crates/managesieve/Cargo.toml index 297ac005..df8a66e3 100644 --- a/crates/managesieve/Cargo.toml +++ b/crates/managesieve/Cargo.toml @@ -7,7 +7,6 @@ resolver = "2" [dependencies] imap_proto = { path = "../imap-proto" } imap = { path = "../imap" } -jmap = { path = "../jmap" } jmap_proto = { path = "../jmap-proto" } directory = { path = "../directory" } common = { path = "../common" } diff --git a/crates/managesieve/src/core/client.rs b/crates/managesieve/src/core/client.rs index 0fe1b703..bd78c7d9 100644 --- a/crates/managesieve/src/core/client.rs +++ b/crates/managesieve/src/core/client.rs @@ -5,8 +5,8 @@ */ use common::{ - listener::{SessionResult, SessionStream}, KV_RATE_LIMIT_IMAP, + listener::{SessionResult, SessionStream}, }; use imap_proto::receiver::{self, Request}; use jmap_proto::types::{collection::Collection, property::Property}; @@ -70,9 +70,10 @@ impl Session { } Ok(false) => {} Err(err) => { - trc::error!(err - .span_id(self.session_id) - .details("Failed to check for fail2ban")); + trc::error!( + err.span_id(self.session_id) + .details("Failed to check for fail2ban") + ); } } } @@ -278,7 +279,7 @@ impl Session { .filter( account_id, Collection::SieveScript, - vec![Filter::eq(Property::Name, name)], + vec![Filter::eq(Property::Name, name.as_bytes().to_vec())], ) .await .caused_by(trc::location!()) diff --git a/crates/managesieve/src/op/deletescript.rs b/crates/managesieve/src/op/deletescript.rs index 2a2e5b51..b13ae6f6 100644 --- a/crates/managesieve/src/op/deletescript.rs +++ b/crates/managesieve/src/op/deletescript.rs @@ -8,8 +8,8 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; +use email::sieve::delete::SieveScriptDelete; use imap_proto::receiver::Request; -use jmap::sieve::set::SieveScriptSet; use jmap_proto::types::collection::Collection; use store::write::log::ChangeLogBuilder; use trc::AddContext; diff --git a/crates/managesieve/src/op/getscript.rs b/crates/managesieve/src/op/getscript.rs index 17eb9f58..cb7378cc 100644 --- a/crates/managesieve/src/op/getscript.rs +++ b/crates/managesieve/src/op/getscript.rs @@ -8,11 +8,12 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; -use email::sieve::SieveScript; +use email::sieve::ArchivedSieveScript; use imap_proto::receiver::Request; -use jmap::blob::download::BlobDownload; -use jmap_proto::types::{collection::Collection, property::Property}; +use jmap_proto::types::{blob::BlobSection, collection::Collection, property::Property}; +use store::write::ArchivedValue; use trc::AddContext; +use utils::BlobHash; use crate::core::{Command, ResponseCode, Session, StatusResponse}; @@ -34,9 +35,9 @@ impl Session { })?; let account_id = self.state.access_token().primary_id(); let document_id = self.get_script_id(account_id, &name).await?; - let (blob_section, blob_hash) = self + let sieve_ = self .server - .get_property::( + .get_property::>( account_id, Collection::SieveScript, document_id, @@ -44,16 +45,23 @@ impl Session { ) .await .caused_by(trc::location!())? - .and_then(|id| (id.blob_id.section?, id.blob_id.hash).into()) .ok_or_else(|| { trc::ManageSieveEvent::Error .into_err() .details("Script not found") .code(ResponseCode::NonExistent) })?; + let sieve = sieve_.unarchive().caused_by(trc::location!())?; + let blob_size = u32::from(sieve.size) as usize; let script = self .server - .get_blob_section(&blob_hash, &blob_section) + .get_blob_section( + &BlobHash::from(&sieve.blob_hash), + &BlobSection { + size: blob_size, + ..Default::default() + }, + ) .await .caused_by(trc::location!())? .ok_or_else(|| { @@ -62,11 +70,11 @@ impl Session { .details("Script blob not found") .code(ResponseCode::NonExistent) })?; - debug_assert_eq!(script.len(), blob_section.size); + debug_assert_eq!(script.len(), blob_size); let mut response = Vec::with_capacity(script.len() + 32); response.push(b'{'); - response.extend_from_slice(blob_section.size.to_string().as_bytes()); + response.extend_from_slice(blob_size.to_string().as_bytes()); response.extend_from_slice(b"}\r\n"); response.extend(script); response.extend_from_slice(b"\r\n"); diff --git a/crates/managesieve/src/op/listscripts.rs b/crates/managesieve/src/op/listscripts.rs index 300e43f0..bfe4a483 100644 --- a/crates/managesieve/src/op/listscripts.rs +++ b/crates/managesieve/src/op/listscripts.rs @@ -8,8 +8,9 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; -use email::sieve::SieveScript; +use email::sieve::ArchivedSieveScript; use jmap_proto::types::{collection::Collection, property::Property}; +use store::write::ArchivedValue; use trc::AddContext; use crate::core::{Session, StatusResponse}; @@ -36,9 +37,9 @@ impl Session { let count = document_ids.len(); for document_id in document_ids { - if let Some(script) = self + if let Some(script_) = self .server - .get_property::( + .get_property::>( account_id, Collection::SieveScript, document_id, @@ -47,6 +48,7 @@ impl Session { .await .caused_by(trc::location!())? { + let script = script_.unarchive().caused_by(trc::location!())?; response.push(b'\"'); for ch in script.name.as_bytes() { if [b'\\', b'\"'].contains(ch) { diff --git a/crates/managesieve/src/op/putscript.rs b/crates/managesieve/src/op/putscript.rs index 5f3ff7d5..952b5cb3 100644 --- a/crates/managesieve/src/op/putscript.rs +++ b/crates/managesieve/src/op/putscript.rs @@ -6,20 +6,16 @@ use std::time::Instant; -use common::listener::SessionStream; +use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; -use email::sieve::SieveScript; +use email::sieve::{ArchivedSieveScript, SieveScript}; use imap_proto::receiver::Request; -use jmap::JmapMethods; -use jmap_proto::{ - object::index::ObjectIndexBuilder, - types::{blob::BlobId, collection::Collection, property::Property}, -}; +use jmap_proto::types::{blob::BlobId, collection::Collection, property::Property}; use sieve::compiler::ErrorType; use store::{ BlobClass, query::Filter, - write::{BatchBuilder, BlobOp, DirectoryClass, assert::HashedValue, log::LogInsert}, + write::{ArchivedValue, BatchBuilder, BlobOp, assert::HashedValue, log::LogInsert}, }; use trc::AddContext; @@ -105,7 +101,7 @@ impl Session { // Obtain script values let script = self .server - .get_property::>( + .get_property::>>( account_id, Collection::SieveScript, document_id, @@ -118,7 +114,9 @@ impl Session { .into_err() .details("Script not found") .code(ResponseCode::NonExistent) - })?; + })? + .into_deserialized() + .caused_by(trc::location!())?; // Write script blob let blob_id = BlobId::new( @@ -134,12 +132,12 @@ impl Session { }, ) .with_section_size(script_size as usize); - let prev_blob_id_hash = script.inner.blob_id.hash.clone(); - let blob_id_hash = blob_id.hash.clone(); + let prev_blob_hash = script.inner.blob_hash.clone(); + let blob_hash = blob_id.hash.clone(); // Write record let mut obj = ObjectIndexBuilder::new() - .with_changes(script.inner.clone().with_blob_id(blob_id)) + .with_changes(script.inner.clone().with_blob_hash(blob_hash.clone())) .with_current(script); // Update tenant quota @@ -156,10 +154,11 @@ impl Session { .with_collection(Collection::SieveScript) .update_document(document_id) .clear(BlobOp::Link { - hash: prev_blob_id_hash, + hash: prev_blob_hash, }) - .set(BlobOp::Link { hash: blob_id_hash }, Vec::new()) - .custom(obj); + .set(BlobOp::Link { hash: blob_hash }, Vec::new()) + .custom(obj) + .caused_by(trc::location!())?; self.server .store() @@ -177,23 +176,18 @@ impl Session { ); } else { // Write script blob - let blob_id = BlobId::new( - self.server - .put_blob(account_id, &script_bytes, false) - .await? - .hash, - BlobClass::Linked { - account_id, - collection: Collection::SieveScript.into(), - document_id: 0, - }, - ) - .with_section_size(script_size as usize); - let blob_id_hash = blob_id.hash.clone(); + let blob_hash = self + .server + .put_blob(account_id, &script_bytes, false) + .await? + .hash; // Write record - let mut obj = ObjectIndexBuilder::new() - .with_changes(SieveScript::new(name.clone(), blob_id).with_is_active(false)); + let mut obj = ObjectIndexBuilder::new().with_changes( + SieveScript::new(name.clone(), blob_hash.clone()) + .with_is_active(false) + .with_size(script_size as u32), + ); // Update tenant quota #[cfg(feature = "enterprise")] @@ -209,9 +203,9 @@ impl Session { .with_collection(Collection::SieveScript) .create_document() .log(LogInsert()) - .add(DirectoryClass::UsedQuota(account_id), script_size) - .set(BlobOp::Link { hash: blob_id_hash }, Vec::new()) - .custom(obj); + .set(BlobOp::Link { hash: blob_hash }, Vec::new()) + .custom(obj) + .caused_by(trc::location!())?; let assigned_ids = self .server @@ -248,10 +242,11 @@ impl Session { } else { Ok(self .server + .store() .filter( account_id, Collection::SieveScript, - vec![Filter::eq(Property::Name, name)], + vec![Filter::eq(Property::Name, name.to_lowercase().into_bytes())], ) .await .caused_by(trc::location!())? diff --git a/crates/managesieve/src/op/renamescript.rs b/crates/managesieve/src/op/renamescript.rs index 243b369d..737d6993 100644 --- a/crates/managesieve/src/op/renamescript.rs +++ b/crates/managesieve/src/op/renamescript.rs @@ -6,15 +6,12 @@ use std::time::Instant; -use common::listener::SessionStream; +use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; -use email::sieve::SieveScript; +use email::sieve::ArchivedSieveScript; use imap_proto::receiver::Request; -use jmap_proto::{ - object::index::ObjectIndexBuilder, - types::{collection::Collection, property::Property}, -}; -use store::write::{BatchBuilder, assert::HashedValue, log::ChangeLogBuilder}; +use jmap_proto::types::{collection::Collection, property::Property}; +use store::write::{ArchivedValue, BatchBuilder, assert::HashedValue, log::ChangeLogBuilder}; use trc::AddContext; use crate::core::{Command, ResponseCode, Session, StatusResponse}; @@ -63,7 +60,7 @@ impl Session { // Obtain script values let script = self .server - .get_property::>( + .get_property::>>( account_id, Collection::SieveScript, document_id, @@ -76,7 +73,9 @@ impl Session { .into_err() .details("Script not found") .code(ResponseCode::NonExistent) - })?; + })? + .into_deserialized() + .caused_by(trc::location!())?; // Write record let mut batch = BatchBuilder::new(); @@ -88,7 +87,8 @@ impl Session { ObjectIndexBuilder::new() .with_changes(script.inner.clone().with_name(new_name.clone())) .with_current(script), - ); + ) + .caused_by(trc::location!())?; if !batch.is_empty() { self.server .store() diff --git a/crates/managesieve/src/op/setactive.rs b/crates/managesieve/src/op/setactive.rs index e4a8c7d5..9e7f0358 100644 --- a/crates/managesieve/src/op/setactive.rs +++ b/crates/managesieve/src/op/setactive.rs @@ -8,8 +8,8 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; +use email::sieve::activate::SieveScriptActivate; use imap_proto::receiver::Request; -use jmap::sieve::set::SieveScriptSet; use jmap_proto::types::collection::Collection; use store::write::log::ChangeLogBuilder; use trc::AddContext; diff --git a/crates/pop3/Cargo.toml b/crates/pop3/Cargo.toml index 5dd07e5e..0429013e 100644 --- a/crates/pop3/Cargo.toml +++ b/crates/pop3/Cargo.toml @@ -8,7 +8,6 @@ resolver = "2" store = { path = "../store" } common = { path = "../common" } directory = { path = "../directory" } -jmap = { path = "../jmap" } imap = { path = "../imap" } utils = { path = "../utils" } trc = { path = "../trc" } diff --git a/crates/pop3/src/mailbox.rs b/crates/pop3/src/mailbox.rs index 8221d8f9..b4c79487 100644 --- a/crates/pop3/src/mailbox.rs +++ b/crates/pop3/src/mailbox.rs @@ -7,10 +7,12 @@ use std::collections::BTreeMap; use common::listener::SessionStream; -use email::mailbox::{INBOX_ID, UidMailbox, manage::MailboxFnc}; +use email::mailbox::{ArchivedMailbox, INBOX_ID, UidMailbox, manage::MailboxFnc}; use jmap_proto::types::{collection::Collection, property::Property}; use store::{ - IndexKey, IterateParams, Serialize, U32_LEN, ahash::AHashMap, write::key::DeserializeBigEndian, + IndexKey, IterateParams, SerializeInfallible, U32_LEN, + ahash::AHashMap, + write::{ArchivedValue, key::DeserializeBigEndian}, }; use trc::AddContext; @@ -59,24 +61,27 @@ impl Session { .mailbox_get_or_create(account_id) .await .caused_by(trc::location!())?; - let uid_validity = self - .server - .get_property::( - account_id, - Collection::Mailbox, - INBOX_ID, - &Property::Value, - ) - .await - .caused_by(trc::location!())? - .ok_or_else(|| { - trc::StoreEvent::UnexpectedError - .caused_by(trc::location!()) - .details("Failed to obtain UID validity") - .account_id(account_id) - .document_id(INBOX_ID) - })? - .uid_validity; + let uid_validity = u32::from( + self.server + .get_property::>( + account_id, + Collection::Mailbox, + INBOX_ID, + &Property::Value, + ) + .await + .caused_by(trc::location!())? + .ok_or_else(|| { + trc::StoreEvent::UnexpectedError + .caused_by(trc::location!()) + .details("Failed to obtain UID validity") + .account_id(account_id) + .document_id(INBOX_ID) + })? + .unarchive() + .caused_by(trc::location!())? + .uid_validity, + ); // Obtain message sizes self.server diff --git a/crates/pop3/src/op/fetch.rs b/crates/pop3/src/op/fetch.rs index d74d9a6c..28743e2f 100644 --- a/crates/pop3/src/op/fetch.rs +++ b/crates/pop3/src/op/fetch.rs @@ -9,7 +9,6 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; use email::message::metadata::MessageMetadata; -use jmap::blob::download::BlobDownload; use jmap_proto::types::{collection::Collection, property::Property}; use store::write::Bincode; use trc::AddContext; @@ -39,7 +38,8 @@ impl Session { { if let Some(bytes) = self .server - .get_blob(&metadata.inner.blob_hash, 0..usize::MAX) + .blob_store() + .get_blob(metadata.inner.blob_hash.as_slice(), 0..usize::MAX) .await .caused_by(trc::location!())? { diff --git a/crates/smtp/src/queue/spool.rs b/crates/smtp/src/queue/spool.rs index 7612aca0..97708bed 100644 --- a/crates/smtp/src/queue/spool.rs +++ b/crates/smtp/src/queue/spool.rs @@ -6,13 +6,13 @@ use crate::queue::DomainPart; use common::ipc::QueueEvent; -use common::{Server, KV_LOCK_QUEUE_MESSAGE}; +use common::{KV_LOCK_QUEUE_MESSAGE, Server}; use std::borrow::Cow; use std::future::Future; use std::time::{Duration, SystemTime}; use store::write::key::DeserializeBigEndian; -use store::write::{now, BatchBuilder, Bincode, BlobOp, QueueClass, ValueClass}; -use store::{IterateParams, Serialize, ValueKey, U64_LEN}; +use store::write::{BatchBuilder, Bincode, BlobOp, QueueClass, ValueClass, now}; +use store::{IterateParams, Serialize, SerializeInfallible, U64_LEN, ValueKey}; use trc::ServerEvent; use utils::BlobHash; @@ -104,9 +104,10 @@ impl SmtpSpool for Server { .await; if let Err(err) = result { - trc::error!(err - .details("Failed to read queue.") - .caused_by(trc::location!())); + trc::error!( + err.details("Failed to read queue.") + .caused_by(trc::location!()) + ); } events @@ -125,9 +126,10 @@ impl SmtpSpool for Server { result } Err(err) => { - trc::error!(err - .details("Failed to lock event.") - .caused_by(trc::location!())); + trc::error!( + err.details("Failed to lock event.") + .caused_by(trc::location!()) + ); false } } @@ -139,9 +141,10 @@ impl SmtpSpool for Server { .remove_lock(KV_LOCK_QUEUE_MESSAGE, &queue_id.to_be_bytes()) .await { - trc::error!(err - .details("Failed to unlock event.") - .caused_by(trc::location!())); + trc::error!( + err.details("Failed to unlock event.") + .caused_by(trc::location!()) + ); } } @@ -156,9 +159,10 @@ impl SmtpSpool for Server { Ok(Some(message)) => Some(message.inner), Ok(None) => None, Err(err) => { - trc::error!(err - .details("Failed to read message.") - .caused_by(trc::location!())); + trc::error!( + err.details("Failed to read message.") + .caused_by(trc::location!()) + ); None } @@ -202,10 +206,11 @@ impl Message { 0u32.serialize(), ); if let Err(err) = server.store().write(batch.build()).await { - trc::error!(err - .details("Failed to write to store.") - .span_id(session_id) - .caused_by(trc::location!())); + trc::error!( + err.details("Failed to write to store.") + .span_id(session_id) + .caused_by(trc::location!()) + ); return false; } @@ -214,10 +219,11 @@ impl Message { .put_blob(self.blob_hash.as_slice(), message.as_ref()) .await { - trc::error!(err - .details("Failed to write blob.") - .span_id(session_id) - .caused_by(trc::location!())); + trc::error!( + err.details("Failed to write blob.") + .span_id(session_id) + .caused_by(trc::location!()) + ); return false; } @@ -292,14 +298,25 @@ impl Message { ) .set( ValueClass::Queue(QueueClass::Message(self.queue_id)), - Bincode::new(self).serialize(), + match Bincode::new(self).serialize() { + Ok(data) => data, + Err(err) => { + trc::error!( + err.details("Failed to serialize message.") + .span_id(session_id) + .caused_by(trc::location!()) + ); + return false; + } + }, ); if let Err(err) = server.store().write(batch.build()).await { - trc::error!(err - .details("Failed to write to store.") - .span_id(session_id) - .caused_by(trc::location!())); + trc::error!( + err.details("Failed to write to store.") + .span_id(session_id) + .caused_by(trc::location!()) + ); return false; } @@ -413,14 +430,25 @@ impl Message { let span_id = self.span_id; batch.set( ValueClass::Queue(QueueClass::Message(self.queue_id)), - Bincode::new(self).serialize(), + match Bincode::new(self).serialize() { + Ok(data) => data, + Err(err) => { + trc::error!( + err.details("Failed to serialize message.") + .span_id(span_id) + .caused_by(trc::location!()) + ); + return false; + } + }, ); if let Err(err) = server.store().write(batch.build()).await { - trc::error!(err - .details("Failed to save changes.") - .span_id(span_id) - .caused_by(trc::location!())); + trc::error!( + err.details("Failed to save changes.") + .span_id(span_id) + .caused_by(trc::location!()) + ); false } else { true @@ -459,10 +487,11 @@ impl Message { .clear(ValueClass::Queue(QueueClass::Message(self.queue_id))); if let Err(err) = server.store().write(batch.build()).await { - trc::error!(err - .details("Failed to write to update queue.") - .span_id(self.span_id) - .caused_by(trc::location!())); + trc::error!( + err.details("Failed to write to update queue.") + .span_id(self.span_id) + .caused_by(trc::location!()) + ); false } else { true diff --git a/crates/smtp/src/reporting/analysis.rs b/crates/smtp/src/reporting/analysis.rs index 8a554558..a9cf415c 100644 --- a/crates/smtp/src/reporting/analysis.rs +++ b/crates/smtp/src/reporting/analysis.rs @@ -14,14 +14,14 @@ use ahash::AHashMap; use common::Server; use mail_auth::{ flate2::read::GzDecoder, - report::{tlsrpt::TlsReport, ActionDisposition, DmarcResult, Feedback, Report}, + report::{ActionDisposition, DmarcResult, Feedback, Report, tlsrpt::TlsReport}, zip, }; use mail_parser::{Message, MimeHeaders, PartType}; use store::{ - write::{now, BatchBuilder, Bincode, ReportClass, ValueClass}, Serialize, + write::{BatchBuilder, Bincode, ReportClass, ValueClass, now}, }; use trc::IncomingReportEvent; @@ -287,7 +287,8 @@ impl AnalyzeReport for Server { subject, report, }) - .serialize(), + .serialize() + .unwrap_or_default(), ); } Format::Tls(report) => { @@ -299,7 +300,8 @@ impl AnalyzeReport for Server { subject, report, }) - .serialize(), + .serialize() + .unwrap_or_default(), ); } Format::Arf(report) => { @@ -311,16 +313,18 @@ impl AnalyzeReport for Server { subject, report, }) - .serialize(), + .serialize() + .unwrap_or_default(), ); } } let batch = batch.build(); if let Err(err) = core.core.storage.data.write(batch).await { - trc::error!(err - .span_id(session_id) - .caused_by(trc::location!()) - .details("Failed to write report")); + trc::error!( + err.span_id(session_id) + .caused_by(trc::location!()) + .details("Failed to write report") + ); } } return; diff --git a/crates/smtp/src/reporting/dmarc.rs b/crates/smtp/src/reporting/dmarc.rs index b55a0757..7b589eab 100644 --- a/crates/smtp/src/reporting/dmarc.rs +++ b/crates/smtp/src/reporting/dmarc.rs @@ -8,21 +8,21 @@ use std::{collections::hash_map::Entry, future::Future}; use ahash::AHashMap; use common::{ + Server, config::smtp::report::AggregateFrequency, ipc::{DmarcEvent, ToHash}, listener::SessionStream, - Server, }; use mail_auth::{ + ArcOutput, AuthenticatedMessage, AuthenticationResults, DkimOutput, DkimResult, DmarcOutput, + SpfResult, common::verify::VerifySignature, dmarc::{self, URI}, report::{AuthFailureType, IdentityAlignment, PolicyPublished, Record, Report, SPFDomainScope}, - ArcOutput, AuthenticatedMessage, AuthenticationResults, DkimOutput, DkimResult, DmarcOutput, - SpfResult, }; use store::{ - write::{now, BatchBuilder, Bincode, QueueClass, ReportEvent, ValueClass}, Deserialize, IterateParams, Serialize, ValueKey, + write::{BatchBuilder, Bincode, QueueClass, ReportEvent, ValueClass, now}, }; use trc::{AddContext, OutgoingReportEvent}; use utils::config::Rate; @@ -551,7 +551,7 @@ impl DmarcReporting for Server { Entry::Vacant(e) => { if serialized_size .as_deref_mut() - .is_none_or( |serialized_size| { + .is_none_or(|serialized_size| { serde::Serialize::serialize(e.key(), serialized_size).is_ok() }) { @@ -597,18 +597,20 @@ impl DmarcReporting for Server { ) .await { - trc::error!(err - .caused_by(trc::location!()) - .details("Failed to delete DMARC report")); + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to delete DMARC report") + ); return; } let mut batch = BatchBuilder::new(); batch.clear(ValueClass::Queue(QueueClass::DmarcReportHeader(event))); if let Err(err) = self.core.storage.data.write(batch.build()).await { - trc::error!(err - .caused_by(trc::location!()) - .details("Failed to delete DMARC report")); + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to delete DMARC report") + ); } } @@ -648,7 +650,16 @@ impl DmarcReporting for Server { // Write report builder.set( ValueClass::Queue(QueueClass::DmarcReportHeader(report_event.clone())), - Bincode::new(entry).serialize(), + match Bincode::new(entry).serialize() { + Ok(data) => data, + Err(err) => { + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to serialize DMARC report") + ); + return; + } + }, ); } @@ -656,13 +667,23 @@ impl DmarcReporting for Server { report_event.seq_id = self.inner.data.queue_id_gen.generate().unwrap_or_else(now); builder.set( ValueClass::Queue(QueueClass::DmarcReportEvent(report_event)), - Bincode::new(event.report_record).serialize(), + match Bincode::new(event.report_record).serialize() { + Ok(data) => data, + Err(err) => { + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to serialize DMARC report") + ); + return; + } + }, ); if let Err(err) = self.core.storage.data.write(builder.build()).await { - trc::error!(err - .caused_by(trc::location!()) - .details("Failed to write DMARC report")); + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to write DMARC report") + ); } } } diff --git a/crates/smtp/src/reporting/tls.rs b/crates/smtp/src/reporting/tls.rs index 1da29ea9..ce236f18 100644 --- a/crates/smtp/src/reporting/tls.rs +++ b/crates/smtp/src/reporting/tls.rs @@ -8,15 +8,15 @@ use std::{collections::hash_map::Entry, future::Future, sync::Arc, time::Duratio use ahash::AHashMap; use common::{ + Server, USER_AGENT, config::smtp::{ report::AggregateFrequency, resolver::{Mode, MxPattern}, }, ipc::{TlsEvent, ToHash}, - Server, USER_AGENT, }; use mail_auth::{ - flate2::{write::GzEncoder, Compression}, + flate2::{Compression, write::GzEncoder}, mta_sts::{ReportUri, TlsRpt}, report::tlsrpt::{ DateRange, FailureDetails, Policy, PolicyDetails, PolicyType, Summary, TlsReport, @@ -27,8 +27,8 @@ use mail_parser::DateTime; use reqwest::header::CONTENT_TYPE; use std::fmt::Write; use store::{ - write::{now, BatchBuilder, Bincode, QueueClass, ReportEvent, ValueClass}, Deserialize, IterateParams, Serialize, ValueKey, + write::{BatchBuilder, Bincode, QueueClass, ReportEvent, ValueClass, now}, }; use trc::{AddContext, OutgoingReportEvent}; @@ -113,10 +113,11 @@ impl TlsReporting for Server { return; } Err(err) => { - trc::error!(err - .span_id(span_id) - .caused_by(trc::location!()) - .details("Failed to read TLS report")); + trc::error!( + err.span_id(span_id) + .caused_by(trc::location!()) + .details("Failed to read TLS report") + ); return; } }; @@ -348,7 +349,7 @@ impl TlsReporting for Server { Entry::Vacant(e) => { if serialized_size .as_deref_mut() - .is_none_or( |serialized_size| { + .is_none_or(|serialized_size| { serde::Serialize::serialize(e.key(), serialized_size) .is_ok() }) @@ -490,7 +491,16 @@ impl TlsReporting for Server { // Write report builder.set( ValueClass::Queue(QueueClass::TlsReportHeader(report_event.clone())), - Bincode::new(entry).serialize(), + match Bincode::new(entry).serialize() { + Ok(data) => data, + Err(err) => { + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to serialize TLS report") + ); + return; + } + }, ); } @@ -498,13 +508,23 @@ impl TlsReporting for Server { report_event.seq_id = self.inner.data.queue_id_gen.generate().unwrap_or_else(now); builder.set( ValueClass::Queue(QueueClass::TlsReportEvent(report_event)), - Bincode::new(event.failure).serialize(), + match Bincode::new(event.failure).serialize() { + Ok(data) => data, + Err(err) => { + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to serialize TLS report") + ); + return; + } + }, ); if let Err(err) = self.core.storage.data.write(builder.build()).await { - trc::error!(err - .caused_by(trc::location!()) - .details("Failed to write TLS report")); + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to write TLS report") + ); } } @@ -536,9 +556,10 @@ impl TlsReporting for Server { ) .await { - trc::error!(err - .caused_by(trc::location!()) - .details("Failed to delete TLS reports")); + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to delete TLS reports") + ); return; } @@ -548,9 +569,10 @@ impl TlsReporting for Server { } if let Err(err) = self.core.storage.data.write(batch.build()).await { - trc::error!(err - .caused_by(trc::location!()) - .details("Failed to delete TLS reports")); + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to delete TLS reports") + ); } } } diff --git a/crates/spam-filter/src/analysis/reputation.rs b/crates/spam-filter/src/analysis/reputation.rs index bac8573a..2e102954 100644 --- a/crates/spam-filter/src/analysis/reputation.rs +++ b/crates/spam-filter/src/analysis/reputation.rs @@ -7,15 +7,15 @@ use std::{borrow::Cow, future::Future}; use common::{ - ip_to_bytes, Server, KV_REPUTATION_ASN, KV_REPUTATION_DOMAIN, KV_REPUTATION_FROM, - KV_REPUTATION_IP, + KV_REPUTATION_ASN, KV_REPUTATION_DOMAIN, KV_REPUTATION_FROM, KV_REPUTATION_IP, Server, + ip_to_bytes, }; use mail_auth::DmarcResult; -use store::{dispatch::lookup::KeyValue, Deserialize, Serialize}; +use store::{Deserialize, Serialize, dispatch::lookup::KeyValue}; use crate::{ - modules::{key_get, key_set}, SpamFilterContext, + modules::{key_get, key_set}, }; pub trait SpamFilterAnalyzeReputation: Sync + Send { @@ -104,7 +104,8 @@ impl SpamFilterAnalyzeReputation for Server { count: 1, score: ctx.result.score, } - .serialize(), + .serialize() + .unwrap(), ) .expires(config.expiry), ) @@ -131,7 +132,8 @@ impl SpamFilterAnalyzeReputation for Server { count: updated_count, score: updated_score, } - .serialize(), + .serialize() + .unwrap(), ) .expires(config.expiry), ) @@ -168,12 +170,12 @@ impl Type { } } -impl Serialize for &Reputation { - fn serialize(self) -> Vec { +impl Serialize for Reputation { + fn serialize(&self) -> trc::Result> { let mut buf = Vec::with_capacity(12); buf.extend_from_slice(&self.count.to_be_bytes()); buf.extend_from_slice(&self.score.to_be_bytes()); - buf + Ok(buf) } } diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 4e2dc351..63ea3655 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -9,8 +9,8 @@ utils = { path = "../utils" } nlp = { path = "../nlp" } trc = { path = "../trc" } rocksdb = { version = "0.23", optional = true, features = ["multi-threaded-cf"] } -foundationdb = { version = "0.9.2", features = ["fdb-7_3"], optional = true } -rusqlite = { version = "0.32", features = ["bundled"], optional = true } +foundationdb = { version = "0.9.2", features = ["embedded-fdb-include", "fdb-7_3"], optional = true } +rusqlite = { version = "0.34", features = ["bundled"], optional = true } rust-s3 = { version = "=0.35.0-alpha.2", default-features = false, features = ["tokio-rustls-tls", "no-verify-ssl"], optional = true } azure_core = { version = "0.21.0", optional = true } azure_storage = { version = "0.21.0", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"], optional = true } @@ -44,11 +44,13 @@ serde_json = {version = "1.0.64", optional = true } regex = "1.7.0" flate2 = "1.0" async-trait = "0.1.68" -redis = { version = "0.27", features = [ "tokio-comp", "tokio-rustls-comp", "tls-rustls-insecure", "tls-rustls-webpki-roots", "cluster-async"], optional = true } +redis = { version = "0.29", features = [ "tokio-comp", "tokio-rustls-comp", "tls-rustls-insecure", "tls-rustls-webpki-roots", "cluster-async"], optional = true } deadpool = { version = "0.12", features = ["managed"], optional = true } bincode = "1.3.3" arc-swap = "1.6.0" bitpacking = "0.9.2" +memchr = { version = "2" } +rkyv = { version = "0.8.10", features = ["little_endian"] } [dev-dependencies] tokio = { version = "1.23", features = ["full"] } diff --git a/crates/store/src/backend/foundationdb/read.rs b/crates/store/src/backend/foundationdb/read.rs index c4de2520..622efd8b 100644 --- a/crates/store/src/backend/foundationdb/read.rs +++ b/crates/store/src/backend/foundationdb/read.rs @@ -5,23 +5,23 @@ */ use foundationdb::{ + KeySelector, RangeOption, Transaction, future::FdbSlice, options::{self, StreamingMode}, - KeySelector, RangeOption, Transaction, }; use futures::TryStreamExt; use roaring::RoaringBitmap; use crate::{ + BitmapKey, Deserialize, IterateParams, Key, U32_LEN, ValueKey, WITH_SUBSPACE, backend::deserialize_i64_le, write::{ - key::{DeserializeBigEndian, KeySerializer}, BitmapClass, ValueClass, + key::{DeserializeBigEndian, KeySerializer}, }, - BitmapKey, Deserialize, IterateParams, Key, ValueKey, U32_LEN, WITH_SUBSPACE, }; -use super::{into_error, FdbStore, ReadVersion, TimedTransaction, MAX_VALUE_SIZE}; +use super::{FdbStore, MAX_VALUE_SIZE, ReadVersion, TimedTransaction, into_error}; #[allow(dead_code)] pub(crate) enum ChunkedValue { @@ -40,7 +40,7 @@ impl FdbStore { match read_chunked_value(&key, &trx, true).await? { ChunkedValue::Single(bytes) => U::deserialize(&bytes).map(Some), - ChunkedValue::Chunked { bytes, .. } => U::deserialize(&bytes).map(Some), + ChunkedValue::Chunked { bytes, .. } => U::deserialize_owned(bytes).map(Some), ChunkedValue::None => Ok(None), } } diff --git a/crates/store/src/backend/mysql/read.rs b/crates/store/src/backend/mysql/read.rs index 0b1f573d..72f03d49 100644 --- a/crates/store/src/backend/mysql/read.rs +++ b/crates/store/src/backend/mysql/read.rs @@ -5,15 +5,15 @@ */ use futures::TryStreamExt; -use mysql_async::{prelude::Queryable, Row}; +use mysql_async::{Row, prelude::Queryable}; use roaring::RoaringBitmap; use crate::{ - write::{key::DeserializeBigEndian, BitmapClass, ValueClass}, - BitmapKey, Deserialize, IterateParams, Key, ValueKey, U32_LEN, + BitmapKey, Deserialize, IterateParams, Key, U32_LEN, ValueKey, + write::{BitmapClass, ValueClass, key::DeserializeBigEndian}, }; -use super::{into_error, MysqlStore}; +use super::{MysqlStore, into_error}; impl MysqlStore { pub(crate) async fn get_value(&self, key: impl Key) -> trc::Result> @@ -34,7 +34,7 @@ impl MysqlStore { .map_err(into_error) .and_then(|r| { if let Some(r) = r { - Ok(Some(U::deserialize(&r)?)) + Ok(Some(U::deserialize_owned(r)?)) } else { Ok(None) } @@ -90,8 +90,8 @@ impl MysqlStore { } (true, false) => { format!( - "SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC LIMIT 1" - ) + "SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC LIMIT 1" + ) } (false, true) => { format!("SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC") diff --git a/crates/store/src/backend/postgres/read.rs b/crates/store/src/backend/postgres/read.rs index b070bec1..d6cb5a29 100644 --- a/crates/store/src/backend/postgres/read.rs +++ b/crates/store/src/backend/postgres/read.rs @@ -4,15 +4,15 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use futures::{pin_mut, TryStreamExt}; +use futures::{TryStreamExt, pin_mut}; use roaring::RoaringBitmap; use crate::{ - write::{key::DeserializeBigEndian, BitmapClass, ValueClass}, - BitmapKey, Deserialize, IterateParams, Key, ValueKey, U32_LEN, + BitmapKey, Deserialize, IterateParams, Key, U32_LEN, ValueKey, + write::{BitmapClass, ValueClass, key::DeserializeBigEndian}, }; -use super::{into_error, PostgresStore}; +use super::{PostgresStore, into_error}; impl PostgresStore { pub(crate) async fn get_value(&self, key: impl Key) -> trc::Result> diff --git a/crates/store/src/backend/redis/lookup.rs b/crates/store/src/backend/redis/lookup.rs index d60b4fda..e9afc444 100644 --- a/crates/store/src/backend/redis/lookup.rs +++ b/crates/store/src/backend/redis/lookup.rs @@ -8,7 +8,7 @@ use redis::AsyncCommands; use crate::Deserialize; -use super::{into_error, RedisPool, RedisStore}; +use super::{RedisPool, RedisStore, into_error}; impl RedisStore { pub async fn key_set(&self, key: &[u8], value: &[u8], expires: Option) -> trc::Result<()> { @@ -136,7 +136,7 @@ impl RedisStore { .await .map_err(into_error)? { - T::deserialize(&value).map(Some) + T::deserialize_owned(value).map(Some) } else { Ok(None) } diff --git a/crates/store/src/backend/rocksdb/read.rs b/crates/store/src/backend/rocksdb/read.rs index 893443ba..c1d4b496 100644 --- a/crates/store/src/backend/rocksdb/read.rs +++ b/crates/store/src/backend/rocksdb/read.rs @@ -7,12 +7,12 @@ use roaring::RoaringBitmap; use rocksdb::{Direction, IteratorMode}; -use super::{into_error, RocksDbStore}; +use super::{RocksDbStore, into_error}; use crate::{ + BitmapKey, Deserialize, IterateParams, Key, U32_LEN, ValueKey, backend::rocksdb::CfHandle, - write::{key::DeserializeBigEndian, BitmapClass, ValueClass}, - BitmapKey, Deserialize, IterateParams, Key, ValueKey, U32_LEN, + write::{BitmapClass, ValueClass, key::DeserializeBigEndian}, }; impl RocksDbStore { diff --git a/crates/store/src/dispatch/lookup.rs b/crates/store/src/dispatch/lookup.rs index e6652fc8..81d7cacd 100644 --- a/crates/store/src/dispatch/lookup.rs +++ b/crates/store/src/dispatch/lookup.rs @@ -9,18 +9,19 @@ use std::borrow::Cow; use trc::AddContext; use utils::config::Rate; -use crate::{ - backend::http::lookup::HttpStoreGet, - write::{assert::AssertValue, InMemoryClass, MaybeDynamicId}, - Serialize, -}; #[allow(unused_imports)] use crate::{ + Deserialize, InMemoryStore, IterateParams, QueryResult, Store, U64_LEN, Value, ValueKey, write::{ + BatchBuilder, Operation, ValueClass, ValueOp, key::{DeserializeBigEndian, KeySerializer}, - now, BatchBuilder, Operation, ValueClass, ValueOp, + now, }, - Deserialize, InMemoryStore, IterateParams, QueryResult, Store, Value, ValueKey, U64_LEN, +}; +use crate::{ + SerializeInfallible, + backend::http::lookup::HttpStoreGet, + write::{InMemoryClass, MaybeDynamicId, assert::AssertValue}, }; pub struct KeyValue { diff --git a/crates/store/src/fts/index.rs b/crates/store/src/fts/index.rs index fd2d3ba2..57c9e9f9 100644 --- a/crates/store/src/fts/index.rs +++ b/crates/store/src/fts/index.rs @@ -9,25 +9,25 @@ use std::{borrow::Cow, fmt::Display}; use ahash::AHashMap; use nlp::{ language::{ + Language, detect::{LanguageDetector, MIN_LANGUAGE_SCORE}, stemmer::Stemmer, - Language, }, tokenizers::word::WordTokenizer, }; use trc::AddContext; use crate::{ + IterateParams, SerializeInfallible, Store, U32_LEN, ValueKey, backend::MAX_TOKEN_LENGTH, dispatch::DocumentSet, write::{ - hash::TokenType, key::DeserializeBigEndian, BatchBuilder, BitmapHash, MaybeDynamicId, - Operation, ValueClass, ValueOp, + BatchBuilder, BitmapHash, MaybeDynamicId, Operation, ValueClass, ValueOp, hash::TokenType, + key::DeserializeBigEndian, }, - IterateParams, Serialize, Store, ValueKey, U32_LEN, }; -use super::{postings::Postings, Field}; +use super::{Field, postings::Postings}; pub const TERM_INDEX_VERSION: u8 = 1; #[derive(Debug)] diff --git a/crates/store/src/fts/postings.rs b/crates/store/src/fts/postings.rs index fa4ba394..0c8a94f8 100644 --- a/crates/store/src/fts/postings.rs +++ b/crates/store/src/fts/postings.rs @@ -10,7 +10,7 @@ use ahash::AHashSet; use bitpacking::{BitPacker, BitPacker1x, BitPacker4x, BitPacker8x}; use utils::codec::leb128::Leb128Reader; -use crate::{write::key::KeySerializer, Serialize}; +use crate::{SerializeInfallible, write::key::KeySerializer}; #[derive(Default)] pub(super) struct Postings { @@ -159,13 +159,13 @@ impl Iterator for PostingsIterator<'_> { } } -impl Serialize for Postings { - fn serialize(self) -> Vec { +impl SerializeInfallible for Postings { + fn serialize(&self) -> Vec { // Serialize fields let mut serializer = KeySerializer::new((self.fields.len() + 1) + (self.postings.len() * 2)); - for field in self.fields { - serializer = serializer.write(field); + for field in &self.fields { + serializer = serializer.write(*field); } serializer = serializer.write(u8::MAX); diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 62bf9c3e..8b56e390 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -14,15 +14,17 @@ pub mod query; pub mod write; pub use ahash; -use ahash::AHashMap; -use backend::{fs::FsStore, http::HttpStore, memory::StaticMemoryStore}; pub use blake3; pub use parking_lot; pub use rand; +pub use rkyv; pub use roaring; +pub use xxhash_rust; + +use ahash::AHashMap; +use backend::{fs::FsStore, http::HttpStore, memory::StaticMemoryStore}; use utils::config::cron::SimpleCron; use write::{BitmapClass, ValueClass}; -pub use xxhash_rust; #[cfg(feature = "s3")] use backend::s3::S3Store; @@ -53,10 +55,17 @@ use backend::azure::AzureStore; pub trait Deserialize: Sized + Sync + Send { fn deserialize(bytes: &[u8]) -> trc::Result; + fn deserialize_owned(bytes: Vec) -> trc::Result { + Self::deserialize(&bytes) + } } pub trait Serialize { - fn serialize(self) -> Vec; + fn serialize(&self) -> trc::Result>; +} + +pub trait SerializeInfallible { + fn serialize(&self) -> Vec; } // Key serialization flags diff --git a/crates/store/src/query/filter.rs b/crates/store/src/query/filter.rs index 022b8c94..ee670e1f 100644 --- a/crates/store/src/query/filter.rs +++ b/crates/store/src/query/filter.rs @@ -12,8 +12,8 @@ use roaring::RoaringBitmap; use trc::AddContext; use crate::{ - backend::MAX_TOKEN_LENGTH, write::key::DeserializeBigEndian, BitmapKey, IndexKey, - IndexKeyPrefix, IterateParams, Key, Store, U32_LEN, + BitmapKey, IndexKey, IndexKeyPrefix, IterateParams, Key, Store, U32_LEN, + backend::MAX_TOKEN_LENGTH, write::key::DeserializeBigEndian, }; use super::{Filter, Operator, ResultSet}; @@ -177,6 +177,7 @@ impl Store { match_value: &[u8], op: Operator, ) -> trc::Result> { + let mut finder = None; let (begin, end) = match op { Operator::LowerThan => ( IndexKey { @@ -258,6 +259,26 @@ impl Store { key: match_value, }, ), + Operator::Contains => { + finder = memchr::memmem::Finder::new(match_value).into(); + + ( + IndexKey { + account_id, + collection, + document_id: 0, + field, + key: &[][..], + }, + IndexKey { + account_id, + collection, + document_id: u32::MAX, + field: field + 1, + key: &[u8::MAX, u8::MAX, u8::MAX, u8::MAX][..], + }, + ) + } }; let mut bm = RoaringBitmap::new(); @@ -286,6 +307,7 @@ impl Store { Operator::GreaterThan => value > match_value, Operator::GreaterEqualThan => value >= match_value, Operator::Equal => value == match_value, + Operator::Contains => finder.as_ref().unwrap().find(value).is_some(), }; if matches { diff --git a/crates/store/src/query/mod.rs b/crates/store/src/query/mod.rs index 9a9822f1..885e8c89 100644 --- a/crates/store/src/query/mod.rs +++ b/crates/store/src/query/mod.rs @@ -12,8 +12,8 @@ pub mod sort; use roaring::RoaringBitmap; use crate::{ + BitmapKey, IterateParams, Key, write::{BitmapClass, BitmapHash, TagValue}, - BitmapKey, IterateParams, Key, Serialize, }; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -23,6 +23,7 @@ pub enum Operator { GreaterThan, GreaterEqualThan, Equal, + Contains, } #[derive(Debug)] @@ -79,51 +80,59 @@ impl ResultSet { } impl Filter { - pub fn cond(field: impl Into, op: Operator, value: impl Serialize) -> Self { + pub fn cond(field: impl Into, op: Operator, value: Vec) -> Self { Filter::MatchValue { field: field.into(), op, - value: value.serialize(), + value, } } - pub fn eq(field: impl Into, value: impl Serialize) -> Self { + pub fn eq(field: impl Into, value: Vec) -> Self { Filter::MatchValue { field: field.into(), op: Operator::Equal, - value: value.serialize(), + value, } } - pub fn lt(field: impl Into, value: impl Serialize) -> Self { + pub fn lt(field: impl Into, value: Vec) -> Self { Filter::MatchValue { field: field.into(), op: Operator::LowerThan, - value: value.serialize(), + value, } } - pub fn le(field: impl Into, value: impl Serialize) -> Self { + pub fn le(field: impl Into, value: Vec) -> Self { Filter::MatchValue { field: field.into(), op: Operator::LowerEqualThan, - value: value.serialize(), + value, } } - pub fn gt(field: impl Into, value: impl Serialize) -> Self { + pub fn gt(field: impl Into, value: Vec) -> Self { Filter::MatchValue { field: field.into(), op: Operator::GreaterThan, - value: value.serialize(), + value, } } - pub fn ge(field: impl Into, value: impl Serialize) -> Self { + pub fn ge(field: impl Into, value: Vec) -> Self { Filter::MatchValue { field: field.into(), op: Operator::GreaterEqualThan, - value: value.serialize(), + value, + } + } + + pub fn contains(field: impl Into, value: Vec) -> Self { + Filter::MatchValue { + field: field.into(), + op: Operator::Contains, + value, } } diff --git a/crates/store/src/write/assert.rs b/crates/store/src/write/assert.rs index 7f19cfac..5892e89c 100644 --- a/crates/store/src/write/assert.rs +++ b/crates/store/src/write/assert.rs @@ -7,7 +7,7 @@ use crate::{Deserialize, U32_LEN, U64_LEN}; #[derive(Debug, Clone)] -pub struct HashedValue { +pub struct HashedValue { pub hash: u64, pub inner: T, } @@ -55,13 +55,13 @@ impl ToAssertValue for u32 { } } -impl ToAssertValue for HashedValue { +impl ToAssertValue for HashedValue { fn to_assert_value(&self) -> AssertValue { AssertValue::Hash(self.hash) } } -impl ToAssertValue for &HashedValue { +impl ToAssertValue for &HashedValue { fn to_assert_value(&self) -> AssertValue { AssertValue::Hash(self.hash) } @@ -90,4 +90,11 @@ impl Deserialize for HashedValue { inner: T::deserialize(bytes)?, }) } + + fn deserialize_owned(bytes: Vec) -> trc::Result { + Ok(HashedValue { + hash: xxhash_rust::xxh3::xxh3_64(&bytes), + inner: T::deserialize_owned(bytes)?, + }) + } } diff --git a/crates/store/src/write/batch.rs b/crates/store/src/write/batch.rs index 0587fa89..aa151151 100644 --- a/crates/store/src/write/batch.rs +++ b/crates/store/src/write/batch.rs @@ -5,9 +5,8 @@ */ use super::{ - assert::ToAssertValue, Batch, BatchBuilder, BitmapClass, HasFlag, IntoOperations, - MaybeDynamicId, MaybeDynamicValue, Operation, Serialize, TagValue, ToBitmaps, ValueClass, - ValueOp, F_BITMAP, F_CLEAR, F_INDEX, F_VALUE, + Batch, BatchBuilder, BitmapClass, IntoOperations, MaybeDynamicId, MaybeDynamicValue, Operation, + TagValue, ValueClass, ValueOp, assert::ToAssertValue, }; impl BatchBuilder { @@ -86,40 +85,65 @@ impl BatchBuilder { self } - pub fn value( + pub fn set_and_index(&mut self, field: impl Into, value: impl Into>) -> &mut Self { + let field = field.into(); + let value = value.into(); + + self.ops.push(Operation::Index { + field, + key: value.clone(), + set: true, + }); + self.ops.push(Operation::Value { + class: ValueClass::Property(field), + op: ValueOp::Set(value.into()), + }); + + self + } + + pub fn unset_and_unindex( &mut self, field: impl Into, - value: impl Serialize + ToBitmaps, - options: u32, + value: impl Into>, ) -> &mut Self { let field = field.into(); - let is_set = !options.has_flag(F_CLEAR); + let value = value.into(); - if options.has_flag(F_BITMAP) { - value.to_bitmaps(&mut self.ops, field, is_set); - } + self.ops.push(Operation::Index { + field, + key: value, + set: false, + }); + self.ops.push(Operation::Value { + class: ValueClass::Property(field), + op: ValueOp::Clear, + }); - let value = value.serialize(); + self + } - if options.has_flag(F_INDEX) { - self.ops.push(Operation::Index { - field, - key: value.clone(), - set: is_set, - }); - } + pub fn index(&mut self, field: impl Into, value: impl Into>) -> &mut Self { + let field = field.into(); + let value = value.into(); - if options.has_flag(F_VALUE) { - self.ops.push(Operation::Value { - class: ValueClass::Property(field), - op: if is_set { - ValueOp::Set(value.into()) - } else { - ValueOp::Clear - }, - }); - } + self.ops.push(Operation::Index { + field, + key: value.clone(), + set: true, + }); + self + } + pub fn unindex(&mut self, field: impl Into, value: impl Into>) -> &mut Self { + let field = field.into(); + let value = value.into(); + + self.ops.push(Operation::Index { + field, + key: value.clone(), + set: false, + }); self } @@ -127,18 +151,56 @@ impl BatchBuilder { &mut self, field: impl Into, value: impl Into>, - options: u32, ) -> &mut Self { self.ops.push(Operation::Bitmap { class: BitmapClass::Tag { field: field.into(), value: value.into(), }, - set: !options.has_flag(F_CLEAR), + set: true, }); self } + pub fn untag( + &mut self, + field: impl Into, + value: impl Into>, + ) -> &mut Self { + self.ops.push(Operation::Bitmap { + class: BitmapClass::Tag { + field: field.into(), + value: value.into(), + }, + set: false, + }); + self + } + + pub fn tag_many(&mut self, field: impl Into, values: T) -> &mut Self + where + T: Iterator, + V: Into>, + { + let field = field.into(); + for value in values { + self.tag(field, value); + } + self + } + + pub fn untag_many(&mut self, field: impl Into, values: T) -> &mut Self + where + T: Iterator, + V: Into>, + { + let field = field.into(); + for value in values { + self.untag(field, value); + } + self + } + pub fn add(&mut self, class: impl Into>, value: i64) -> &mut Self { self.ops.push(Operation::Value { class: class.into(), @@ -184,9 +246,9 @@ impl BatchBuilder { self } - pub fn custom(&mut self, value: impl IntoOperations) -> &mut Self { - value.build(self); - self + pub fn custom(&mut self, value: impl IntoOperations) -> trc::Result<&mut Self> { + value.build(self)?; + Ok(self) } pub fn build(self) -> Batch { diff --git a/crates/store/src/write/log.rs b/crates/store/src/write/log.rs index 86e42de8..48ffbf8c 100644 --- a/crates/store/src/write/log.rs +++ b/crates/store/src/write/log.rs @@ -7,7 +7,7 @@ use ahash::AHashSet; use utils::{codec::leb128::Leb128Vec, map::vec_map::VecMap}; -use crate::Serialize; +use crate::SerializeInfallible; use super::{IntoOperations, MaybeDynamicValue, Operation, SerializeWithId}; @@ -126,7 +126,7 @@ impl ChangeLogBuilder { } impl IntoOperations for ChangeLogBuilder { - fn build(self, batch: &mut super::BatchBuilder) { + fn build(self, batch: &mut super::BatchBuilder) -> trc::Result<()> { batch.with_change_id(self.change_id); for (collection, changes) in self.changes { batch.ops.push(Operation::Collection { collection }); @@ -134,6 +134,7 @@ impl IntoOperations for ChangeLogBuilder { set: changes.serialize().into(), }); } + Ok(()) } } @@ -183,8 +184,8 @@ impl Changes { } } -impl Serialize for &Changes { - fn serialize(self) -> Vec { +impl SerializeInfallible for Changes { + fn serialize(&self) -> Vec { let mut buf = Vec::with_capacity( 1 + (self.inserts.len() + self.updates.len() diff --git a/crates/store/src/write/mod.rs b/crates/store/src/write/mod.rs index 6e6987ff..d5a6aec4 100644 --- a/crates/store/src/write/mod.rs +++ b/crates/store/src/write/mod.rs @@ -13,6 +13,7 @@ use std::{ time::{Duration, SystemTime}, }; +use assert::HashedValue; use nlp::tokenizers::word::WordTokenizer; use rand::Rng; use roaring::RoaringBitmap; @@ -21,7 +22,9 @@ use utils::{ codec::leb128::{Leb128Iterator, Leb128Vec}, }; -use crate::{BlobClass, Deserialize, Serialize, Value, backend::MAX_TOKEN_LENGTH}; +use crate::{ + BlobClass, Deserialize, Serialize, SerializeInfallible, Value, backend::MAX_TOKEN_LENGTH, +}; use self::assert::AssertValue; @@ -70,11 +73,6 @@ pub(crate) const MAX_COMMIT_ATTEMPTS: u32 = 1000; #[cfg(feature = "test_mode")] pub(crate) const MAX_COMMIT_TIME: Duration = Duration::from_secs(3600); -pub const F_VALUE: u32 = 1 << 0; -pub const F_INDEX: u32 = 1 << 1; -pub const F_BITMAP: u32 = 1 << 2; -pub const F_CLEAR: u32 = 1 << 3; - #[derive(Debug)] pub struct Batch { pub ops: Vec, @@ -305,64 +303,51 @@ impl From<()> for TagValue { } } -impl Serialize for u32 { - fn serialize(self) -> Vec { +impl SerializeInfallible for u32 { + fn serialize(&self) -> Vec { self.to_be_bytes().to_vec() } } -impl Serialize for u64 { - fn serialize(self) -> Vec { +impl SerializeInfallible for u64 { + fn serialize(&self) -> Vec { self.to_be_bytes().to_vec() } } -impl Serialize for i64 { - fn serialize(self) -> Vec { +impl SerializeInfallible for i64 { + fn serialize(&self) -> Vec { self.to_be_bytes().to_vec() } } -impl Serialize for u16 { - fn serialize(self) -> Vec { +impl SerializeInfallible for u16 { + fn serialize(&self) -> Vec { self.to_be_bytes().to_vec() } } -impl Serialize for f64 { - fn serialize(self) -> Vec { +impl SerializeInfallible for f64 { + fn serialize(&self) -> Vec { self.to_be_bytes().to_vec() } } -impl Serialize for &str { - fn serialize(self) -> Vec { +impl SerializeInfallible for &str { + fn serialize(&self) -> Vec { self.as_bytes().to_vec() } } -impl Serialize for &String { - fn serialize(self) -> Vec { - self.as_bytes().to_vec() - } -} - -impl Serialize for String { - fn serialize(self) -> Vec { - self.into_bytes() - } -} - -impl Serialize for Vec { - fn serialize(self) -> Vec { - self - } -} - impl Deserialize for String { fn deserialize(bytes: &[u8]) -> trc::Result { Ok(String::from_utf8_lossy(bytes).into_owned()) } + + fn deserialize_owned(bytes: Vec) -> trc::Result { + Ok(String::from_utf8(bytes) + .unwrap_or_else(|err| String::from_utf8_lossy(err.as_bytes()).into_owned())) + } } impl Deserialize for u64 { @@ -397,20 +382,19 @@ pub trait DeserializeFrom: Sized { fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option; } -impl Serialize for &Vec { - fn serialize(self) -> Vec { - let mut bytes = Vec::with_capacity(self.len() * 4); - bytes.push_leb128(self.len()); - for item in self { - item.serialize_into(&mut bytes); - } - bytes - } +pub struct ArchivedValue { + inner: Vec, + _phantom: std::marker::PhantomData, } impl Serialize for Vec { - fn serialize(self) -> Vec { - (&self).serialize() + fn serialize(&self) -> trc::Result> { + let mut bytes = Vec::with_capacity(self.len() * 4); + bytes.push_leb128(self.len()); + for item in self.iter() { + item.serialize_into(&mut bytes); + } + Ok(bytes) } } @@ -490,44 +474,11 @@ impl Deserialize for Vec { } } -trait HasFlag { - fn has_flag(&self, flag: u32) -> bool; -} - -impl HasFlag for u32 { - #[inline(always)] - fn has_flag(&self, flag: u32) -> bool { - self & flag == flag - } -} - -pub trait ToBitmaps { - fn to_bitmaps(&self, ops: &mut Vec, field: u8, set: bool); -} - pub trait TokenizeText { fn tokenize_into(&self, tokens: &mut HashSet); fn to_tokens(&self) -> HashSet; } -impl ToBitmaps for &str { - fn to_bitmaps(&self, ops: &mut Vec, field: u8, set: bool) { - let mut tokens = HashSet::new(); - - self.tokenize_into(&mut tokens); - - for token in tokens { - ops.push(Operation::Bitmap { - class: BitmapClass::Text { - field, - token: BitmapHash::new(token), - }, - set, - }); - } - } -} - impl TokenizeText for &str { fn tokenize_into(&self, tokens: &mut HashSet) { for token in WordTokenizer::new(self, MAX_TOKEN_LENGTH) { @@ -542,59 +493,9 @@ impl TokenizeText for &str { } } -impl ToBitmaps for String { - fn to_bitmaps(&self, ops: &mut Vec, field: u8, set: bool) { - self.as_str().to_bitmaps(ops, field, set) - } -} - -impl ToBitmaps for u32 { - fn to_bitmaps(&self, ops: &mut Vec, field: u8, set: bool) { - ops.push(Operation::Bitmap { - class: BitmapClass::Tag { - field, - value: TagValue::Id(MaybeDynamicId::Static(*self)), - }, - set, - }); - } -} - -impl ToBitmaps for u64 { - fn to_bitmaps(&self, ops: &mut Vec, field: u8, set: bool) { - ops.push(Operation::Bitmap { - class: BitmapClass::Tag { - field, - value: TagValue::Id(MaybeDynamicId::Static(*self as u32)), - }, - set, - }); - } -} - -impl ToBitmaps for f64 { - fn to_bitmaps(&self, _ops: &mut Vec, _field: u8, _set: bool) { - unreachable!() - } -} - -impl ToBitmaps for Vec { - fn to_bitmaps(&self, ops: &mut Vec, field: u8, set: bool) { - for item in self { - item.to_bitmaps(ops, field, set); - } - } -} - impl Serialize for () { - fn serialize(self) -> Vec { - Vec::with_capacity(0) - } -} - -impl ToBitmaps for () { - fn to_bitmaps(&self, _ops: &mut Vec, _field: u8, _set: bool) { - unreachable!() + fn serialize(&self) -> trc::Result> { + Ok(Vec::with_capacity(0)) } } @@ -605,7 +506,7 @@ impl Deserialize for () { } pub trait IntoOperations { - fn build(self, batch: &mut BatchBuilder); + fn build(self, batch: &mut BatchBuilder) -> trc::Result<()>; } impl Operation { @@ -690,15 +591,15 @@ impl From> for } } -impl Serialize for &Bincode { - fn serialize(self) -> Vec { - lz4_flex::compress_prepend_size(&bincode::serialize(&self.inner).unwrap_or_default()) - } -} - impl Serialize for Bincode { - fn serialize(self) -> Vec { - lz4_flex::compress_prepend_size(&bincode::serialize(&self.inner).unwrap_or_default()) + fn serialize(&self) -> trc::Result> { + bincode::serialize(&self.inner) + .map(|bytes| lz4_flex::compress_prepend_size(&bytes)) + .map_err(|err| { + trc::StoreEvent::DeserializeError + .caused_by(trc::location!()) + .reason(err) + }) } } @@ -725,15 +626,69 @@ impl De } } -impl ToBitmaps for Bincode { - fn to_bitmaps(&self, _ops: &mut Vec, _field: u8, _set: bool) { - unreachable!() +impl Deserialize for ArchivedValue { + fn deserialize(bytes: &[u8]) -> trc::Result { + Ok(ArchivedValue { + inner: bytes.to_vec(), + _phantom: std::marker::PhantomData, + }) + } + + fn deserialize_owned(bytes: Vec) -> trc::Result { + Ok(ArchivedValue { + inner: bytes, + _phantom: std::marker::PhantomData, + }) } } -impl ToBitmaps for &Bincode { - fn to_bitmaps(&self, _ops: &mut Vec, _field: u8, _set: bool) { - unreachable!() +impl ArchivedValue +where + T: rkyv::Portable + + for<'a> rkyv::bytecheck::CheckBytes> + + Sync + + Send, +{ + pub fn unarchive(&self) -> trc::Result<&T> { + rkyv::access::(&self.inner).map_err(Into::into) + } + + pub fn unarchive_unsafe(&self) -> &T { + unsafe { rkyv::access_unchecked::(&self.inner) } + } + + pub fn deserialize(&self) -> trc::Result + where + T: rkyv::Deserialize>, + { + rkyv::access::(&self.inner) + .and_then(|value| rkyv::deserialize::(value)) + .map_err(Into::into) + } +} + +impl HashedValue> +where + T: rkyv::Portable + + for<'a> rkyv::bytecheck::CheckBytes> + + Sync + + Send, +{ + pub fn to_unarchived(&self) -> trc::Result> { + self.inner.unarchive().map(|inner| HashedValue { + hash: self.hash, + inner, + }) + } + + pub fn into_deserialized(self) -> trc::Result> + where + T: rkyv::Deserialize>, + { + self.inner.deserialize().map(|inner| HashedValue { + hash: self.hash, + inner, + }) } } diff --git a/crates/trc/Cargo.toml b/crates/trc/Cargo.toml index 13221373..2c162519 100644 --- a/crates/trc/Cargo.toml +++ b/crates/trc/Cargo.toml @@ -17,6 +17,7 @@ rtrb = "0.3.1" parking_lot = "0.12.3" tokio = { version = "1.23", features = ["net", "macros"] } ahash = "0.8.11" +rkyv = { version = "0.8.10", features = ["little_endian"] } [features] test_mode = [] diff --git a/crates/trc/src/event/conv.rs b/crates/trc/src/event/conv.rs index b37fcb69..cb2201dc 100644 --- a/crates/trc/src/event/conv.rs +++ b/crates/trc/src/event/conv.rs @@ -371,6 +371,14 @@ impl From<&mail_auth::SpfOutput> for Error { } } +impl From for Error { + fn from(value: rkyv::rancor::Error) -> Self { + Error::new(EventType::Store(StoreEvent::DeserializeError)) + .reason(value) + .details("Rkyv de/serialization failed") + } +} + pub trait AssertSuccess where Self: Sized, diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 012e87d2..ae76fa5f 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -37,6 +37,7 @@ quick_cache = "0.6.9" downcast-rs = "2.0.1" fast-float = "0.2.0" erased-serde = "0.4.5" +rkyv = { version = "0.8.10", features = ["little_endian"] } [target.'cfg(unix)'.dependencies] privdrop = "0.5.3" diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index f6c6bc4f..f3b7f078 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -28,8 +28,20 @@ pub use erased_serde; pub const BLOB_HASH_LEN: usize = 32; -#[derive(Clone, Debug, Default, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] -pub struct BlobHash([u8; BLOB_HASH_LEN]); +#[derive( + rkyv::Archive, + rkyv::Deserialize, + rkyv::Serialize, + Clone, + Debug, + Default, + PartialEq, + Eq, + Hash, + serde::Serialize, + serde::Deserialize, +)] +pub struct BlobHash(pub [u8; BLOB_HASH_LEN]); impl BlobHash { pub fn new_max() -> Self { @@ -59,6 +71,12 @@ impl From<&[u8]> for BlobHash { } } +impl From<&ArchivedBlobHash> for BlobHash { + fn from(value: &ArchivedBlobHash) -> Self { + value.0.as_slice().into() + } +} + impl From> for BlobHash { fn from(value: Vec) -> Self { value.as_slice().into() diff --git a/crates/utils/src/map/bitmap.rs b/crates/utils/src/map/bitmap.rs index 0759df74..368d2126 100644 --- a/crates/utils/src/map/bitmap.rs +++ b/crates/utils/src/map/bitmap.rs @@ -7,11 +7,25 @@ use std::ops::Deref; #[derive( - Debug, serde::Serialize, serde::Deserialize, Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash, + Debug, + rkyv::Archive, + rkyv::Deserialize, + rkyv::Serialize, + serde::Serialize, + serde::Deserialize, + Clone, + Copy, + PartialOrd, + Ord, + PartialEq, + Eq, + Hash, )] +#[rkyv(compare(PartialEq), derive(Debug))] pub struct Bitmap { pub bitmap: u64, #[serde(skip)] + #[rkyv(omit_bounds)] _state: std::marker::PhantomData, } @@ -38,6 +52,11 @@ impl Bitmap { self.bitmap |= items.bitmap; } + #[inline(always)] + pub fn union_raw(&mut self, items: impl Into) { + self.bitmap |= items.into(); + } + #[inline(always)] pub fn intersection(&mut self, items: &Bitmap) { self.bitmap &= items.bitmap; @@ -103,6 +122,24 @@ impl Bitmap { } } +impl From> for Bitmap { + fn from(value: ArchivedBitmap) -> Self { + Self { + bitmap: value.bitmap.into(), + _state: std::marker::PhantomData, + } + } +} + +impl From<&ArchivedBitmap> for Bitmap { + fn from(value: &ArchivedBitmap) -> Self { + Self { + bitmap: value.bitmap.into(), + _state: std::marker::PhantomData, + } + } +} + impl From for Bitmap { fn from(value: u64) -> Self { Self { diff --git a/crates/utils/src/map/vec_map.rs b/crates/utils/src/map/vec_map.rs index 5d49498b..37cd35cd 100644 --- a/crates/utils/src/map/vec_map.rs +++ b/crates/utils/src/map/vec_map.rs @@ -6,18 +6,19 @@ use std::{borrow::Borrow, cmp::Ordering, fmt, hash::Hash}; +use rkyv::Archive; use serde::{Deserialize, Serialize, de::DeserializeOwned, ser::SerializeMap}; // A map implemented using vectors // used for small datasets of less than 20 items // and when deserializing from JSON -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] pub struct VecMap { inner: Vec>, } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq, Hash)] pub struct KeyValue { key: K, value: V, @@ -231,6 +232,28 @@ impl VecMap { } } +impl ArchivedVecMap { + pub fn len(&self) -> usize { + self.inner.len() + } + + pub fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + #[inline(always)] + pub fn iter( + &self, + ) -> impl Iterator< + Item = ( + &::Archived, + &::Archived, + ), + > { + self.inner.iter().map(|kv| (&kv.key, &kv.value)) + } +} + impl IntoIterator for VecMap { type Item = (K, V); diff --git a/tests/Cargo.toml b/tests/Cargo.toml index b054e41f..ee34c523 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -5,8 +5,8 @@ edition = "2024" resolver = "2" [features] -#default = ["sqlite", "postgres", "mysql", "rocks", "elastic", "s3", "redis", "azure", "foundationdb"] -default = ["sqlite", "postgres", "mysql", "rocks", "s3", "redis"] +default = ["sqlite", "postgres", "mysql", "rocks", "elastic", "s3", "redis", "azure", "foundationdb"] +#default = ["sqlite", "postgres", "mysql", "rocks", "s3", "redis"] #default = ["rocks", "redis", "s3"] sqlite = ["store/sqlite"] foundationdb = ["store/foundation", "common/foundation"] @@ -29,7 +29,7 @@ imap_proto = { path = "../crates/imap-proto" } pop3 = { path = "../crates/pop3", features = ["test_mode"] } smtp = { path = "../crates/smtp", features = ["test_mode", "enterprise"] } common = { path = "../crates/common", features = ["test_mode", "enterprise"] } -email = { path = "../crates/email", features = ["test_mode"] } +email = { path = "../crates/email", features = ["test_mode", "enterprise"] } spam-filter = { path = "../crates/spam-filter", features = ["test_mode", "enterprise"] } trc = { path = "../crates/trc" } managesieve = { path = "../crates/managesieve", features = ["test_mode", "enterprise"] } diff --git a/tests/src/jmap/email_changes.rs b/tests/src/jmap/email_changes.rs index 10702cd8..a8e6eb68 100644 --- a/tests/src/jmap/email_changes.rs +++ b/tests/src/jmap/email_changes.rs @@ -5,12 +5,12 @@ */ use jmap_proto::{ - parser::{json::Parser, JsonObjectParser}, + parser::{JsonObjectParser, json::Parser}, types::{collection::Collection, id::Id, state::State}, }; use store::{ ahash::AHashSet, - write::{log::ChangeLogBuilder, BatchBuilder}, + write::{BatchBuilder, log::ChangeLogBuilder}, }; use crate::jmap::assert_is_empty; @@ -160,6 +160,7 @@ pub async fn test(params: &mut JMAPTest) { .with_account_id(1) .with_collection(Collection::Email) .custom(changelog) + .unwrap() .build_batch(), ) .await diff --git a/tests/src/jmap/email_query_changes.rs b/tests/src/jmap/email_query_changes.rs index fea2f678..addbfc73 100644 --- a/tests/src/jmap/email_query_changes.rs +++ b/tests/src/jmap/email_query_changes.rs @@ -13,7 +13,7 @@ use jmap_proto::types::{collection::Collection, id::Id, property::Property, stat use store::{ ahash::{AHashMap, AHashSet}, - write::{log::ChangeLogBuilder, BatchBuilder, MaybeDynamicId, TagValue, F_BITMAP, F_CLEAR}, + write::{BatchBuilder, MaybeDynamicId, TagValue, log::ChangeLogBuilder}, }; use crate::jmap::{ @@ -142,18 +142,15 @@ pub async fn test(params: &mut JMAPTest) { .create_document() .with_collection(Collection::Email) .update_document(id.document_id()) - .value(Property::ThreadId, id.prefix_id(), F_BITMAP | F_CLEAR) + .untag(Property::ThreadId, id.prefix_id()) .set(Property::ThreadId, MaybeDynamicId::Dynamic(0)) - .tag( - Property::ThreadId, - TagValue::Id(MaybeDynamicId::Dynamic(0)), - 0, - ) + .tag(Property::ThreadId, TagValue::Id(MaybeDynamicId::Dynamic(0))) .custom(server.begin_changes(1).unwrap().with_log_move( Collection::Email, id, new_id, )) + .unwrap() .build_batch(), ) .await diff --git a/tests/src/store/blob.rs b/tests/src/store/blob.rs index 0163a492..082d4419 100644 --- a/tests/src/store/blob.rs +++ b/tests/src/store/blob.rs @@ -6,12 +6,12 @@ use ahash::AHashMap; use store::{ - write::{blob::BlobQuota, now, BatchBuilder, BlobOp}, - BlobClass, BlobStore, Serialize, Stores, + BlobClass, BlobStore, SerializeInfallible, Stores, + write::{BatchBuilder, BlobOp, blob::BlobQuota, now}, }; -use utils::{config::Config, BlobHash}; +use utils::{BlobHash, config::Config}; -use crate::store::{TempDir, CONFIG}; +use crate::store::{CONFIG, TempDir}; #[tokio::test] pub async fn blob_tests() { @@ -74,35 +74,41 @@ pub async fn blob_tests() { // Blob hash should now exist assert!(store.blob_exists(&hash).await.unwrap()); - assert!(blob_store - .get_blob(hash.as_ref(), 0..usize::MAX) - .await - .unwrap() - .is_some()); + assert!( + blob_store + .get_blob(hash.as_ref(), 0..usize::MAX) + .await + .unwrap() + .is_some() + ); // AccountId 0 should be able to read blob - assert!(store - .blob_has_access( - &hash, - BlobClass::Reserved { - account_id: 0, - expires: until - } - ) - .await - .unwrap()); + assert!( + store + .blob_has_access( + &hash, + BlobClass::Reserved { + account_id: 0, + expires: until + } + ) + .await + .unwrap() + ); // AccountId 1 should not be able to read blob - assert!(!store - .blob_has_access( - &hash, - BlobClass::Reserved { - account_id: 1, - expires: until - } - ) - .await - .unwrap()); + assert!( + !store + .blob_has_access( + &hash, + BlobClass::Reserved { + account_id: 1, + expires: until + } + ) + .await + .unwrap() + ); // Blob already expired, quota should be 0 tokio::time::sleep(std::time::Duration::from_secs(1)).await; @@ -118,23 +124,27 @@ pub async fn blob_tests() { assert!(!store.blob_exists(&hash).await.unwrap()); // AccountId 0 should not be able to read blob - assert!(!store - .blob_has_access( - &hash, - BlobClass::Reserved { - account_id: 0, - expires: until - } - ) - .await - .unwrap()); + assert!( + !store + .blob_has_access( + &hash, + BlobClass::Reserved { + account_id: 0, + expires: until + } + ) + .await + .unwrap() + ); // Blob should no longer be in store - assert!(blob_store - .get_blob(hash.as_ref(), 0..usize::MAX) - .await - .unwrap() - .is_none()); + assert!( + blob_store + .get_blob(hash.as_ref(), 0..usize::MAX) + .await + .unwrap() + .is_none() + ); // Upload one linked blob to accountId 1, two linked blobs to accountId 0, and three unlinked (reserved) blobs to accountId 2 let expiry_times = AHashMap::from_iter([ @@ -260,17 +270,19 @@ pub async fn blob_tests() { } // AccountId 0 should not have access to accountId 1's blobs - assert!(!store - .blob_has_access( - BlobHash::from(b"123".as_slice()), - BlobClass::Linked { - account_id: 0, - collection: 0, - document_id: 0, - } - ) - .await - .unwrap()); + assert!( + !store + .blob_has_access( + BlobHash::from(b"123".as_slice()), + BlobClass::Linked { + account_id: 0, + collection: 0, + document_id: 0, + } + ) + .await + .unwrap() + ); // Unlink blob store @@ -432,11 +444,13 @@ async fn test_store(store: BlobStore) { std::str::from_utf8(&DATA[11..57]).unwrap() ); assert!(store.delete_blob(hash.as_slice()).await.unwrap()); - assert!(store - .get_blob(hash.as_slice(), 0..usize::MAX) - .await - .unwrap() - .is_none()); + assert!( + store + .get_blob(hash.as_slice(), 0..usize::MAX) + .await + .unwrap() + .is_none() + ); // Test large blob let mut data = Vec::with_capacity(50 * 1024 * 1024); @@ -471,9 +485,11 @@ async fn test_store(store: BlobStore) { std::str::from_utf8(&data[3000111..4000999]).unwrap() ); assert!(store.delete_blob(hash.as_slice()).await.unwrap()); - assert!(store - .get_blob(hash.as_slice(), 0..usize::MAX) - .await - .unwrap() - .is_none()); + assert!( + store + .get_blob(hash.as_slice(), 0..usize::MAX) + .await + .unwrap() + .is_none() + ); } diff --git a/tests/src/store/ops.rs b/tests/src/store/ops.rs index c3ae4e3c..b6547ace 100644 --- a/tests/src/store/ops.rs +++ b/tests/src/store/ops.rs @@ -8,10 +8,8 @@ use std::collections::HashSet; use jmap_proto::types::{collection::Collection, property::Property}; use store::{ - write::{ - BatchBuilder, BitmapClass, DirectoryClass, MaybeDynamicId, TagValue, ValueClass, F_CLEAR, - }, BitmapKey, Store, ValueKey, + write::{BatchBuilder, BitmapClass, DirectoryClass, MaybeDynamicId, TagValue, ValueClass}, }; // FDB max value @@ -107,11 +105,7 @@ pub async fn test(db: Store) { .create_document() .with_collection(Collection::Email) .create_document() - .tag( - Property::ThreadId, - TagValue::Id(MaybeDynamicId::Dynamic(0)), - 0, - ) + .tag(Property::ThreadId, TagValue::Id(MaybeDynamicId::Dynamic(0))) .set(Property::ThreadId, MaybeDynamicId::Dynamic(0)); let assigned_ids = db.write(builder.build_batch()).await.unwrap(); @@ -180,10 +174,9 @@ pub async fn test(db: Store) { .delete_document(thread_id) .with_collection(Collection::Email) .delete_document(email_id) - .tag( + .untag( Property::ThreadId, TagValue::Id(MaybeDynamicId::Static(thread_id)), - F_CLEAR, ) .clear(Property::ThreadId); db.write(builder.build_batch()).await.unwrap(); diff --git a/tests/src/store/query.rs b/tests/src/store/query.rs index ffb4531c..8853079a 100644 --- a/tests/src/store/query.rs +++ b/tests/src/store/query.rs @@ -14,17 +14,17 @@ use std::{ use jmap_proto::types::keyword::Keyword; use nlp::language::Language; use store::{ + FtsStore, SerializeInfallible, ahash::AHashMap, - fts::{index::FtsDocument, Field, FtsFilter}, + fts::{Field, FtsFilter, index::FtsDocument}, query::sort::Pagination, write::ValueClass, - FtsStore, }; use store::{ - query::{Comparator, Filter}, - write::{BatchBuilder, F_BITMAP, F_INDEX, F_VALUE}, Store, ValueKey, + query::{Comparator, Filter}, + write::BatchBuilder, }; use crate::store::deflate_test_resource; @@ -145,10 +145,9 @@ pub async fn test(db: Store, fts_store: FtsStore, do_insert: bool) { match FIELDS_OPTIONS[pos] { FieldType::Text => { if !field.is_empty() { - builder.value( - field_id, - field.to_lowercase(), - F_VALUE | F_BITMAP, + builder.tag(field_id, field.to_lowercase()).set( + ValueClass::Property(field_id), + field.to_lowercase().into_bytes(), ); } } @@ -160,24 +159,25 @@ pub async fn test(db: Store, fts_store: FtsStore, do_insert: bool) { Language::English, ); if field_id == 7 { - builder.value(field_id, field.to_lowercase(), F_INDEX); + builder.index(field_id, field.to_lowercase()); } } } FieldType::Integer => { - builder.value( - field_id, - field.parse::().unwrap_or(0), - F_VALUE | F_INDEX, - ); + let field = field.parse::().unwrap_or(0); + builder + .index(field_id, field.serialize()) + .set(ValueClass::Property(field_id), field.serialize()); } FieldType::Keyword => { if !field.is_empty() { - builder.value( - field_id, - Keyword::Other(field.to_lowercase()), - F_VALUE | F_INDEX | F_BITMAP, - ); + builder + .set( + ValueClass::Property(field_id), + field.to_lowercase().into_bytes(), + ) + .tag(field_id, Keyword::Other(field.to_lowercase())) + .index(field_id, field.to_lowercase()); } } } @@ -270,7 +270,7 @@ pub async fn test_filter(db: Store, fts: FtsStore) { .await .unwrap(), ), - Filter::eq(fields_u8["year"], 1979u32), + Filter::eq(fields_u8["year"], 1979u32.serialize()), ], vec!["p11293"], ), @@ -288,9 +288,9 @@ pub async fn test_filter(db: Store, fts: FtsStore) { .await .unwrap(), ), - Filter::gt(fields_u8["year"], 2000u32), - Filter::lt(fields_u8["width"], 180u32), - Filter::gt(fields_u8["width"], 0u32), + Filter::gt(fields_u8["year"], 2000u32.serialize()), + Filter::lt(fields_u8["width"], 180u32.serialize()), + Filter::gt(fields_u8["width"], 0u32.serialize()), ], vec!["p79426", "p79427", "p79428", "p79429", "p79430"], ), @@ -332,8 +332,8 @@ pub async fn test_filter(db: Store, fts: FtsStore) { Keyword::Other("artist".to_string()), ), Filter::Or, - Filter::eq(fields_u8["year"], 1969u32), - Filter::eq(fields_u8["year"], 1971u32), + Filter::eq(fields_u8["year"], 1969u32.serialize()), + Filter::eq(fields_u8["year"], 1971u32.serialize()), Filter::End, ], vec!["p01764", "t05843"], @@ -356,12 +356,12 @@ pub async fn test_filter(db: Store, fts: FtsStore) { ), Filter::Or, Filter::And, - Filter::ge(fields_u8["year"], 1900u32), - Filter::lt(fields_u8["year"], 1910u32), + Filter::ge(fields_u8["year"], 1900u32.serialize()), + Filter::lt(fields_u8["year"], 1910u32.serialize()), Filter::End, Filter::And, - Filter::ge(fields_u8["year"], 2000u32), - Filter::lt(fields_u8["year"], 2010u32), + Filter::ge(fields_u8["year"], 2000u32.serialize()), + Filter::lt(fields_u8["year"], 2010u32.serialize()), Filter::End, Filter::End, ], @@ -391,14 +391,14 @@ pub async fn test_filter(db: Store, fts: FtsStore) { Filter::End, Filter::Not, Filter::Or, - Filter::gt(fields_u8["year"], 1980u32), + Filter::gt(fields_u8["year"], 1980u32.serialize()), Filter::And, - Filter::gt(fields_u8["width"], 500u32), - Filter::gt(fields_u8["height"], 500u32), + Filter::gt(fields_u8["width"], 500u32.serialize()), + Filter::gt(fields_u8["height"], 500u32.serialize()), Filter::End, Filter::End, Filter::End, - Filter::eq(fields_u8["acquisitionYear"], 2008u32), + Filter::eq(fields_u8["acquisitionYear"], 2008u32.serialize()), Filter::End, ], vec!["ar00039", "t12600"], @@ -425,8 +425,8 @@ pub async fn test_filter(db: Store, fts: FtsStore) { .await .unwrap(), ), - Filter::gt(fields_u8["year"], 1900u32), - Filter::gt(fields_u8["acquisitionYear"], 2000u32), + Filter::gt(fields_u8["year"], 1900u32.serialize()), + Filter::gt(fields_u8["acquisitionYear"], 2000u32.serialize()), ], vec![ "p80042", "p80043", "p80044", "p80045", "p80203", "t11937", "t12172", @@ -473,9 +473,9 @@ pub async fn test_sort(db: Store) { let tests = [ ( vec![ - Filter::gt(fields["year"], 0u32), - Filter::gt(fields["acquisitionYear"], 0u32), - Filter::gt(fields["width"], 0u32), + Filter::gt(fields["year"], 0u32.serialize()), + Filter::gt(fields["acquisitionYear"], 0u32.serialize()), + Filter::gt(fields["width"], 0u32.serialize()), ], vec![ Comparator::descending(fields["year"]), @@ -495,8 +495,8 @@ pub async fn test_sort(db: Store) { ), ( vec![ - Filter::gt(fields["width"], 0u32), - Filter::gt(fields["height"], 0u32), + Filter::gt(fields["width"], 0u32.serialize()), + Filter::gt(fields["height"], 0u32.serialize()), ], vec![ Comparator::descending(fields["width"]),