From d16119f54ba73a6cf33620fb78bfbd4beedf5dc4 Mon Sep 17 00:00:00 2001 From: Mauro D Date: Thu, 8 Feb 2024 20:03:57 -0300 Subject: [PATCH] Distributed SMTP queues (untested) --- Cargo.lock | 630 +++++++++--------- crates/jmap-proto/src/types/blob.rs | 11 +- crates/jmap/src/blob/download.rs | 3 +- crates/jmap/src/blob/upload.rs | 3 +- crates/jmap/src/email/copy.rs | 4 +- crates/jmap/src/email/get.rs | 4 +- crates/jmap/src/email/index.rs | 7 +- crates/jmap/src/email/metadata.rs | 2 +- crates/jmap/src/email/set.rs | 7 +- crates/jmap/src/email/snippet.rs | 4 +- crates/jmap/src/lib.rs | 58 +- crates/jmap/src/services/index.rs | 4 +- crates/jmap/src/services/ingest.rs | 10 +- crates/jmap/src/sieve/get.rs | 4 +- crates/jmap/src/sieve/ingest.rs | 4 +- crates/jmap/src/submission/get.rs | 38 +- crates/jmap/src/submission/set.rs | 33 +- crates/main/Cargo.toml | 2 +- crates/smtp/Cargo.toml | 2 +- crates/smtp/src/config/shared.rs | 11 + crates/smtp/src/core/eval.rs | 47 +- crates/smtp/src/core/management.rs | 592 +++++++++-------- crates/smtp/src/core/mod.rs | 23 +- crates/smtp/src/core/throttle.rs | 108 ++-- crates/smtp/src/core/worker.rs | 10 +- crates/smtp/src/inbound/data.rs | 43 +- crates/smtp/src/lib.rs | 18 +- crates/smtp/src/outbound/delivery.rs | 178 +++--- crates/smtp/src/outbound/local.rs | 2 +- crates/smtp/src/outbound/mod.rs | 6 +- crates/smtp/src/outbound/session.rs | 85 +-- crates/smtp/src/queue/dsn.rs | 114 ++-- crates/smtp/src/queue/manager.rs | 476 ++------------ crates/smtp/src/queue/mod.rs | 84 +-- crates/smtp/src/queue/quota.rs | 147 +++-- crates/smtp/src/queue/serialize.rs | 565 ----------------- crates/smtp/src/queue/spool.rs | 441 ++++++++----- crates/smtp/src/queue/throttle.rs | 74 +-- crates/smtp/src/reporting/dkim.rs | 2 +- crates/smtp/src/reporting/dmarc.rs | 534 +++++++++------- crates/smtp/src/reporting/mod.rs | 71 +-- crates/smtp/src/reporting/scheduler.rs | 629 +++--------------- crates/smtp/src/reporting/spf.rs | 2 +- crates/smtp/src/reporting/tls.rs | 703 +++++++++++---------- crates/smtp/src/scripts/event_loop.rs | 32 +- crates/smtp/src/scripts/plugins/bayes.rs | 26 +- crates/smtp/src/scripts/plugins/lookup.rs | 60 +- crates/store/Cargo.toml | 1 + crates/store/src/backend/redis/lookup.rs | 145 ++++- crates/store/src/dispatch/lookup.rs | 336 +++++++--- crates/store/src/dispatch/store.rs | 7 +- crates/store/src/lib.rs | 37 +- crates/store/src/write/blob.rs | 5 +- crates/store/src/write/key.rs | 98 ++- crates/store/src/write/mod.rs | 163 +++-- crates/utils/Cargo.toml | 1 + crates/utils/src/ipc.rs | 34 +- crates/utils/src/lib.rs | 60 ++ tests/Cargo.toml | 2 +- tests/resources/scripts/create_test_env.sh | 16 +- 60 files changed, 2990 insertions(+), 3828 deletions(-) delete mode 100644 crates/smtp/src/queue/serialize.rs diff --git a/Cargo.lock b/Cargo.lock index 79bea81c..fcd08e7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,7 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ "crypto-common", - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -125,9 +125,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.5" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" +checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5" dependencies = [ "anstyle", "anstyle-parse", @@ -139,9 +139,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" +checksum = "2faccea4cc4ab4a667ce676a30e8ec13922a692c99bb8f5b11f1502c72e04220" [[package]] name = "anstyle-parse" @@ -185,9 +185,9 @@ checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" [[package]] name = "argon2" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ba4cac0a46bc1d2912652a751c47f2a9f3a7fe89bcae2275d418f5270402f9" +checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" dependencies = [ "base64ct", "blake2", @@ -257,9 +257,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc2d0cfb2a7388d34f590e76686704c494ed7aaceed62ee1ba35cbf363abc2a5" +checksum = "a116f46a969224200a0a97f29cfd4c50e7534e4b4826bd23ea2c3c533039c82c" dependencies = [ "flate2", "futures-core", @@ -276,7 +276,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -298,7 +298,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -309,7 +309,7 @@ checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -440,9 +440,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.5" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "base64ct" @@ -491,17 +491,17 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.47", + "syn 2.0.48", "which", ] [[package]] name = "bindgen" -version = "0.69.1" +version = "0.69.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ffcebc3849946a7170a05992aac39da343a90676ab392c51a4280981d6379c2" +checksum = "a4c69fae65a523209d34240b60abe0c42d33d1045d445c0839d8a4894a736e2d" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cexpr", "clang-sys", "lazy_static", @@ -512,7 +512,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -538,9 +538,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" [[package]] name = "bitvec" @@ -592,7 +592,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -601,7 +601,7 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -610,7 +610,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -636,9 +636,9 @@ dependencies = [ [[package]] name = "borsh" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d4d6dafc1a3bb54687538972158f07b2c948bc57d5890df22c0739098b3028" +checksum = "f58b559fd6448c6e2fd0adb5720cd98a2506594cafa4737ff98c396f3e82f667" dependencies = [ "borsh-derive", "cfg_aliases", @@ -646,15 +646,15 @@ dependencies = [ [[package]] name = "borsh-derive" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4918709cc4dd777ad2b6303ed03cb37f3ca0ccede8c1b0d28ac6db8f4710e0" +checksum = "7aadb5b6ccbd078890f6d7003694e33816e6b784358f18e15e7e6d9f065a57cd" dependencies = [ "once_cell", - "proc-macro-crate 2.0.0", + "proc-macro-crate 3.1.0", "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", "syn_derive", ] @@ -707,9 +707,9 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6" +checksum = "ed2490600f404f2b94c167e31d3ed1d5f3c225a0f3b80230053b3e0b7b962bd9" [[package]] name = "byteorder" @@ -834,9 +834,9 @@ checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "9f13690e35a5e4ace198e7beea2895d29f3a9cc55015fcebe6336bd2010af9eb" dependencies = [ "android-tzdata", "iana-time-zone", @@ -844,7 +844,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.48.5", + "windows-targets 0.52.0", ] [[package]] @@ -853,7 +853,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12f8e7987cbd042a63249497f41aed09f8e65add917ea6566effbc56578d6801" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -880,9 +880,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.12" +version = "4.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcfab8ba68f3668e89f6ff60f5b205cea56aa7b769451a59f34b8682f51c056d" +checksum = "1e578d6ec4194633722ccf9544794b71b1385c3c027efe0c55db226fc880865c" dependencies = [ "clap_builder", "clap_derive", @@ -890,9 +890,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.12" +version = "4.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb7fb5e4e979aec3be7791562fcba452f94ad85e954da024396433e0e25a79e9" +checksum = "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7" dependencies = [ "anstream", "anstyle", @@ -909,7 +909,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -960,15 +960,15 @@ dependencies = [ [[package]] name = "console" -version = "0.15.7" +version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" dependencies = [ "encode_unicode 0.3.6", "lazy_static", "libc", "unicode-width", - "windows-sys 0.45.0", + "windows-sys 0.52.0", ] [[package]] @@ -1013,9 +1013,9 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] @@ -1037,11 +1037,10 @@ dependencies = [ [[package]] name = "crossbeam" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eb9105919ca8e40d437fc9cbb8f1975d916f1bd28afe795a48aae32a2cc8920" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" dependencies = [ - "cfg-if", "crossbeam-channel", "crossbeam-deque", "crossbeam-epoch", @@ -1051,54 +1050,46 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82a9b73a36529d9c47029b9fb3a6f0ea3cc916a261195352ba19e770fc1748b2" +checksum = "176dc175b78f56c0f321911d9c8eb2b77a78a4860b9c19db83835fea1a46649b" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ - "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.17" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e3681d554572a651dda4186cd47240627c3d0114d45a95f6ad27f2f22e7548d" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc6598521bb5a83d491e8c1fe51db7296019d2ca3cb93cc6c2a20369a4d78a2" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.18" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c" -dependencies = [ - "cfg-if", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crunchy" @@ -1112,7 +1103,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ - "generic-array", + "generic-array 0.14.7", "rand_core", "subtle", "zeroize", @@ -1124,7 +1115,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "generic-array", + "generic-array 0.14.7", "rand_core", "typenum", ] @@ -1135,7 +1126,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6" dependencies = [ - "generic-array", + "generic-array 0.14.7", "subtle", ] @@ -1194,7 +1185,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -1209,12 +1200,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.3" +version = "0.20.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" +checksum = "fc5d6b04b3fd0ba9926f945895de7d806260a2d7431ba82e7edaecb043c4c6b8" dependencies = [ - "darling_core 0.20.3", - "darling_macro 0.20.3", + "darling_core 0.20.5", + "darling_macro 0.20.5", ] [[package]] @@ -1233,16 +1224,16 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.3" +version = "0.20.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" +checksum = "04e48a959bcd5c761246f5d090ebc2fbf7b9cd527a492b07a67510c108f1e7e3" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -1258,13 +1249,13 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.20.3" +version = "0.20.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" +checksum = "1d1545d67a2149e1d93b7e5c7752dce5a7426eb5d1357ddcfd89336b94444f77" dependencies = [ - "darling_core 0.20.3", + "darling_core 0.20.5", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -1292,7 +1283,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd2735a791158376708f9347fe8faba9667589d82427ef3aed6794a8981de3d9" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -1403,7 +1394,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -1439,7 +1430,7 @@ dependencies = [ "pbkdf2 0.12.2", "pwhash", "regex", - "rustls 0.22.1", + "rustls 0.22.2", "rustls-pki-types", "scrypt", "serde", @@ -1502,7 +1493,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -1519,9 +1510,9 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "dsa" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d88afbb2443ba68ef8593de497e830b2e276434e1408f85cd760b1107b44ead0" +checksum = "48bc224a9084ad760195584ce5abb3c2c34a225fa312a128ad245a6b412b7689" dependencies = [ "digest 0.10.7", "num-bigint-dig", @@ -1577,11 +1568,11 @@ dependencies = [ [[package]] name = "ece" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dd5463ffecc0677adcd786c4481f73b215714d4757edf2eb37a573c03d00459" +checksum = "c2ea1d2f2cc974957a4e2575d8e5bb494549bab66338d6320c2789abcfff5746" dependencies = [ - "base64 0.13.1", + "base64 0.21.7", "byteorder", "hex", "hkdf", @@ -1654,7 +1645,7 @@ dependencies = [ "crypto-bigint", "digest 0.10.7", "ff", - "generic-array", + "generic-array 0.14.7", "group", "hkdf", "pem-rfc7468", @@ -1704,7 +1695,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -1748,7 +1739,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" dependencies = [ "bit-set", - "regex-automata 0.4.3", + "regex-automata 0.4.5", "regex-syntax 0.8.2", ] @@ -1905,7 +1896,7 @@ checksum = "83c8d52fe8b46ab822b4decdcc0d6d85aeedfc98f0d52ba2bd4aec4a97807516" dependencies = [ "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", "try_map", ] @@ -1943,7 +1934,7 @@ checksum = "b0fa992f1656e1707946bbba340ad244f0814009ef8c0118eb7b658395f19a2e" dependencies = [ "frunk_proc_macro_helpers", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -1955,7 +1946,7 @@ dependencies = [ "frunk_core", "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -1967,7 +1958,7 @@ dependencies = [ "frunk_core", "frunk_proc_macro_helpers", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -2032,7 +2023,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -2085,6 +2076,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "generic-array" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe739944a5406424e080edccb6add95685130b9f160d5407c639c7df0c5836b0" +dependencies = [ + "typenum", +] + [[package]] name = "gethostname" version = "0.4.3" @@ -2097,9 +2097,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if", "js-sys", @@ -2143,9 +2143,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.22" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" dependencies = [ "bytes", "fnv", @@ -2153,7 +2153,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.11", - "indexmap 2.1.0", + "indexmap 2.2.1", "slab", "tokio", "tokio-util", @@ -2162,9 +2162,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d308f63daf4181410c242d34c11f928dcb3aa105852019e043c9d1f4e4368a" +checksum = "31d030e59af851932b72ceebadf4a2b5986dba4c3b99dd2493f8273a0f151943" dependencies = [ "bytes", "fnv", @@ -2172,7 +2172,7 @@ dependencies = [ "futures-sink", "futures-util", "http 1.0.0", - "indexmap 2.1.0", + "indexmap 2.2.1", "slab", "tokio", "tokio-util", @@ -2215,9 +2215,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" [[package]] name = "hex" @@ -2408,7 +2408,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.3.22", + "h2 0.3.24", "http 0.2.11", "http-body 0.4.6", "httparse", @@ -2431,7 +2431,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.0", + "h2 0.4.2", "http 1.0.0", "http-body 1.0.0", "httparse", @@ -2469,12 +2469,11 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdea9aac0dbe5a9240d68cfd9501e2db94222c6dc06843e06640b9e07f0fdc67" +checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" dependencies = [ "bytes", - "futures-channel", "futures-util", "http 1.0.0", "http-body 1.0.0", @@ -2482,7 +2481,6 @@ dependencies = [ "pin-project-lite", "socket2 0.5.5", "tokio", - "tracing", ] [[package]] @@ -2565,7 +2563,7 @@ dependencies = [ "nlp", "parking_lot", "rand", - "rustls 0.22.1", + "rustls 0.22.2", "rustls-pemfile 2.0.0", "store", "tokio", @@ -2598,9 +2596,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.1.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +checksum = "433de089bd45971eecf4668ee0ee8f4cec17db4f8bd8f7bc3197a6ce37aa7d9b" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -2635,7 +2633,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" dependencies = [ "block-padding", - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -2735,7 +2733,7 @@ dependencies = [ "aes-gcm-siv", "async-stream", "async-trait", - "base64 0.21.5", + "base64 0.21.7", "bincode", "cbc", "chrono", @@ -2793,7 +2791,7 @@ dependencies = [ "maybe-async", "parking_lot", "reqwest", - "rustls 0.22.1", + "rustls 0.22.2", "rustls-pki-types", "serde", "serde_json", @@ -2827,9 +2825,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" +checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" dependencies = [ "wasm-bindgen", ] @@ -2840,7 +2838,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.1", ] [[package]] @@ -2945,9 +2943,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.151" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libloading" @@ -2971,7 +2969,7 @@ version = "0.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "libc", "redox_syscall", ] @@ -3005,9 +3003,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.12" +version = "1.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +checksum = "037731f5d3aaa87a5675e895b63ddff1a87624bc29f77004ea829809654e48f6" dependencies = [ "cc", "pkg-config", @@ -3022,9 +3020,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" @@ -3044,9 +3042,9 @@ checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "lru" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2994eeba8ed550fd9b47a0b38f0242bc3344e496483c6180b69139cc2fa5d1d7" +checksum = "db2c024b41519440580066ba82aab04092b333e09066a5eb86c7c4890df31f22" dependencies = [ "hashbrown 0.14.3", ] @@ -3072,9 +3070,9 @@ dependencies = [ [[package]] name = "lz4_flex" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ea9b256699eda7b0387ffbc776dd625e28bde3918446381781245b7a50349d8" +checksum = "912b45c753ff5f7f5208307e8ace7d2a2e30d024e26d3509f3dce546c044ce15" dependencies = [ "twox-hash", ] @@ -3125,10 +3123,10 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f5982137dccf24ca038daa022eb0ea97a31e9739477445a787a8ef77e015ebb" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "gethostname", "md5", - "rustls 0.22.1", + "rustls 0.22.2", "rustls-pki-types", "smtp-proto", "tokio", @@ -3168,7 +3166,7 @@ dependencies = [ "mail-send", "md5", "parking_lot", - "rustls 0.22.1", + "rustls 0.22.2", "rustls-pemfile 2.0.0", "sieve-rs", "store", @@ -3207,13 +3205,13 @@ checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "maybe-async" -version = "0.2.7" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f1b8c13cb1f814b634a96b2c725449fe7ed464a7b8781de8688be5ffbd3f305" +checksum = "afc95a651c82daf7004c824405aa1019723644950d488571bd718e3ed84646ed" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.48", ] [[package]] @@ -3313,14 +3311,14 @@ version = "0.30.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56b0d8a0db9bf6d2213e11f2c701cb91387b0614361625ab7b9743b41aa4938f" dependencies = [ - "darling 0.20.3", + "darling 0.20.5", "heck", "num-bigint", "proc-macro-crate 1.3.1", "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", "termcolor", "thiserror", ] @@ -3368,10 +3366,10 @@ version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06f19e4cfa0ab5a76b627cec2d81331c49b034988eaf302c3bafeada684eadef" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "bigdecimal", - "bindgen 0.69.1", - "bitflags 2.4.1", + "bindgen 0.69.2", + "bitflags 2.4.2", "bitvec", "btoi", "byteorder", @@ -3576,11 +3574,11 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.62" +version = "0.10.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cde4d2d9200ad5909f8dac647e29482e07c3a35de8a13fce7c9c7747ad9f671" +checksum = "15c9d69dd87a29568d4d017cfe8ec518706046a05184e5aea92d0af890b803c8" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cfg-if", "foreign-types", "libc", @@ -3597,7 +3595,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -3608,18 +3606,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "300.2.1+3.2.0" +version = "300.2.2+3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fe476c29791a5ca0d1273c697e96085bbabbbea2ef7afd5617e78a4b40332d3" +checksum = "8bbfad0063610ac26ee79f7484739e2b07555a75c42453b89263830b5c8103bc" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.98" +version = "0.9.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1665caf8ab2dc9aef43d1c0023bd904633a6a05cb30b0ad59bec2ae986e57a7" +checksum = "22e1bf214306098e4832460f797824c05d25aacdf896f64a985fb0fd992454ae" dependencies = [ "cc", "libc", @@ -3636,7 +3634,7 @@ checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" dependencies = [ "futures-core", "futures-sink", - "indexmap 2.1.0", + "indexmap 2.2.1", "js-sys", "once_cell", "pin-project-lite", @@ -3701,9 +3699,9 @@ dependencies = [ [[package]] name = "opentelemetry_sdk" -version = "0.21.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "968ba3f2ca03e90e5187f5e4f46c791ef7f2c163ae87789c8ce5f5ca3b7b7de5" +checksum = "2f16aec8a98a457a52664d69e0091bac3a0abd18ead9b641cb00202ba4e0efe4" dependencies = [ "async-trait", "crossbeam-channel", @@ -3839,7 +3837,7 @@ version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b8fcc794035347fb64beda2d3b462595dd2753e3f268d89c5aae77e8cf2c310" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "serde", ] @@ -3865,7 +3863,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.1.0", + "indexmap 2.2.1", ] [[package]] @@ -3908,7 +3906,7 @@ dependencies = [ "phf_shared 0.11.2", "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -3931,22 +3929,22 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" dependencies = [ "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -3984,9 +3982,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a" +checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" [[package]] name = "platforms" @@ -4018,7 +4016,7 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49b6c5ef183cd3ab4ba005f1ca64c21e8bd97ce4699cfea9e8d9a2c4958ca520" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "byteorder", "bytes", "fallible-iterator 0.2.0", @@ -4066,7 +4064,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" dependencies = [ "proc-macro2", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -4114,11 +4112,11 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "2.0.0" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8366a6159044a37876a2b9817124296703c586a5c92e2c53751fa06d8d43e8" +checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" dependencies = [ - "toml_edit 0.20.7", + "toml_edit 0.21.1", ] [[package]] @@ -4147,9 +4145,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.75" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "907a61bd0f64c2f29cd1cf1dc34d05176426a3f504a78010f08416ddb7b13708" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" dependencies = [ "unicode-ident", ] @@ -4361,9 +4359,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +checksum = "fa7237101a77a10773db45d62004a272517633fbcc3df19d96455ede1122e051" dependencies = [ "either", "rayon-core", @@ -4371,9 +4369,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ "crossbeam-deque", "crossbeam-utils", @@ -4381,9 +4379,9 @@ dependencies = [ [[package]] name = "rcgen" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d918c80c5a4c7560db726763020bd16db179e4d5b828078842274a443addb5d" +checksum = "48406db8ac1f3cbc7dcdb56ec355343817958a356ff430259bb07baf7607e1e1" dependencies = [ "pem", "ring 0.17.7", @@ -4444,13 +4442,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.2" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.3", + "regex-automata 0.4.5", "regex-syntax 0.8.2", ] @@ -4465,9 +4463,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" dependencies = [ "aho-corasick", "memchr", @@ -4503,17 +4501,17 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.23" +version = "0.11.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" +checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" dependencies = [ "async-compression", - "base64 0.21.5", + "base64 0.21.7", "bytes", "encoding_rs", "futures-core", "futures-util", - "h2 0.3.22", + "h2 0.3.24", "http 0.2.11", "http-body 0.4.6", "hyper 0.14.28", @@ -4531,6 +4529,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", "system-configuration", "tokio", "tokio-rustls 0.24.1", @@ -4706,7 +4705,7 @@ version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "fallible-iterator 0.3.0", "fallible-streaming-iterator", "hashlink", @@ -4768,9 +4767,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.33.1" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06676aec5ccb8fc1da723cc8c0f9a46549f21ebb8753d3915c6c41db1e7f1dc4" +checksum = "d7de2711cae7bdec993f4d2319352599ceb0d003e9f7900ea7c6ef4c5fc16831" dependencies = [ "arrayvec", "borsh", @@ -4823,11 +4822,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.28" +version = "0.38.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" +checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "errno", "libc", "linux-raw-sys", @@ -4860,9 +4859,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.22.1" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe6b63262c9fcac8659abfaa96cac103d28166d3ff3eaf8f412e19f3ae9e5a48" +checksum = "e87c9956bd9807afa1f77e0f7594af32566e830e088a5576d27c5b6f30f49d41" dependencies = [ "log", "ring 0.17.7", @@ -4890,7 +4889,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", ] [[package]] @@ -4899,7 +4898,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "rustls-pki-types", ] @@ -5027,7 +5026,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct", "der", - "generic-array", + "generic-array 0.14.7", "pkcs8", "subtle", "zeroize", @@ -5079,14 +5078,14 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "sequoia-openpgp" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ea026cf8a70d331c742e3ad7e68fd405d0743ff86630fb4334a1bf8d0e194c7" +checksum = "26e9c71323d9848404e343a6b5c3a73de10bc496ca3481b66586ba9064de027e" dependencies = [ "aes", "aes-gcm", "anyhow", - "base64 0.21.5", + "base64 0.21.7", "block-padding", "blowfish 0.9.1", "buffered-reader", @@ -5106,7 +5105,7 @@ dependencies = [ "ed25519-dalek", "getrandom", "idea", - "idna 0.4.0", + "idna 0.5.0", "lalrpop", "lalrpop-util", "lazy_static", @@ -5133,9 +5132,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.194" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b114498256798c94a0689e1a15fec6005dee8ac1f41de56404b67afc2a4b773" +checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" dependencies = [ "serde_derive", ] @@ -5151,20 +5150,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.194" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3385e45322e8f9931410f01b3031ec534c3947d0e94c18049af4d9f9907d4e0" +checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" dependencies = [ "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] name = "serde_json" -version = "1.0.110" +version = "1.0.113" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fbd975230bada99c8bb618e0c365c2eefa219158d5c6c29610fd09ff1833257" +checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" dependencies = [ "itoa", "ryu", @@ -5227,7 +5226,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -5262,13 +5261,13 @@ checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" [[package]] name = "sha1collisiondetection" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c0b86a052106b16741199985c9ec2bf501f619f70c48fa479b44b093ad9a68" +checksum = "f1d5c4be690002e8a5d7638b0b7323f03c268c7a919bd8af69ce963a4dc83220" dependencies = [ "const-oid", "digest 0.10.7", - "generic-array", + "generic-array 1.0.0", ] [[package]] @@ -5312,9 +5311,9 @@ checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" [[package]] name = "shlex" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "sieve-rs" @@ -5379,9 +5378,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.2" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" [[package]] name = "smtp" @@ -5414,7 +5413,7 @@ dependencies = [ "rayon", "regex", "reqwest", - "rustls 0.22.1", + "rustls 0.22.2", "rustls-pemfile 2.0.0", "rustls-pki-types", "serde", @@ -5436,9 +5435,12 @@ dependencies = [ [[package]] name = "smtp-proto" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20b37ae016fedcac2174dd0e3029870154830a4d3fb10f533317f2604e72b343" +checksum = "34344dcc7dd10b3de9224fd68e5f019fff2246d9cdf8e4322f770f48030e0c83" +dependencies = [ + "serde", +] [[package]] name = "snafu" @@ -5533,7 +5535,7 @@ dependencies = [ name = "stalwart-install" version = "0.5.3" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "clap", "dialoguer", "flate2", @@ -5561,6 +5563,7 @@ version = "0.1.0" dependencies = [ "ahash 0.8.7", "async-trait", + "bincode", "blake3", "bytes", "deadpool", @@ -5588,7 +5591,7 @@ dependencies = [ "rocksdb", "rusqlite", "rust-s3", - "rustls 0.22.1", + "rustls 0.22.2", "rustls-pki-types", "serde", "serde_json", @@ -5659,9 +5662,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.47" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1726efe18f42ae774cc644f330953a5e7b3c3003d3edcecf18850fe9d4dd9afb" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -5677,7 +5680,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -5762,9 +5765,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ "winapi-util", ] @@ -5775,7 +5778,7 @@ version = "0.1.0" dependencies = [ "ahash 0.8.7", "async-trait", - "base64 0.21.5", + "base64 0.21.7", "bytes", "chrono", "csv", @@ -5801,7 +5804,7 @@ dependencies = [ "num_cpus", "rayon", "reqwest", - "rustls 0.22.1", + "rustls 0.22.2", "rustls-pemfile 2.0.0", "rustls-pki-types", "serde", @@ -5835,7 +5838,7 @@ checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -5948,7 +5951,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -5993,7 +5996,7 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" dependencies = [ - "rustls 0.22.1", + "rustls 0.22.2", "rustls-pki-types", "tokio", ] @@ -6017,7 +6020,7 @@ checksum = "c83b561d025642014097b66e6c1bb422783339e0909e4429cde4749d1990bc38" dependencies = [ "futures-util", "log", - "rustls 0.22.1", + "rustls 0.22.2", "rustls-pki-types", "tokio", "tokio-rustls 0.25.0", @@ -6051,18 +6054,18 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.1", "toml_datetime", "winnow", ] [[package]] name = "toml_edit" -version = "0.20.7" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" +checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.1", "toml_datetime", "winnow", ] @@ -6075,11 +6078,11 @@ checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" dependencies = [ "async-trait", "axum", - "base64 0.21.5", + "base64 0.21.7", "bytes", "futures-core", "futures-util", - "h2 0.3.22", + "h2 0.3.24", "http 0.2.11", "http-body 0.4.6", "hyper 0.14.28", @@ -6158,7 +6161,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -6254,7 +6257,7 @@ dependencies = [ "httparse", "log", "rand", - "rustls 0.22.1", + "rustls 0.22.2", "rustls-pki-types", "sha1", "thiserror", @@ -6290,9 +6293,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "typewit" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6779a69cc5f9a7388274a0a8a353eb1c9e45195f9ae74a26690b055a7cf9592a" +checksum = "c6fb9ae6a3cafaf0a5d14c2302ca525f9ae8e07a0f0e6949de88d882c37a6e24" dependencies = [ "typewit_proc_macros", ] @@ -6314,9 +6317,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" @@ -6418,7 +6421,8 @@ version = "0.5.3" dependencies = [ "ahash 0.8.7", "arc-swap", - "base64 0.21.5", + "base64 0.21.7", + "blake3", "chrono", "dashmap", "futures", @@ -6437,7 +6441,7 @@ dependencies = [ "regex", "reqwest", "ring 0.17.7", - "rustls 0.22.1", + "rustls 0.22.2", "rustls-pemfile 2.0.0", "rustls-pki-types", "serde", @@ -6456,9 +6460,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" +checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" dependencies = [ "getrandom", ] @@ -6504,9 +6508,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" +checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -6514,24 +6518,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" +checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.39" +version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" +checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" dependencies = [ "cfg-if", "js-sys", @@ -6541,9 +6545,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" +checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -6551,28 +6555,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" +checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" +checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" [[package]] name = "wasm-streams" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" dependencies = [ "futures-util", "js-sys", @@ -6583,9 +6587,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" +checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" dependencies = [ "js-sys", "wasm-bindgen", @@ -6646,11 +6650,11 @@ dependencies = [ [[package]] name = "whatlang" -version = "0.16.3" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcdcd0195a5b871e50926da8e881277f36a4621b3220d85092e7b91cc85f6bd9" +checksum = "471d1c1645d361eb782a1650b1786a8fb58dd625e681a04c09f5ff7c8764a7b0" dependencies = [ - "hashbrown 0.12.3", + "hashbrown 0.14.3", "once_cell", ] @@ -6722,15 +6726,6 @@ dependencies = [ "windows-targets 0.52.0", ] -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" @@ -6749,21 +6744,6 @@ dependencies = [ "windows-targets 0.52.0", ] -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-targets" version = "0.48.5" @@ -6794,12 +6774,6 @@ dependencies = [ "windows_x86_64_msvc 0.52.0", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -6812,12 +6786,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -6830,12 +6798,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -6848,12 +6810,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -6866,12 +6822,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -6884,12 +6834,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -6902,12 +6846,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -6922,9 +6860,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.32" +version = "0.5.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8434aeec7b290e8da5c3f0d628cb0eac6cabcb31d14bb74f779a08109a5914d6" +checksum = "818ce546a11a9986bc24f93d0cdf38a8a1a400f1473ea8c82e59f6e0ffab9249" dependencies = [ "memchr", ] @@ -6978,9 +6916,9 @@ dependencies = [ [[package]] name = "xattr" -version = "1.2.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "914566e6413e7fa959cc394fb30e563ba80f3541fbd40816d4c05a0fc3f2a0f1" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" dependencies = [ "libc", "linux-raw-sys", @@ -7025,7 +6963,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -7045,7 +6983,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -7070,9 +7008,9 @@ dependencies = [ [[package]] name = "zip-extract" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb654964c003959ed64cbd0d7b329bcdcbd9690facd50c8617748d3622543972" +checksum = "e109e5a291403b4c1e514d39f8a22d3f98d257e691a52bb1f16051bb1ffed63e" dependencies = [ "log", "thiserror", diff --git a/crates/jmap-proto/src/types/blob.rs b/crates/jmap-proto/src/types/blob.rs index 3f9a2f7b..34b1ebae 100644 --- a/crates/jmap-proto/src/types/blob.rs +++ b/crates/jmap-proto/src/types/blob.rs @@ -25,11 +25,14 @@ use std::{borrow::Borrow, io::Write}; use store::{ write::{DeserializeFrom, SerializeInto}, - BlobClass, BlobHash, + BlobClass, }; -use utils::codec::{ - base32_custom::{Base32Reader, Base32Writer}, - leb128::{Leb128Iterator, Leb128Writer}, +use utils::{ + codec::{ + base32_custom::{Base32Reader, Base32Writer}, + leb128::{Leb128Iterator, Leb128Writer}, + }, + BlobHash, }; use crate::parser::{base32::JsonBase32Reader, json::Parser, JsonObjectParser}; diff --git a/crates/jmap/src/blob/download.rs b/crates/jmap/src/blob/download.rs index 14ae820d..a01e3f2e 100644 --- a/crates/jmap/src/blob/download.rs +++ b/crates/jmap/src/blob/download.rs @@ -35,7 +35,8 @@ use mail_parser::{ decoders::{base64::base64_decode, quoted_printable::quoted_printable_decode}, Encoding, }; -use store::{BlobClass, BlobHash}; +use store::BlobClass; +use utils::BlobHash; use crate::{auth::AccessToken, JMAP}; diff --git a/crates/jmap/src/blob/upload.rs b/crates/jmap/src/blob/upload.rs index 086e5d9f..8bf493bf 100644 --- a/crates/jmap/src/blob/upload.rs +++ b/crates/jmap/src/blob/upload.rs @@ -33,8 +33,9 @@ use jmap_proto::{ }; use store::{ write::{now, BatchBuilder, BlobOp}, - BlobClass, BlobHash, Serialize, + BlobClass, Serialize, }; +use utils::BlobHash; use crate::{auth::AccessToken, JMAP}; diff --git a/crates/jmap/src/email/copy.rs b/crates/jmap/src/email/copy.rs index 36845853..7c4cb5a6 100644 --- a/crates/jmap/src/email/copy.rs +++ b/crates/jmap/src/email/copy.rs @@ -48,12 +48,12 @@ use jmap_proto::{ }; use mail_parser::{parsers::fields::thread::thread_name, HeaderName, HeaderValue}; use store::{ - write::{BatchBuilder, ValueClass, F_BITMAP, F_VALUE}, + write::{BatchBuilder, Bincode, ValueClass, F_BITMAP, F_VALUE}, BlobClass, }; use utils::map::vec_map::VecMap; -use crate::{auth::AccessToken, mailbox::UidMailbox, services::housekeeper::Event, Bincode, JMAP}; +use crate::{auth::AccessToken, mailbox::UidMailbox, services::housekeeper::Event, JMAP}; use super::{ index::{EmailIndexBuilder, TrimTextValue, VisitValues, MAX_ID_LENGTH, MAX_SORT_FIELD_LENGTH}, diff --git a/crates/jmap/src/email/get.rs b/crates/jmap/src/email/get.rs index fe05fe05..c3e64202 100644 --- a/crates/jmap/src/email/get.rs +++ b/crates/jmap/src/email/get.rs @@ -37,9 +37,9 @@ use jmap_proto::{ }, }; use mail_parser::HeaderName; -use store::BlobClass; +use store::{write::Bincode, BlobClass}; -use crate::{auth::AccessToken, email::headers::HeaderToValue, mailbox::UidMailbox, Bincode, JMAP}; +use crate::{auth::AccessToken, email::headers::HeaderToValue, mailbox::UidMailbox, JMAP}; use super::{ body::{ToBodyPart, TruncateBody}, diff --git a/crates/jmap/src/email/index.rs b/crates/jmap/src/email/index.rs index ee28e9b4..33769c6f 100644 --- a/crates/jmap/src/email/index.rs +++ b/crates/jmap/src/email/index.rs @@ -35,12 +35,13 @@ use store::{ backend::MAX_TOKEN_LENGTH, fts::{index::FtsDocument, Field}, write::{ - BatchBuilder, BlobOp, DirectoryClass, IntoOperations, F_BITMAP, F_CLEAR, F_INDEX, F_VALUE, + BatchBuilder, Bincode, BlobOp, DirectoryClass, IntoOperations, F_BITMAP, F_CLEAR, F_INDEX, + F_VALUE, }, - BlobHash, }; +use utils::BlobHash; -use crate::{mailbox::UidMailbox, Bincode}; +use crate::mailbox::UidMailbox; use super::metadata::MessageMetadata; diff --git a/crates/jmap/src/email/metadata.rs b/crates/jmap/src/email/metadata.rs index e51eec1e..677b34cc 100644 --- a/crates/jmap/src/email/metadata.rs +++ b/crates/jmap/src/email/metadata.rs @@ -32,7 +32,7 @@ use mail_parser::{ MessagePartId, MimeHeaders, PartType, }; use serde::{Deserialize, Serialize}; -use store::BlobHash; +use utils::BlobHash; #[derive(Debug, Serialize, Deserialize)] pub struct MessageMetadata<'x> { diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index 202e7316..779243e8 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -53,15 +53,14 @@ use mail_parser::MessageParser; use store::{ ahash::AHashSet, write::{ - assert::HashedValue, log::ChangeLogBuilder, BatchBuilder, DeserializeFrom, SerializeInto, - ToBitmaps, ValueClass, F_BITMAP, F_CLEAR, F_VALUE, + assert::HashedValue, log::ChangeLogBuilder, BatchBuilder, Bincode, DeserializeFrom, + SerializeInto, ToBitmaps, ValueClass, F_BITMAP, F_CLEAR, F_VALUE, }, Serialize, }; use crate::{ - auth::AccessToken, mailbox::UidMailbox, services::housekeeper::Event, Bincode, IngestError, - JMAP, + auth::AccessToken, mailbox::UidMailbox, services::housekeeper::Event, IngestError, JMAP, }; use super::{ diff --git a/crates/jmap/src/email/snippet.rs b/crates/jmap/src/email/snippet.rs index 0e5b4ede..4f55dca1 100644 --- a/crates/jmap/src/email/snippet.rs +++ b/crates/jmap/src/email/snippet.rs @@ -31,9 +31,9 @@ use jmap_proto::{ }; use mail_parser::{decoders::html::html_to_text, GetHeader, HeaderName, PartType}; use nlp::language::{search_snippet::generate_snippet, stemmer::Stemmer, Language}; -use store::backend::MAX_TOKEN_LENGTH; +use store::{backend::MAX_TOKEN_LENGTH, write::Bincode}; -use crate::{auth::AccessToken, Bincode, JMAP}; +use crate::{auth::AccessToken, JMAP}; use super::metadata::{MessageMetadata, MetadataPartType}; diff --git a/crates/jmap/src/lib.rs b/crates/jmap/src/lib.rs index 79ce7b89..8ee26c50 100644 --- a/crates/jmap/src/lib.rs +++ b/crates/jmap/src/lib.rs @@ -54,8 +54,8 @@ use store::{ fts::FtsFilter, query::{sort::Pagination, Comparator, Filter, ResultSet, SortedResultSet}, roaring::RoaringBitmap, - write::{BatchBuilder, BitmapClass, DirectoryClass, TagValue, ToBitmaps, ValueClass}, - BitmapKey, BlobStore, Deserialize, FtsStore, Serialize, Store, Stores, ValueKey, + write::{BatchBuilder, BitmapClass, DirectoryClass, TagValue, ValueClass}, + BitmapKey, BlobStore, Deserialize, FtsStore, Store, Stores, ValueKey, }; use tokio::sync::mpsc; use utils::{ @@ -171,10 +171,6 @@ pub struct Config { pub capabilities: BaseCapabilities, } -pub struct Bincode { - pub inner: T, -} - #[derive(Debug)] pub enum IngestError { Temporary, @@ -759,56 +755,6 @@ impl JMAP { } } -impl Bincode { - pub fn new(inner: T) -> Self { - Self { inner } - } -} - -impl Serialize for &Bincode { - fn serialize(self) -> Vec { - lz4_flex::compress_prepend_size(&bincode::serialize(&self.inner).unwrap_or_default()) - } -} - -impl Serialize for Bincode { - fn serialize(self) -> Vec { - lz4_flex::compress_prepend_size(&bincode::serialize(&self.inner).unwrap_or_default()) - } -} - -impl Deserialize - for Bincode -{ - fn deserialize(bytes: &[u8]) -> store::Result { - lz4_flex::decompress_size_prepended(bytes) - .map_err(|err| { - store::Error::InternalError(format!("Bincode decompression failed: {err:?}")) - }) - .and_then(|result| { - bincode::deserialize(&result).map_err(|err| { - store::Error::InternalError(format!( - "Bincode deserialization failed (len {}): {err:?}", - result.len() - )) - }) - }) - .map(|inner| Self { inner }) - } -} - -impl ToBitmaps for Bincode { - fn to_bitmaps(&self, _ops: &mut Vec, _field: u8, _set: bool) { - unreachable!() - } -} - -impl ToBitmaps for &Bincode { - fn to_bitmaps(&self, _ops: &mut Vec, _field: u8, _set: bool) { - unreachable!() - } -} - trait UpdateResults: Sized { fn update_results(&mut self, sorted_results: SortedResultSet) -> Result<(), MethodError>; } diff --git a/crates/jmap/src/services/index.rs b/crates/jmap/src/services/index.rs index 51bbd9d2..1192055f 100644 --- a/crates/jmap/src/services/index.rs +++ b/crates/jmap/src/services/index.rs @@ -24,13 +24,13 @@ use jmap_proto::types::{collection::Collection, property::Property}; use store::{ fts::index::FtsDocument, - write::{key::DeserializeBigEndian, BatchBuilder, ValueClass}, + write::{key::DeserializeBigEndian, BatchBuilder, Bincode, ValueClass}, Deserialize, IterateParams, ValueKey, U32_LEN, U64_LEN, }; use crate::{ email::{index::IndexMessageText, metadata::MessageMetadata}, - Bincode, JMAP, + JMAP, }; use super::housekeeper::Event; diff --git a/crates/jmap/src/services/ingest.rs b/crates/jmap/src/services/ingest.rs index 6e2aa61f..cf2e56ed 100644 --- a/crates/jmap/src/services/ingest.rs +++ b/crates/jmap/src/services/ingest.rs @@ -32,9 +32,13 @@ use crate::{email::ingest::IngestEmail, mailbox::INBOX_ID, IngestError, JMAP}; impl JMAP { pub async fn deliver_message(&self, message: IngestMessage) -> Vec { // Read message - let raw_message = match message.read_message().await { - Ok(raw_message) => raw_message, - Err(_) => { + let raw_message = match self + .blob_store + .get_blob(message.message_blob.as_slice(), 0..u32::MAX) + .await + { + Ok(Some(raw_message)) => raw_message, + _ => { return (0..message.recipients.len()) .map(|_| DeliveryResult::TemporaryFailure { reason: "Temporary I/O error.".into(), diff --git a/crates/jmap/src/sieve/get.rs b/crates/jmap/src/sieve/get.rs index c6cc7466..4b4d05f0 100644 --- a/crates/jmap/src/sieve/get.rs +++ b/crates/jmap/src/sieve/get.rs @@ -32,11 +32,11 @@ use jmap_proto::{ use sieve::Sieve; use store::{ query::Filter, - write::{assert::HashedValue, BatchBuilder, BlobOp}, + write::{assert::HashedValue, BatchBuilder, Bincode, BlobOp}, Deserialize, Serialize, }; -use crate::{sieve::SeenIds, Bincode, JMAP}; +use crate::{sieve::SeenIds, JMAP}; use super::ActiveScript; diff --git a/crates/jmap/src/sieve/ingest.rs b/crates/jmap/src/sieve/ingest.rs index 03b84178..9f87ff36 100644 --- a/crates/jmap/src/sieve/ingest.rs +++ b/crates/jmap/src/sieve/ingest.rs @@ -30,7 +30,7 @@ use sieve::{Envelope, Event, Input, Mailbox, Recipient}; use smtp::core::{Session, SessionAddress}; use store::{ ahash::AHashSet, - write::{now, BatchBuilder, F_VALUE}, + write::{now, BatchBuilder, Bincode, F_VALUE}, }; use utils::listener::stream::NullIo; @@ -38,7 +38,7 @@ use crate::{ email::ingest::{IngestEmail, IngestedEmail}, mailbox::{INBOX_ID, TRASH_ID}, sieve::SeenIdHash, - Bincode, IngestError, JMAP, + IngestError, JMAP, }; use super::ActiveScript; diff --git a/crates/jmap/src/submission/get.rs b/crates/jmap/src/submission/get.rs index 5cca5b29..b8b9e384 100644 --- a/crates/jmap/src/submission/get.rs +++ b/crates/jmap/src/submission/get.rs @@ -27,8 +27,7 @@ use jmap_proto::{ object::Object, types::{collection::Collection, property::Property, value::Value}, }; -use smtp::{core::management::QueueRequest, queue}; -use tokio::sync::oneshot; +use smtp::queue; use crate::JMAP; @@ -97,25 +96,10 @@ impl JMAP { }; // Obtain queueId - let mut queued_message = None; - let (result_tx, result_rx) = oneshot::channel(); - if self + let queued_message = self .smtp - .queue - .tx - .send(queue::Event::Manage(QueueRequest::Status { - queue_ids: vec![push.get(&Property::MessageId).as_uint().unwrap_or(u64::MAX)], - result_tx, - })) - .await - .is_ok() - { - queued_message = result_rx - .await - .ok() - .and_then(|mut result| result.pop()) - .flatten(); - } + .read_message(push.get(&Property::MessageId).as_uint().unwrap_or(u64::MAX)) + .await; let mut result = Object::with_capacity(properties.len()); for property in &properties { @@ -124,11 +108,7 @@ impl JMAP { Property::DeliveryStatus => { match (queued_message.as_ref(), push.remove(property)) { (Some(message), Value::Object(mut status)) => { - for rcpt in message - .domains - .iter() - .flat_map(|rcpts| rcpts.recipients.iter()) - { + for rcpt in &message.recipients { status.set( Property::_T(rcpt.address.clone()), Object::with_capacity(3) @@ -146,10 +126,12 @@ impl JMAP { .with_property( Property::SmtpReply, match &rcpt.status { - queue::Status::Completed(reply) - | queue::Status::TemporaryFailure(reply) + queue::Status::Completed(reply) => { + reply.response.message() + } + queue::Status::TemporaryFailure(reply) | queue::Status::PermanentFailure(reply) => { - reply.as_str() + reply.response.message() } queue::Status::Scheduled => "250 2.1.5 Queued", } diff --git a/crates/jmap/src/submission/set.rs b/crates/jmap/src/submission/set.rs index 1a27b51d..cf4998f0 100644 --- a/crates/jmap/src/submission/set.rs +++ b/crates/jmap/src/submission/set.rs @@ -48,19 +48,15 @@ use jmap_proto::{ }, }; use mail_parser::{HeaderName, HeaderValue}; -use smtp::{ - core::{management::QueueRequest, Session, SessionData, State}, - queue, -}; +use smtp::core::{Session, SessionData, State}; use smtp_proto::{request::parser::Rfc5321Parser, MailFrom, RcptTo}; -use store::write::{assert::HashedValue, log::ChangeLogBuilder, now, BatchBuilder}; -use tokio::sync::oneshot; +use store::write::{assert::HashedValue, log::ChangeLogBuilder, now, BatchBuilder, Bincode}; use utils::{ listener::{stream::NullIo, ServerInstance}, map::vec_map::VecMap, }; -use crate::{email::metadata::MessageMetadata, identity::set::sanitize_email, Bincode, JMAP}; +use crate::{email::metadata::MessageMetadata, identity::set::sanitize_email, JMAP}; pub static SCHEMA: &[IndexProperty] = &[ IndexProperty::new(Property::UndoStatus).index_as(IndexAs::Text { @@ -176,24 +172,11 @@ impl JMAP { match undo_status { Some(undo_status) if undo_status == "canceled" => { - let (result_tx, result_rx) = oneshot::channel(); - if self - .smtp - .queue - .tx - .send(queue::Event::Manage(QueueRequest::Cancel { - queue_ids: vec![queue_id], - item: None, - result_tx, - })) - .await - .is_ok() - && result_rx - .await - .ok() - .and_then(|mut r| r.pop()) - .unwrap_or(false) - { + if let Some(queue_message) = self.smtp.read_message(queue_id).await { + // Delete message from queue + let message_due = queue_message.next_event().unwrap_or_default(); + queue_message.remove(&self.smtp, message_due).await; + // Update record let mut batch = BatchBuilder::new(); batch diff --git a/crates/main/Cargo.toml b/crates/main/Cargo.toml index 5f52b29d..3bcaafb0 100644 --- a/crates/main/Cargo.toml +++ b/crates/main/Cargo.toml @@ -32,7 +32,7 @@ jemallocator = "0.5.0" [features] #default = ["sqlite", "foundationdb", "postgres", "mysql", "rocks", "elastic", "s3", "redis"] -default = ["sqlite", "postgres", "mysql"] +default = ["sqlite", "postgres", "mysql", "redis"] sqlite = ["store/sqlite"] foundationdb = ["store/foundation"] postgres = ["store/postgres"] diff --git a/crates/smtp/Cargo.toml b/crates/smtp/Cargo.toml index 2e689b8c..53580d1f 100644 --- a/crates/smtp/Cargo.toml +++ b/crates/smtp/Cargo.toml @@ -20,7 +20,7 @@ mail-auth = { version = "0.3" } mail-send = { version = "0.4", default-features = false, features = ["cram-md5"] } mail-parser = { version = "0.9", features = ["full_encoding", "ludicrous_mode"] } mail-builder = { version = "0.3", features = ["ludicrous_mode"] } -smtp-proto = { version = "0.1" } +smtp-proto = { version = "0.1", features = ["serde_support"] } sieve-rs = { version = "0.4" } ahash = { version = "0.8" } rustls = "0.22" diff --git a/crates/smtp/src/config/shared.rs b/crates/smtp/src/config/shared.rs index 80d42825..c0134e23 100644 --- a/crates/smtp/src/config/shared.rs +++ b/crates/smtp/src/config/shared.rs @@ -72,6 +72,17 @@ impl ConfigShared for Config { ) })? .clone(), + default_blob_store: self + .value_or_default("storage.blob", "storage.data") + .and_then(|id| ctx.stores.blob_stores.get(id)) + .ok_or_else(|| { + format!( + "Lookup store {:?} not found for key \"storage.blob\".", + self.value_or_default("storage.blob", "storage.data") + .unwrap() + ) + })? + .clone(), }) } diff --git a/crates/smtp/src/core/eval.rs b/crates/smtp/src/core/eval.rs index a7b8d2c8..1b90750a 100644 --- a/crates/smtp/src/core/eval.rs +++ b/crates/smtp/src/core/eval.rs @@ -3,16 +3,13 @@ use std::{borrow::Cow, net::IpAddr, sync::Arc, vec::IntoIter}; use directory::Directory; use mail_auth::IpLookupStrategy; use sieve::Sieve; -use store::{LookupKey, LookupStore, LookupValue}; +use store::{Deserialize, LookupStore}; use utils::{ config::if_block::IfBlock, expr::{Expression, Variable}, }; -use crate::{ - config::{ArcSealer, DkimSigner, RelayHost}, - scripts::plugins::lookup::VariableExists, -}; +use crate::config::{ArcSealer, DkimSigner, RelayHost}; use super::{ResolveVariable, SMTP}; @@ -165,15 +162,9 @@ impl SMTP { let key = params.next_as_string(); self.get_lookup_store(store.as_ref()) - .key_get::(LookupKey::Key(key.into_owned().into_bytes())) + .key_get::(key.into_owned().into_bytes()) .await - .map(|value| { - if let LookupValue::Value { value, .. } = value { - Variable::from(value) - } else { - Variable::default() - } - }) + .map(|value| value.map(|v| v.into_inner()).unwrap_or_default()) .unwrap_or_else(|err| { tracing::warn!( context = "eval_if", @@ -191,9 +182,8 @@ impl SMTP { let key = params.next_as_string(); self.get_lookup_store(store.as_ref()) - .key_get::(LookupKey::Key(key.into_owned().into_bytes())) + .key_exists(key.into_owned().into_bytes()) .await - .map(|value| matches!(value, LookupValue::Value { .. })) .unwrap_or_else(|err| { tracing::warn!( context = "eval_if", @@ -395,3 +385,30 @@ impl<'x> FncParams<'x> { self.params.next().unwrap().into_string() } } + +#[derive(Debug)] +struct VariableWrapper(Variable<'static>); + +impl From for VariableWrapper { + fn from(value: i64) -> Self { + VariableWrapper(Variable::Integer(value)) + } +} + +impl Deserialize for VariableWrapper { + fn deserialize(bytes: &[u8]) -> store::Result { + String::deserialize(bytes).map(|v| VariableWrapper(Variable::String(v.into()))) + } +} + +impl From> for VariableWrapper { + fn from(value: store::Value<'static>) -> Self { + VariableWrapper(value.into()) + } +} + +impl VariableWrapper { + pub fn into_inner(self) -> Variable<'static> { + self.0 + } +} diff --git a/crates/smtp/src/core/management.rs b/crates/smtp/src/core/management.rs index 246156e7..125e6102 100644 --- a/crates/smtp/src/core/management.rs +++ b/crates/smtp/src/core/management.rs @@ -21,7 +21,7 @@ * for more details. */ -use std::{borrow::Cow, fmt::Display, net::IpAddr, sync::Arc, time::Instant}; +use std::{borrow::Cow, net::IpAddr, sync::Arc}; use directory::{AuthResult, Type}; use http_body_util::{combinators::BoxBody, BodyExt, Empty, Full}; @@ -35,70 +35,24 @@ use hyper::{ use hyper_util::rt::TokioIo; use mail_parser::{decoders::base64::base64_decode, DateTime}; use mail_send::Credentials; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use tokio::sync::oneshot; +use serde::{Deserializer, Serializer}; +use store::{ + write::{key::DeserializeBigEndian, now, Bincode, QueueClass, ReportEvent, ValueClass}, + Deserialize, IterateParams, ValueKey, +}; use utils::listener::{limiter::InFlight, SessionData, SessionManager, SessionStream}; -use crate::{ - queue::{self, instant_to_timestamp, InstantFromTimestamp, QueueId, Status}, - reporting::{ - self, - scheduler::{ReportKey, ReportPolicy, ReportType, ReportValue}, - }, -}; +use crate::queue::{self, HostResponse, QueueId, Status}; use super::{SmtpAdminSessionManager, SMTP}; -#[derive(Debug)] -pub enum QueueRequest { - List { - from: Option, - to: Option, - before: Option, - after: Option, - result_tx: oneshot::Sender>, - }, - Status { - queue_ids: Vec, - result_tx: oneshot::Sender>>, - }, - Cancel { - queue_ids: Vec, - item: Option, - result_tx: oneshot::Sender>, - }, - Retry { - queue_ids: Vec, - item: Option, - time: Instant, - result_tx: oneshot::Sender>, - }, -} - -#[derive(Debug)] -pub enum ReportRequest { - List { - type_: Option>, - domain: Option, - result_tx: oneshot::Sender>, - }, - Status { - report_ids: Vec, - result_tx: oneshot::Sender>>, - }, - Cancel { - report_ids: Vec, - result_tx: oneshot::Sender>, - }, -} - -#[derive(Debug, Serialize)] +#[derive(Debug, serde::Serialize)] pub struct Response { data: T, } -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, serde::Serialize, serde::Deserialize, PartialEq, Eq)] pub struct Message { pub return_path: String, pub domains: Vec, @@ -113,7 +67,7 @@ pub struct Message { pub env_id: Option, } -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, serde::Serialize, serde::Deserialize, PartialEq, Eq)] pub struct Domain { pub name: String, pub status: Status, @@ -131,7 +85,7 @@ pub struct Domain { pub expires: DateTime, } -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, serde::Serialize, serde::Deserialize, PartialEq, Eq)] pub struct Recipient { pub address: String, pub status: Status, @@ -139,7 +93,7 @@ pub struct Recipient { pub orcpt: Option, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, serde::Serialize, serde::Deserialize)] pub struct Report { pub domain: String, #[serde(rename = "type")] @@ -362,18 +316,48 @@ impl SMTP { match error { None => { - let (result_tx, result_rx) = oneshot::channel(); - self.send_queue_event( - QueueRequest::List { - from, - to, - before, - after, - result_tx, - }, - result_rx, + let mut result = Vec::new(); + let from_key = ValueKey::from(ValueClass::Queue(QueueClass::Message(0))); + let to_key = + ValueKey::from(ValueClass::Queue(QueueClass::Message(u64::MAX))); + let has_filters = + from.is_some() || to.is_some() || before.is_some() || after.is_some(); + let _ = + self.shared + .default_data_store + .iterate( + IterateParams::new(from_key, to_key).ascending(), + |key, value| { + if has_filters { + let message = + Bincode::::deserialize(value)? + .inner; + if from.as_ref().map_or(true, |from| { + message.return_path.contains(from) + }) && to.as_ref().map_or(true, |to| { + message + .recipients + .iter() + .any(|r| r.address_lcase.contains(to)) + }) && before.as_ref().map_or(true, |before| { + message.next_delivery_event() < *before + }) && after.as_ref().map_or(true, |after| { + message.next_delivery_event() > *after + }) { + result.push(key.deserialize_be_u64(1)?); + } + } else { + result.push(key.deserialize_be_u64(1)?); + } + Ok(true) + }, + ) + .await; + + ( + StatusCode::OK, + serde_json::to_string(&Response { data: result }).unwrap_or_default(), ) - .await } Some(error) => error.into_bad_request(), } @@ -404,22 +388,24 @@ impl SMTP { match error { None => { - let (result_tx, result_rx) = oneshot::channel(); - self.send_queue_event( - QueueRequest::Status { - queue_ids, - result_tx, - }, - result_rx, + let mut result = Vec::with_capacity(queue_ids.len()); + for queue_id in queue_ids { + if let Some(message) = self.read_message(queue_id).await { + result.push(Message::from(&message)); + } + } + + ( + StatusCode::OK, + serde_json::to_string(&Response { data: result }).unwrap_or_default(), ) - .await } Some(error) => error.into_bad_request(), } } (&Method::GET, "queue", "retry") => { let mut queue_ids = Vec::new(); - let mut time = Instant::now(); + let mut time = now(); let mut item = None; let mut error = None; @@ -457,17 +443,49 @@ impl SMTP { match error { None => { - let (result_tx, result_rx) = oneshot::channel(); - self.send_queue_event( - QueueRequest::Retry { - queue_ids, - item, - time, - result_tx, - }, - result_rx, + let mut result = Vec::with_capacity(queue_ids.len()); + + for queue_id in queue_ids { + let mut found = false; + + if let Some(mut message) = self.read_message(queue_id).await { + let prev_event = message.next_event().unwrap_or_default(); + + for domain in &mut message.domains { + if matches!( + domain.status, + Status::Scheduled | Status::TemporaryFailure(_) + ) && item + .as_ref() + .map_or(true, |item| domain.domain.contains(item)) + { + domain.retry.due = time; + if domain.expires > time { + domain.expires = time + 10; + } + found = true; + } + } + + if found { + let next_event = message.next_event().unwrap_or_default(); + message + .save_changes(self, prev_event.into(), next_event.into()) + .await; + } + } + + result.push(found); + } + + if result.iter().any(|r| *r) { + let _ = self.queue.tx.send(queue::Event::Reload).await; + } + + ( + StatusCode::OK, + serde_json::to_string(&Response { data: result }).unwrap_or_default(), ) - .await } Some(error) => error.into_bad_request(), } @@ -502,16 +520,93 @@ impl SMTP { match error { None => { - let (result_tx, result_rx) = oneshot::channel(); - self.send_queue_event( - QueueRequest::Cancel { - queue_ids, - item, - result_tx, - }, - result_rx, + let mut result = Vec::with_capacity(queue_ids.len()); + + for queue_id in queue_ids { + let mut found = false; + + if let Some(mut message) = self.read_message(queue_id).await { + let prev_event = message.next_event().unwrap_or_default(); + + if let Some(item) = &item { + // Cancel delivery for all recipients that match + for rcpt in &mut message.recipients { + if rcpt.address_lcase.contains(item) { + rcpt.status = Status::Completed(HostResponse { + hostname: String::new(), + response: smtp_proto::Response { + code: 0, + esc: [0, 0, 0], + message: "Delivery canceled.".to_string(), + }, + }); + found = true; + } + } + if found { + // Mark as completed domains without any pending deliveries + for (domain_idx, domain) in + message.domains.iter_mut().enumerate() + { + if matches!( + domain.status, + Status::TemporaryFailure(_) | Status::Scheduled + ) { + let mut total_rcpt = 0; + let mut total_completed = 0; + + for rcpt in &message.recipients { + if rcpt.domain_idx == domain_idx { + total_rcpt += 1; + if matches!( + rcpt.status, + Status::PermanentFailure(_) + | Status::Completed(_) + ) { + total_completed += 1; + } + } + } + + if total_rcpt == total_completed { + domain.status = Status::Completed(()); + } + } + } + + // Delete message if there are no pending deliveries + if message.domains.iter().any(|domain| { + matches!( + domain.status, + Status::TemporaryFailure(_) | Status::Scheduled + ) + }) { + let next_event = + message.next_event().unwrap_or_default(); + message + .save_changes( + self, + next_event.into(), + prev_event.into(), + ) + .await; + } else { + message.remove(self, prev_event).await; + } + } + } else { + message.remove(self, prev_event).await; + found = true; + } + } + + result.push(found); + } + + ( + StatusCode::OK, + serde_json::to_string(&Response { data: result }).unwrap_or_default(), ) - .await } Some(error) => error.into_bad_request(), } @@ -526,10 +621,10 @@ impl SMTP { match key.as_ref() { "type" => match value.as_ref() { "dmarc" => { - type_ = ReportType::Dmarc(()).into(); + type_ = 0u8.into(); } "tls" => { - type_ = ReportType::Tls(()).into(); + type_ = 1u8.into(); } _ => { error = format!("Invalid report type {value:?}.").into(); @@ -549,16 +644,54 @@ impl SMTP { match error { None => { - let (result_tx, result_rx) = oneshot::channel(); - self.send_report_event( - ReportRequest::List { - type_, - domain, - result_tx, - }, - result_rx, + let mut result = Vec::new(); + let from_key = ValueKey::from(ValueClass::Queue( + QueueClass::DmarcReportHeader(ReportEvent { + due: 0, + policy_hash: 0, + seq_id: 0, + domain: String::new(), + }), + )); + let to_key = ValueKey::from(ValueClass::Queue( + QueueClass::TlsReportHeader(ReportEvent { + due: u64::MAX, + policy_hash: 0, + seq_id: 0, + domain: String::new(), + }), + )); + let _ = + self.shared + .default_data_store + .iterate( + IterateParams::new(from_key, to_key).ascending().no_values(), + |key, _| { + if type_.map_or(true, |t| t == *key.last().unwrap()) { + let event = ReportEvent::deserialize(key)?; + if domain.as_ref().map_or(true, |d| { + d.eq_ignore_ascii_case(&event.domain) + }) { + result.push( + if *key.last().unwrap() == 0 { + QueueClass::DmarcReportHeader(event) + } else { + QueueClass::TlsReportHeader(event) + } + .queue_id(), + ); + } + } + + Ok(true) + }, + ) + .await; + + ( + StatusCode::OK, + serde_json::to_string(&Response { data: result }).unwrap_or_default(), ) - .await } Some(error) => error.into_bad_request(), } @@ -588,17 +721,13 @@ impl SMTP { } match error { - None => { - let (result_tx, result_rx) = oneshot::channel(); - self.send_report_event( - ReportRequest::Status { - report_ids, - result_tx, - }, - result_rx, - ) - .await - } + None => ( + StatusCode::OK, + serde_json::to_string(&Response { + data: report_ids.into_iter().map(Report::from).collect::>(), + }) + .unwrap_or_default(), + ), Some(error) => error.into_bad_request(), } } @@ -628,15 +757,26 @@ impl SMTP { match error { None => { - let (result_tx, result_rx) = oneshot::channel(); - self.send_report_event( - ReportRequest::Cancel { - report_ids, - result_tx, - }, - result_rx, + let mut result = Vec::with_capacity(report_ids.len()); + + for report_id in report_ids { + match report_id { + QueueClass::DmarcReportHeader(event) => { + self.delete_dmarc_report(event).await; + } + QueueClass::TlsReportHeader(event) => { + self.delete_tls_report(vec![event]).await; + } + _ => (), + } + + result.push(true); + } + + ( + StatusCode::OK, + serde_json::to_string(&Response { data: result }).unwrap_or_default(), ) - .await } Some(error) => error.into_bad_request(), } @@ -660,85 +800,11 @@ impl SMTP { ) .unwrap() } - - async fn send_queue_event( - &self, - request: QueueRequest, - rx: oneshot::Receiver, - ) -> (StatusCode, String) { - match self.queue.tx.send(queue::Event::Manage(request)).await { - Ok(_) => match rx.await { - Ok(result) => { - return ( - StatusCode::OK, - serde_json::to_string(&Response { data: result }).unwrap_or_default(), - ) - } - Err(_) => { - tracing::debug!( - context = "queue", - event = "recv-error", - reason = "Failed to receive manage request response." - ); - } - }, - Err(_) => { - tracing::debug!( - context = "queue", - event = "send-error", - reason = "Failed to send manage request event." - ); - } - } - - ( - StatusCode::INTERNAL_SERVER_ERROR, - "{\"error\": \"internal-error\", \"details\": \"Resource unavailable, try again later.\"}" - .to_string(), - ) - } - - async fn send_report_event( - &self, - request: ReportRequest, - rx: oneshot::Receiver, - ) -> (StatusCode, String) { - match self.report.tx.send(reporting::Event::Manage(request)).await { - Ok(_) => match rx.await { - Ok(result) => { - return ( - StatusCode::OK, - serde_json::to_string(&Response { data: result }).unwrap_or_default(), - ) - } - Err(_) => { - tracing::debug!( - context = "queue", - event = "recv-error", - reason = "Failed to receive manage request response." - ); - } - }, - Err(_) => { - tracing::debug!( - context = "queue", - event = "send-error", - reason = "Failed to send manage request event." - ); - } - } - - ( - StatusCode::INTERNAL_SERVER_ERROR, - "{\"error\": \"internal-error\", \"details\": \"Resource unavailable, try again later.\"}" - .to_string(), - ) - } } impl From<&queue::Message> for Message { fn from(message: &queue::Message) -> Self { - let now = Instant::now(); + let now = now(); Message { return_path: message.return_path.clone(), @@ -764,20 +830,12 @@ impl From<&queue::Message> for Message { }, retry_num: domain.retry.inner, next_retry: if domain.retry.due > now { - DateTime::from_timestamp(instant_to_timestamp(now, domain.retry.due) as i64) - .into() + DateTime::from_timestamp(domain.retry.due as i64).into() } else { None }, next_notify: if domain.notify.due > now { - DateTime::from_timestamp( - instant_to_timestamp( - now, - domain.notify.due, - ) - as i64, - ) - .into() + DateTime::from_timestamp(domain.notify.due as i64).into() } else { None }, @@ -802,61 +860,64 @@ impl From<&queue::Message> for Message { orcpt: rcpt.orcpt.clone(), }) .collect(), - expires: DateTime::from_timestamp( - instant_to_timestamp(now, domain.expires) as i64 - ), + expires: DateTime::from_timestamp(domain.expires as i64), }) .collect(), } } } -impl From<(&ReportKey, &ReportValue)> for Report { - fn from((key, value): (&ReportKey, &ReportValue)) -> Self { - match (key, value) { - (ReportType::Dmarc(domain), ReportType::Dmarc(value)) => Report { - domain: domain.inner.clone(), - range_from: DateTime::from_timestamp(value.created as i64), - range_to: DateTime::from_timestamp( - (value.created + value.deliver_at.as_secs()) as i64, - ), - size: value.size, +impl From for Report { + fn from(value: QueueClass) -> Self { + match value { + QueueClass::DmarcReportHeader(event) => Report { + domain: event.domain, type_: "dmarc".to_string(), + range_from: DateTime::from_timestamp(event.due as i64), + range_to: DateTime::from_timestamp(event.due as i64), + size: 0, }, - (ReportType::Tls(domain), ReportType::Tls(value)) => Report { - domain: domain.clone(), - range_from: DateTime::from_timestamp(value.created as i64), - range_to: DateTime::from_timestamp( - (value.created + value.deliver_at.as_secs()) as i64, - ), - size: value.size, + QueueClass::TlsReportHeader(event) => Report { + domain: event.domain, type_: "tls".to_string(), + range_from: DateTime::from_timestamp(event.due as i64), + range_to: DateTime::from_timestamp(event.due as i64), + size: 0, }, _ => unreachable!(), } } } -impl Display for ReportKey { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +trait GenerateQueueId { + fn queue_id(&self) -> String; +} + +impl GenerateQueueId for QueueClass { + fn queue_id(&self) -> String { match self { - ReportType::Dmarc(policy) => write!(f, "d!{}!{}", policy.inner, policy.policy), - ReportType::Tls(domain) => write!(f, "t!{domain}"), + QueueClass::DmarcReportHeader(h) => { + format!("d!{}!{}!{}!{}", h.domain, h.policy_hash, h.seq_id, h.due) + } + QueueClass::TlsReportHeader(h) => { + format!("t!{}!{}!{}!{}", h.domain, h.policy_hash, h.seq_id, h.due) + } + _ => unreachable!(), } } } trait ParseValues { - fn parse_timestamp(&self) -> Result; + fn parse_timestamp(&self) -> Result; fn parse_queue_ids(&self) -> Result, String>; - fn parse_report_ids(&self) -> Result, String>; + fn parse_report_ids(&self) -> Result, String>; } impl ParseValues for Cow<'_, str> { - fn parse_timestamp(&self) -> Result { + fn parse_timestamp(&self) -> Result { if let Some(dt) = DateTime::parse_rfc3339(self.as_ref()) { - let instant = (dt.to_timestamp() as u64).to_instant(); - if instant >= Instant::now() { + let instant = dt.to_timestamp() as u64; + if instant >= now() { return Ok(instant); } } @@ -881,29 +942,42 @@ impl ParseValues for Cow<'_, str> { Ok(ids) } - fn parse_report_ids(&self) -> Result, String> { + fn parse_report_ids(&self) -> Result, String> { let mut ids = Vec::new(); for id in self.split(',') { if !id.is_empty() { let mut parts = id.split('!'); - match (parts.next(), parts.next()) { - (Some("d"), Some(domain)) if !domain.is_empty() => { - if let Some(policy) = parts.next().and_then(|policy| policy.parse().ok()) { - ids.push(ReportType::Dmarc(ReportPolicy { - inner: domain.to_string(), - policy, - })); - continue; - } + match ( + parts.next(), + parts.next(), + parts.next().and_then(|p| p.parse::().ok()), + parts.next().and_then(|p| p.parse::().ok()), + parts.next().and_then(|p| p.parse::().ok()), + ) { + (Some("d"), Some(domain), Some(policy), Some(seq_id), Some(due)) + if !domain.is_empty() => + { + ids.push(QueueClass::DmarcReportHeader(ReportEvent { + due, + policy_hash: policy, + seq_id, + domain: domain.to_string(), + })); } - (Some("t"), Some(domain)) if !domain.is_empty() => { - ids.push(ReportType::Tls(domain.to_string())); - continue; + (Some("t"), Some(domain), Some(policy), Some(seq_id), Some(due)) + if !domain.is_empty() => + { + ids.push(QueueClass::TlsReportHeader(ReportEvent { + due, + policy_hash: policy, + seq_id, + domain: domain.to_string(), + })); + } + _ => { + return Err(format!("Failed to parse id {id:?}.")); } - _ => (), } - - return Err(format!("Failed to parse id {id:?}.")); } } Ok(ids) @@ -944,7 +1018,7 @@ fn deserialize_maybe_datetime<'de, D>(deserializer: D) -> Result, { - if let Some(value) = Option::<&str>::deserialize(deserializer)? { + if let Some(value) = as serde::Deserialize>::deserialize(deserializer)? { if let Some(value) = DateTime::parse_rfc3339(value) { Ok(Some(value)) } else { @@ -968,6 +1042,8 @@ fn deserialize_datetime<'de, D>(deserializer: D) -> Result where D: Deserializer<'de>, { + use serde::Deserialize; + if let Some(value) = DateTime::parse_rfc3339(<&str>::deserialize(deserializer)?) { Ok(value) } else { diff --git a/crates/smtp/src/core/mod.rs b/crates/smtp/src/core/mod.rs index 28be3737..7808333c 100644 --- a/crates/smtp/src/core/mod.rs +++ b/crates/smtp/src/core/mod.rs @@ -24,7 +24,7 @@ use std::{ hash::Hash, net::IpAddr, - sync::{atomic::AtomicU32, Arc}, + sync::Arc, time::{Duration, Instant}, }; @@ -40,7 +40,7 @@ use smtp_proto::{ }, IntoString, }; -use store::{LookupStore, Store, Value}; +use store::{BlobStore, LookupStore, Store, Value}; use tokio::{ io::{AsyncRead, AsyncWrite}, sync::mpsc, @@ -50,7 +50,12 @@ use tracing::Span; use utils::{ expr, ipc::DeliveryEvent, - listener::{limiter::InFlight, stream::NullIo, ServerInstance, TcpAcceptor}, + listener::{ + limiter::{ConcurrencyLimiter, InFlight}, + stream::NullIo, + ServerInstance, TcpAcceptor, + }, + snowflake::SnowflakeIdGenerator, }; use crate::{ @@ -63,11 +68,11 @@ use crate::{ dane::{DnssecResolver, Tlsa}, mta_sts, }, - queue::{self, DomainPart, QueueId, QuotaLimiter}, + queue::{self, DomainPart, QueueId}, reporting, }; -use self::throttle::{Limiter, ThrottleKey, ThrottleKeyHasherBuilder}; +use self::throttle::{ThrottleKey, ThrottleKeyHasherBuilder}; pub mod eval; pub mod management; @@ -121,6 +126,7 @@ pub struct Shared { // Default store and directory pub default_directory: Arc, pub default_data_store: Store, + pub default_blob_store: BlobStore, pub default_lookup_store: LookupStore, } @@ -145,15 +151,14 @@ pub struct DnsCache { pub struct SessionCore { pub config: SessionConfig, - pub throttle: DashMap, + pub throttle: DashMap, } pub struct QueueCore { pub config: QueueConfig, - pub throttle: DashMap, - pub quota: DashMap, ThrottleKeyHasherBuilder>, + pub throttle: DashMap, pub tx: mpsc::Sender, - pub id_seq: AtomicU32, + pub snowflake_id: SnowflakeIdGenerator, pub connectors: TlsConnectors, } diff --git a/crates/smtp/src/core/throttle.rs b/crates/smtp/src/core/throttle.rs index 6cb3aa03..f2a336dd 100644 --- a/crates/smtp/src/core/throttle.rs +++ b/crates/smtp/src/core/throttle.rs @@ -21,7 +21,7 @@ * for more details. */ -use ::utils::listener::limiter::{ConcurrencyLimiter, RateLimiter}; +use ::utils::listener::limiter::ConcurrencyLimiter; use dashmap::mapref::entry::Entry; use tokio::io::{AsyncRead, AsyncWrite}; use utils::config::Rate; @@ -32,12 +32,6 @@ use crate::config::*; use super::{eval::*, ResolveVariable, Session}; -#[derive(Debug)] -pub struct Limiter { - pub rate: Option, - pub concurrency: Option, -} - #[derive(Debug, Clone, Eq)] pub struct ThrottleKey { hash: [u8; 32], @@ -55,6 +49,12 @@ impl Hash for ThrottleKey { } } +impl AsRef<[u8]> for ThrottleKey { + fn as_ref(&self) -> &[u8] { + &self.hash + } +} + #[derive(Default)] pub struct ThrottleKeyHasher { hash: u64, @@ -236,10 +236,36 @@ impl Session { } // Build throttle key - match self.core.session.throttle.entry(t.new_key(self)) { - Entry::Occupied(mut e) => { - let limiter = e.get_mut(); - if let Some(limiter) = &limiter.concurrency { + let key = t.new_key(self); + + // Check rate + if let Some(rate) = &t.rate { + if self + .core + .shared + .default_lookup_store + .is_rate_allowed(key.hash.as_slice(), rate, false) + .await + .unwrap_or_default() + .is_some() + { + tracing::debug!( + parent: &self.span, + context = "throttle", + event = "rate-limit-exceeded", + max_requests = rate.requests, + max_interval = rate.period.as_secs(), + "Rate limit exceeded." + ); + return false; + } + } + + // Check concurrency + if let Some(concurrency) = &t.concurrency { + match self.core.session.throttle.entry(key) { + Entry::Occupied(mut e) => { + let limiter = e.get_mut(); if let Some(inflight) = limiter.is_allowed() { self.in_flight.push(inflight); } else { @@ -253,35 +279,13 @@ impl Session { return false; } } - if let (Some(limiter), Some(rate)) = (&mut limiter.rate, &t.rate) { - if !limiter.is_allowed(rate) { - tracing::debug!( - parent: &self.span, - context = "throttle", - event = "rate-limit-exceeded", - max_requests = rate.requests, - max_interval = rate.period.as_secs(), - "Rate limit exceeded." - ); - return false; - } - } - } - Entry::Vacant(e) => { - let concurrency = t.concurrency.map(|concurrency| { - let limiter = ConcurrencyLimiter::new(concurrency); + Entry::Vacant(e) => { + let limiter = ConcurrencyLimiter::new(*concurrency); if let Some(inflight) = limiter.is_allowed() { self.in_flight.push(inflight); } - limiter - }); - let rate = t.rate.as_ref().map(|rate| { - let r = RateLimiter::new(rate); - r.is_allowed(rate); - r - }); - - e.insert(Limiter { rate, concurrency }); + e.insert(limiter); + } } } } @@ -290,33 +294,19 @@ impl Session { true } - pub fn throttle_rcpt(&self, rcpt: &str, rate: &Rate, ctx: &str) -> bool { + pub async fn throttle_rcpt(&self, rcpt: &str, rate: &Rate, ctx: &str) -> bool { let mut hasher = blake3::Hasher::new(); hasher.update(rcpt.as_bytes()); hasher.update(ctx.as_bytes()); hasher.update(&rate.period.as_secs().to_ne_bytes()[..]); hasher.update(&rate.requests.to_ne_bytes()[..]); - let key = ThrottleKey { - hash: hasher.finalize().into(), - }; - match self.core.session.throttle.entry(key) { - Entry::Occupied(mut e) => { - if let Some(limiter) = &mut e.get_mut().rate { - limiter.is_allowed(rate) - } else { - false - } - } - Entry::Vacant(e) => { - let limiter = RateLimiter::new(rate); - limiter.is_allowed(rate); - e.insert(Limiter { - rate: limiter.into(), - concurrency: None, - }); - true - } - } + self.core + .shared + .default_lookup_store + .is_rate_allowed(hasher.finalize().as_bytes(), rate, false) + .await + .unwrap_or_default() + .is_none() } } diff --git a/crates/smtp/src/core/worker.rs b/crates/smtp/src/core/worker.rs index 4dafc1e5..56a030f1 100644 --- a/crates/smtp/src/core/worker.rs +++ b/crates/smtp/src/core/worker.rs @@ -54,16 +54,8 @@ impl SMTP { fn cleanup(&self) { for throttle in [&self.session.throttle, &self.queue.throttle] { - throttle.retain(|_, v| { - v.concurrency - .as_ref() - .map_or(false, |c| c.concurrent.load(Ordering::Relaxed) > 0) - || v.rate.as_ref().map_or(false, |r| r.is_active()) - }); + throttle.retain(|_, v| v.concurrent.load(Ordering::Relaxed) > 0); } - self.queue.quota.retain(|_, v| { - v.messages.load(Ordering::Relaxed) > 0 || v.size.load(Ordering::Relaxed) > 0 - }); } } diff --git a/crates/smtp/src/inbound/data.rs b/crates/smtp/src/inbound/data.rs index 72e8f8ca..7a363bca 100644 --- a/crates/smtp/src/inbound/data.rs +++ b/crates/smtp/src/inbound/data.rs @@ -23,10 +23,9 @@ use std::{ borrow::Cow, - path::PathBuf, process::Stdio, sync::Arc, - time::{Duration, Instant, SystemTime}, + time::{Duration, SystemTime}, }; use mail_auth::{ @@ -38,6 +37,7 @@ use sieve::runtime::Variable; use smtp_proto::{ MAIL_BY_RETURN, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, }; +use store::write::now; use tokio::{io::AsyncWriteExt, process::Command}; use utils::{config::Rate, listener::SessionStream}; @@ -654,10 +654,8 @@ impl Session { // Verify queue quota if self.core.has_quota(&mut message).await { let queue_id = message.id; - if self - .core - .queue - .queue_message(message, Some(&headers), &raw_message, &self.span) + if message + .queue(Some(&headers), &raw_message, &self.core, &self.span) .await { self.state = State::Accepted(queue_id); @@ -682,14 +680,14 @@ impl Session { &self, mail_from: SessionAddress, mut rcpt_to: Vec, - ) -> Box { + ) -> Message { // Build message - let mut message = Box::new(Message { - id: self.core.queue.queue_id(), - path: PathBuf::new(), - created: SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .map_or(0, |d| d.as_secs()), + let created = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()); + let mut message = Message { + id: self.core.queue.snowflake_id.generate().unwrap_or(created), + created, return_path: mail_from.address, return_path_lcase: mail_from.address_lcase, return_path_domain: mail_from.domain, @@ -699,8 +697,9 @@ impl Session { priority: self.data.priority, size: 0, env_id: mail_from.dsn_info, - queue_refs: Vec::with_capacity(0), - }); + blob_hash: Default::default(), + quota_keys: Vec::new(), + }; // Add recipients let future_release = Duration::from_secs(self.data.future_release); @@ -711,7 +710,7 @@ impl Session { .last() .map_or(true, |d| d.domain != rcpt.domain) { - let envelope = SimpleEnvelope::new(message.as_ref(), &rcpt.domain); + let envelope = SimpleEnvelope::new(&message, &rcpt.domain); // Set next retry time let retry = if self.data.future_release == 0 { @@ -731,18 +730,19 @@ impl Session { let (notify, expires) = if self.data.delivery_by == 0 { ( queue::Schedule::later(future_release + next_notify), - Instant::now() - + future_release + now() + + future_release.as_secs() + self .core .eval_if(&config.expire, &envelope) .await - .unwrap_or_else(|| Duration::from_secs(5 * 86400)), + .unwrap_or_else(|| Duration::from_secs(5 * 86400)) + .as_secs(), ) } else if (message.flags & MAIL_BY_RETURN) != 0 { ( queue::Schedule::later(future_release + next_notify), - Instant::now() + Duration::from_secs(self.data.delivery_by as u64), + now() + self.data.delivery_by as u64, ) } else { let expire = self @@ -769,7 +769,7 @@ impl Session { let mut notify = queue::Schedule::later(future_release + notify); notify.inner = (num_intervals - 1) as u32; // Disable further notification attempts - (notify, Instant::now() + expire) + (notify, now() + expire_secs) }; message.domains.push(queue::Domain { @@ -779,7 +779,6 @@ impl Session { status: queue::Status::Scheduled, domain: rcpt.domain, disable_tls: false, - changed: false, }); } diff --git a/crates/smtp/src/lib.rs b/crates/smtp/src/lib.rs index 922d8347..bdb63e51 100644 --- a/crates/smtp/src/lib.rs +++ b/crates/smtp/src/lib.rs @@ -42,6 +42,7 @@ use store::Stores; use tokio::sync::mpsc; use utils::{ config::{Config, ServerProtocol, Servers}, + snowflake::SnowflakeIdGenerator, UnwrapFailure, }; @@ -129,15 +130,10 @@ impl SMTP { .unwrap_or(32) .next_power_of_two() as usize, ), - id_seq: 0.into(), - quota: DashMap::with_capacity_and_hasher_and_shard_amount( - config.property("global.shared-map.capacity")?.unwrap_or(2), - ThrottleKeyHasherBuilder::default(), - config - .property::("global.shared-map.shard")? - .unwrap_or(32) - .next_power_of_two() as usize, - ), + snowflake_id: config + .property::("storage.cluster.node-id")? + .map(SnowflakeIdGenerator::with_node_id) + .unwrap_or_else(SnowflakeIdGenerator::new), tx: queue_tx, connectors: TlsConnectors { pki_verify: build_tls_connector(false), @@ -156,10 +152,10 @@ impl SMTP { }); // Spawn queue manager - queue_rx.spawn(core.clone(), core.queue.read_queue().await); + queue_rx.spawn(core.clone()); // Spawn report manager - report_rx.spawn(core.clone(), core.report.read_reports().await); + report_rx.spawn(core.clone()); Ok(core) } diff --git a/crates/smtp/src/outbound/delivery.rs b/crates/smtp/src/outbound/delivery.rs index 14da36b2..c7496a0f 100644 --- a/crates/smtp/src/outbound/delivery.rs +++ b/crates/smtp/src/outbound/delivery.rs @@ -24,7 +24,7 @@ use std::{ net::{IpAddr, Ipv4Addr, SocketAddr}, sync::Arc, - time::{Duration, Instant}, + time::Duration, }; use mail_auth::{ @@ -33,6 +33,7 @@ use mail_auth::{ }; use mail_send::SmtpClient; use smtp_proto::MAIL_REQUIRETLS; +use store::write::now; use utils::config::ServerProtocol; use crate::{ @@ -49,71 +50,80 @@ use super::{ NextHop, }; use crate::queue::{ - manager::Queue, throttle, DeliveryAttempt, Domain, Error, Event, OnHold, QueueEnvelope, - Schedule, Status, WorkerResult, + throttle, DeliveryAttempt, Domain, Error, Event, OnHold, QueueEnvelope, Status, }; impl DeliveryAttempt { - pub async fn try_deliver(mut self, core: Arc, queue: &mut Queue) { - // Check that the message still has recipients to be delivered - let has_pending_delivery = self.has_pending_delivery(); - - // Send any due Delivery Status Notifications - core.send_dsn(&mut self).await; - - if has_pending_delivery { - // Re-queue the message if its not yet due for delivery - let due = self.message.next_delivery_event(); - if due > Instant::now() { - // Save changes to disk - self.message.save_changes().await; - - queue.schedule(Schedule { - due, - inner: self.message, - }); - return; - } - } else { - // All message recipients expired, do not re-queue. (DSN has been already sent) - self.message.remove().await; - return; - } - - // Throttle sender - for throttle in &core.queue.config.throttle.sender { - if let Err(err) = core - .is_allowed( - throttle, - self.message.as_ref(), - &mut self.in_flight, - &self.span, - ) - .await - { - // Save changes to disk - self.message.save_changes().await; - - match err { - throttle::Error::Concurrency { limiter } => { - queue.on_hold(OnHold { - next_due: self.message.next_event_after(Instant::now()), - limiters: vec![limiter], - message: self.message, - }); - } - throttle::Error::Rate { retry_at } => { - queue.schedule(Schedule { - due: retry_at, - inner: self.message, - }); - } - } - return; - } - } - + pub async fn try_deliver(mut self, core: Arc) { tokio::spawn(async move { + // Check that the message still has recipients to be delivered + let has_pending_delivery = self.has_pending_delivery(); + + // Send any due Delivery Status Notifications + core.send_dsn(&mut self).await; + + if has_pending_delivery { + // Re-queue the message if its not yet due for delivery + let due = self.message.next_delivery_event(); + if due > now() { + // Save changes + self.message + .save_changes(&core, self.event.due.into(), due.into()) + .await; + if core.queue.tx.send(Event::Reload).await.is_err() { + tracing::warn!("Channel closed while trying to notify queue manager."); + } + return; + } + } else { + // All message recipients expired, do not re-queue. (DSN has been already sent) + self.message.remove(&core, self.event.due).await; + if core.queue.tx.send(Event::Reload).await.is_err() { + tracing::warn!("Channel closed while trying to notify queue manager."); + } + + return; + } + + // Throttle sender + for throttle in &core.queue.config.throttle.sender { + if let Err(err) = core + .is_allowed(throttle, &self.message, &mut self.in_flight, &self.span) + .await + { + let event = match err { + throttle::Error::Concurrency { limiter } => { + // Save changes to disk + let next_due = self.message.next_event_after(now()); + self.message.save_changes(&core, None, None).await; + + Event::OnHold(OnHold { + next_due, + limiters: vec![limiter], + message: self.event, + }) + } + throttle::Error::Rate { retry_at } => { + // Save changes to disk + let next_event = std::cmp::min( + retry_at, + self.message.next_event_after(now()).unwrap_or(u64::MAX), + ); + self.message + .save_changes(&core, self.event.due.into(), next_event.into()) + .await; + + Event::Reload + } + }; + + if core.queue.tx.send(event).await.is_err() { + tracing::warn!("Channel closed while trying to notify queue manager."); + } + return; + } + } + let queue_config = &core.queue.config; let mut on_hold = Vec::new(); let no_ip = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)); @@ -123,7 +133,7 @@ impl DeliveryAttempt { 'next_domain: for (domain_idx, domain) in domains.iter_mut().enumerate() { // Only process domains due for delivery if !matches!(&domain.status, Status::Scheduled | Status::TemporaryFailure(_) - if domain.retry.due <= Instant::now()) + if domain.retry.due <= now()) { continue; } @@ -138,7 +148,7 @@ impl DeliveryAttempt { // Build envelope let mut envelope = QueueEnvelope { - message: self.message.as_ref(), + message: &self.message, domain: &domain.domain, mx: "", remote_ip: no_ip, @@ -672,6 +682,7 @@ impl DeliveryAttempt { .unwrap_or_else(|| "localhost".to_string()); let params = SessionParams { span: &span, + core: &core, credentials: remote_host.credentials(), is_smtp: remote_host.is_smtp(), hostname: envelope.mx, @@ -1018,11 +1029,9 @@ impl DeliveryAttempt { // Notify queue manager let span = self.span; let result = if !on_hold.is_empty() { - // Release quota for completed deliveries - self.message.release_quota(); - // Save changes to disk - self.message.save_changes().await; + let next_due = self.message.next_event_after(now()); + self.message.save_changes(&core, None, None).await; tracing::info!( parent: &span, @@ -1032,17 +1041,16 @@ impl DeliveryAttempt { "Too many outbound concurrent connections, message moved to on-hold queue." ); - WorkerResult::OnHold(OnHold { - next_due: self.message.next_event_after(Instant::now()), + Event::OnHold(OnHold { + next_due, limiters: on_hold, - message: self.message, + message: self.event, }) } else if let Some(due) = self.message.next_event() { - // Release quota for completed deliveries - self.message.release_quota(); - // Save changes to disk - self.message.save_changes().await; + self.message + .save_changes(&core, self.event.due.into(), due.into()) + .await; tracing::info!( parent: &span, @@ -1052,13 +1060,10 @@ impl DeliveryAttempt { "Delivery was not possible, message re-queued for delivery." ); - WorkerResult::Retry(Schedule { - due, - inner: self.message, - }) + Event::Reload } else { // Delete message from queue - self.message.remove().await; + self.message.remove(&core, self.event.due).await; tracing::info!( parent: &span, @@ -1067,9 +1072,9 @@ impl DeliveryAttempt { "Delivery completed." ); - WorkerResult::Done + Event::Reload }; - if core.queue.tx.send(Event::Done(result)).await.is_err() { + if core.queue.tx.send(result).await.is_err() { tracing::warn!( parent: &span, "Channel closed while trying to notify queue manager." @@ -1080,7 +1085,7 @@ impl DeliveryAttempt { /// Marks as failed all domains that reached their expiration time pub fn has_pending_delivery(&mut self) -> bool { - let now = Instant::now(); + let now = now(); let mut has_pending_delivery = false; let span = self.span.clone(); @@ -1103,7 +1108,6 @@ impl DeliveryAttempt { domain.status = std::mem::replace(&mut domain.status, Status::Scheduled).into_permanent(); - domain.changed = true; } Status::Scheduled if domain.expires <= now => { tracing::info!( @@ -1123,7 +1127,6 @@ impl DeliveryAttempt { domain.status = Status::PermanentFailure(Error::Io( "Queue rate limit exceeded.".to_string(), )); - domain.changed = true; } Status::Completed(_) | Status::PermanentFailure(_) => (), _ => { @@ -1139,7 +1142,6 @@ impl DeliveryAttempt { impl Domain { pub fn set_status(&mut self, status: impl Into>, schedule: &[Duration]) { self.status = status.into(); - self.changed = true; if matches!( &self.status, Status::TemporaryFailure(_) | Status::Scheduled @@ -1149,8 +1151,8 @@ impl Domain { } pub fn retry(&mut self, schedule: &[Duration]) { - self.retry.due = - Instant::now() + schedule[std::cmp::min(self.retry.inner as usize, schedule.len() - 1)]; + self.retry.due = now() + + schedule[std::cmp::min(self.retry.inner as usize, schedule.len() - 1)].as_secs(); self.retry.inner += 1; } } diff --git a/crates/smtp/src/outbound/local.rs b/crates/smtp/src/outbound/local.rs index 17bf09c2..a8962deb 100644 --- a/crates/smtp/src/outbound/local.rs +++ b/crates/smtp/src/outbound/local.rs @@ -63,7 +63,7 @@ impl Message { message: IngestMessage { sender_address: self.return_path_lcase.clone(), recipients: recipient_addresses, - message_path: self.path.clone(), + message_blob: self.blob_hash.clone(), message_size: self.size, }, result_tx, diff --git a/crates/smtp/src/outbound/mod.rs b/crates/smtp/src/outbound/mod.rs index d3164015..b5f3398c 100644 --- a/crates/smtp/src/outbound/mod.rs +++ b/crates/smtp/src/outbound/mod.rs @@ -25,6 +25,7 @@ use std::borrow::Cow; use mail_send::Credentials; use smtp_proto::{Response, Severity}; +use store::write::QueueEvent; use utils::config::ServerProtocol; use crate::{ @@ -211,8 +212,8 @@ impl From for Status<(), Error> { } } -impl From> for DeliveryAttempt { - fn from(message: Box) -> Self { +impl DeliveryAttempt { + pub fn new(message: Message, event: QueueEvent) -> Self { DeliveryAttempt { span: tracing::info_span!( "delivery", @@ -227,6 +228,7 @@ impl From> for DeliveryAttempt { ), in_flight: Vec::new(), message, + event, } } } diff --git a/crates/smtp/src/outbound/session.rs b/crates/smtp/src/outbound/session.rs index eb8c4420..74c55418 100644 --- a/crates/smtp/src/outbound/session.rs +++ b/crates/smtp/src/outbound/session.rs @@ -30,7 +30,6 @@ use smtp_proto::{ use std::fmt::Write; use std::time::Duration; use tokio::{ - fs, io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}, net::TcpStream, }; @@ -38,6 +37,7 @@ use tokio_rustls::{client::TlsStream, TlsConnector}; use crate::{ config::{RequireOptional, TlsStrategy}, + core::SMTP, queue::{ErrorDetails, HostResponse, RCPT_STATUS_CHANGED}, }; @@ -45,6 +45,7 @@ use crate::queue::{Error, Message, Recipient, Status}; pub struct SessionParams<'x> { pub span: &'x tracing::Span, + pub core: &'x SMTP, pub hostname: &'x str, pub credentials: Option<&'x Credentials>, pub is_smtp: bool, @@ -532,43 +533,53 @@ pub async fn send_message( bdat_cmd: &Option, params: &SessionParams<'_>, ) -> Result<(), Status<(), Error>> { - let mut raw_message = vec![0u8; message.size]; - let mut file = fs::File::open(&message.path).await.map_err(|err| { - tracing::error!(parent: params.span, - context = "queue", - event = "error", - "Failed to open message file {}: {}", - message.path.display(), - err); - Status::TemporaryFailure(Error::Io("Queue system error.".to_string())) - })?; - file.read_exact(&mut raw_message).await.map_err(|err| { - tracing::error!(parent: params.span, - context = "queue", - event = "error", - "Failed to read {} bytes file {} from disk: {}", - message.size, - message.path.display(), - err); - Status::TemporaryFailure(Error::Io("Queue system error.".to_string())) - })?; - tokio::time::timeout(params.timeout_data, async { - if let Some(bdat_cmd) = bdat_cmd { - write_chunks(smtp_client, &[bdat_cmd.as_bytes(), &raw_message]).await - } else { - write_chunks(smtp_client, &[b"DATA\r\n"]).await?; - smtp_client.read().await?.assert_code(354)?; - smtp_client - .write_message(&raw_message) - .await - .map_err(mail_send::Error::from) + match params + .core + .shared + .default_blob_store + .get_blob(message.blob_hash.as_slice(), 0..u32::MAX) + .await + { + Ok(Some(raw_message)) => tokio::time::timeout(params.timeout_data, async { + if let Some(bdat_cmd) = bdat_cmd { + write_chunks(smtp_client, &[bdat_cmd.as_bytes(), &raw_message]).await + } else { + write_chunks(smtp_client, &[b"DATA\r\n"]).await?; + smtp_client.read().await?.assert_code(354)?; + smtp_client + .write_message(&raw_message) + .await + .map_err(mail_send::Error::from) + } + }) + .await + .map_err(|_| Status::timeout(params.hostname, "sending message"))? + .map_err(|err| { + Status::from_smtp_error(params.hostname, bdat_cmd.as_deref().unwrap_or("DATA"), err) + }), + Ok(None) => { + tracing::error!(parent: params.span, + context = "queue", + event = "error", + "BlobHash {:?} does not exist.", + message.blob_hash, + ); + Err(Status::TemporaryFailure(Error::Io( + "Queue system error.".to_string(), + ))) } - }) - .await - .map_err(|_| Status::timeout(params.hostname, "sending message"))? - .map_err(|err| { - Status::from_smtp_error(params.hostname, bdat_cmd.as_deref().unwrap_or("DATA"), err) - }) + Err(err) => { + tracing::error!(parent: params.span, + context = "queue", + event = "error", + "Failed to fetch blobId {:?}: {}", + message.blob_hash, + err); + Err(Status::TemporaryFailure(Error::Io( + "Queue system error.".to_string(), + ))) + } + } } pub async fn say_helo( diff --git a/crates/smtp/src/queue/dsn.rs b/crates/smtp/src/queue/dsn.rs index 4027edb3..aef875ac 100644 --- a/crates/smtp/src/queue/dsn.rs +++ b/crates/smtp/src/queue/dsn.rs @@ -30,22 +30,21 @@ use smtp_proto::{ Response, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, }; use std::fmt::Write; -use std::time::{Duration, Instant}; -use tokio::fs::File; -use tokio::io::AsyncReadExt; +use std::time::Duration; +use store::write::now; use crate::core::SMTP; use super::{ - instant_to_timestamp, DeliveryAttempt, Domain, Error, ErrorDetails, HostResponse, Message, - Recipient, SimpleEnvelope, Status, RCPT_DSN_SENT, RCPT_STATUS_CHANGED, + DeliveryAttempt, Domain, Error, ErrorDetails, HostResponse, Message, Recipient, SimpleEnvelope, + Status, RCPT_DSN_SENT, RCPT_STATUS_CHANGED, }; impl SMTP { pub async fn send_dsn(&self, attempt: &mut DeliveryAttempt) { if !attempt.message.return_path.is_empty() { if let Some(dsn) = attempt.build_dsn(self).await { - let mut dsn_message = Message::new_boxed("", "", ""); + let mut dsn_message = self.queue.new_message("", "", ""); dsn_message .add_recipient_parts( &attempt.message.return_path, @@ -64,8 +63,8 @@ impl SMTP { &attempt.span, ) .await; - self.queue - .queue_message(dsn_message, signature.as_deref(), &dsn, &attempt.span) + dsn_message + .queue(signature.as_deref(), &dsn, self, &attempt.span) .await; } } else { @@ -77,7 +76,7 @@ impl SMTP { impl DeliveryAttempt { pub async fn build_dsn(&mut self, core: &SMTP) -> Option> { let config = &core.queue.config; - let now = Instant::now(); + let now = now(); let mut txt_success = String::new(); let mut txt_delay = String::new(); @@ -245,11 +244,10 @@ impl DeliveryAttempt { }) { domain.notify.inner += 1; - domain.notify.due = Instant::now() + next_notify; + domain.notify.due = now + next_notify.as_secs(); } else { - domain.notify.due = domain.expires + Duration::from_secs(10); + domain.notify.due = domain.expires + 10; } - domain.changed = true; } } self.message.domains = domains; @@ -257,15 +255,15 @@ impl DeliveryAttempt { // Obtain hostname and sender addresses let from_name = core - .eval_if(&config.dsn.name, self.message.as_ref()) + .eval_if(&config.dsn.name, &self.message) .await .unwrap_or_else(|| String::from("Mail Delivery Subsystem")); let from_addr = core - .eval_if(&config.dsn.address, self.message.as_ref()) + .eval_if(&config.dsn.address, &self.message) .await .unwrap_or_else(|| String::from("MAILER-DAEMON@localhost")); let reporting_mta = core - .eval_if(&config.hostname, self.message.as_ref()) + .eval_if(&config.hostname, &self.message) .await .unwrap_or_else(|| String::from("localhost")); @@ -276,55 +274,54 @@ impl DeliveryAttempt { let dsn = dsn_header + &dsn; // Fetch up to 1024 bytes of message headers - let headers = match File::open(&self.message.path).await { - Ok(mut file) => { - let mut buf = vec![0u8; std::cmp::min(self.message.size, 1024)]; - match file.read(&mut buf).await { - Ok(br) => { - let mut prev_ch = 0; - let mut last_lf = br; - for (pos, &ch) in buf.iter().enumerate() { - match ch { - b'\n' => { - last_lf = pos + 1; - if prev_ch != b'\n' { - prev_ch = ch; - } else { - break; - } - } - b'\r' => (), - 0 => break, - _ => { - prev_ch = ch; - } + let headers = match core + .shared + .default_blob_store + .get_blob(self.message.blob_hash.as_slice(), 0..1024) + .await + { + Ok(Some(mut buf)) => { + let mut prev_ch = 0; + let mut last_lf = buf.len(); + for (pos, &ch) in buf.iter().enumerate() { + match ch { + b'\n' => { + last_lf = pos + 1; + if prev_ch != b'\n' { + prev_ch = ch; + } else { + break; } } - if last_lf < 1024 { - buf.truncate(last_lf); + b'\r' => (), + 0 => break, + _ => { + prev_ch = ch; } - String::from_utf8(buf).unwrap_or_default() - } - Err(err) => { - tracing::error!( - parent: &self.span, - context = "queue", - event = "error", - "Failed to read from {}: {}", - self.message.path.display(), - err - ); - String::new() } } + if last_lf < 1024 { + buf.truncate(last_lf); + } + String::from_utf8(buf).unwrap_or_default() + } + Ok(None) => { + tracing::error!( + parent: &self.span, + context = "queue", + event = "error", + "Failed to open blob {:?}: not found", + self.message.blob_hash + ); + String::new() } Err(err) => { tracing::error!( parent: &self.span, context = "queue", event = "error", - "Failed to open file {}: {}", - self.message.path.display(), + "Failed to open blob {:?}: {}", + self.message.blob_hash, err ); String::new() @@ -387,10 +384,10 @@ impl DeliveryAttempt { } } - let now = Instant::now(); + let now = now(); for domain in &mut message.domains { if domain.notify.due <= now { - domain.notify.due = domain.expires + Duration::from_secs(10); + domain.notify.due = domain.expires + 10; } } @@ -520,13 +517,10 @@ impl Recipient { impl Domain { fn write_dsn_will_retry_until(&self, dsn: &mut String) { - let now = Instant::now(); + let now = now(); if self.expires > now { dsn.push_str("Will-Retry-Until: "); - dsn.push_str( - &DateTime::from_timestamp(instant_to_timestamp(now, self.expires) as i64) - .to_rfc822(), - ); + dsn.push_str(&DateTime::from_timestamp(self.expires as i64).to_rfc822()); dsn.push_str("\r\n"); } } diff --git a/crates/smtp/src/queue/manager.rs b/crates/smtp/src/queue/manager.rs index d44eaa35..4dfdc9a5 100644 --- a/crates/smtp/src/queue/manager.rs +++ b/crates/smtp/src/queue/manager.rs @@ -22,274 +22,84 @@ */ use std::{ - collections::BinaryHeap, sync::{atomic::Ordering, Arc}, - time::{Duration, Instant}, + time::Duration, }; -use ahash::AHashMap; -use smtp_proto::Response; +use store::write::{now, BatchBuilder, QueueClass, QueueEvent, ValueClass}; use tokio::sync::mpsc; -use crate::core::{ - management::{self}, - QueueCore, SMTP, -}; +use crate::core::SMTP; -use super::{ - DeliveryAttempt, Event, HostResponse, Message, OnHold, QueueId, Schedule, Status, WorkerResult, - RCPT_STATUS_CHANGED, -}; +use super::{DeliveryAttempt, Event, Message, OnHold, Status}; + +pub(crate) const SHORT_WAIT: Duration = Duration::from_millis(1); +pub(crate) const LONG_WAIT: Duration = Duration::from_secs(86400 * 365); #[derive(Debug)] pub struct Queue { - short_wait: Duration, - long_wait: Duration, - pub scheduled: BinaryHeap>, - pub on_hold: Vec>, - pub messages: AHashMap>, + pub on_hold: Vec>, } impl SpawnQueue for mpsc::Receiver { - fn spawn(mut self, core: Arc, mut queue: Queue) { + fn spawn(mut self, core: Arc) { tokio::spawn(async move { - loop { - let result = tokio::time::timeout(queue.wake_up_time(), self.recv()).await; + let mut queue = Queue::default(); + let mut next_wake_up = SHORT_WAIT; - // Deliver scheduled messages - while let Some(message) = queue.next_due() { - DeliveryAttempt::from(message) - .try_deliver(core.clone(), &mut queue) - .await; + loop { + let on_hold = match tokio::time::timeout(next_wake_up, self.recv()).await { + Ok(Some(Event::OnHold(on_hold))) => on_hold.into(), + Ok(Some(Event::Stop)) | Ok(None) => { + break; + } + _ => None, + }; + + // Deliver any concurrency limited messages + let mut delete_events = Vec::new(); + while let Some(queue_event) = queue.next_on_hold() { + if let Some(message) = core.read_message(queue_event.queue_id).await { + DeliveryAttempt::new(message, queue_event) + .try_deliver(core.clone()) + .await; + } else { + delete_events.push(queue_event); + } } - match result { - Ok(Some(event)) => match event { - Event::Queue(item) => { - // Deliver any concurrency limited messages - while let Some(message) = queue.next_on_hold() { - DeliveryAttempt::from(message) - .try_deliver(core.clone(), &mut queue) - .await; - } - - if item.due <= Instant::now() { - DeliveryAttempt::from(item.inner) - .try_deliver(core.clone(), &mut queue) - .await; - } else { - queue.schedule(item); - } + // Deliver scheduled messages + let now = now(); + next_wake_up = LONG_WAIT; + for queue_event in core.next_event().await { + if queue_event.due <= now { + if let Some(message) = core.read_message(queue_event.queue_id).await { + DeliveryAttempt::new(message, queue_event) + .try_deliver(core.clone()) + .await; + } else { + delete_events.push(queue_event); } - Event::Done(result) => { - // A worker is done, try delivering concurrency limited messages - while let Some(message) = queue.next_on_hold() { - DeliveryAttempt::from(message) - .try_deliver(core.clone(), &mut queue) - .await; - } - match result { - WorkerResult::Done => (), - WorkerResult::Retry(schedule) => { - queue.schedule(schedule); - } - WorkerResult::OnHold(on_hold) => { - queue.on_hold(on_hold); - } - } + } else { + next_wake_up = Duration::from_secs(queue_event.due - now); + } + } + + // Delete unlinked events + if !delete_events.is_empty() { + let core = core.clone(); + tokio::spawn(async move { + let mut batch = BatchBuilder::new(); + for queue_event in delete_events { + batch.clear(ValueClass::Queue(QueueClass::MessageEvent(queue_event))); } - Event::Manage(request) => match request { - management::QueueRequest::List { - from, - to, - before, - after, - result_tx, - } => { - let mut result = Vec::with_capacity(queue.messages.len()); - for message in queue.messages.values() { - if from.as_ref().map_or(false, |from| { - !message.return_path_lcase.contains(from) - }) { - continue; - } - if to.as_ref().map_or(false, |to| { - !message - .recipients - .iter() - .any(|rcpt| rcpt.address_lcase.contains(to)) - }) { - continue; - } + let _ = core.shared.default_data_store.write(batch.build()).await; + }); + } - if (before.is_some() || after.is_some()) - && !message.domains.iter().any(|domain| { - matches!( - &domain.status, - Status::Scheduled | Status::TemporaryFailure(_) - ) && match (&before, &after) { - (Some(before), Some(after)) => { - domain.retry.due.lt(before) - && domain.retry.due.gt(after) - } - (Some(before), None) => domain.retry.due.lt(before), - (None, Some(after)) => domain.retry.due.gt(after), - (None, None) => false, - } - }) - { - continue; - } - - result.push(message.id); - } - result.sort_unstable_by_key(|id| *id & 0xFFFFFFFF); - let _ = result_tx.send(result); - } - management::QueueRequest::Status { - queue_ids, - result_tx, - } => { - let mut result = Vec::with_capacity(queue_ids.len()); - for queue_id in queue_ids { - result.push( - queue - .messages - .get(&queue_id) - .map(|message| message.as_ref().into()), - ); - } - let _ = result_tx.send(result); - } - management::QueueRequest::Cancel { - queue_ids, - item, - result_tx, - } => { - let mut result = Vec::with_capacity(queue_ids.len()); - for queue_id in &queue_ids { - let mut found = false; - if let Some(item) = &item { - if let Some(message) = queue.messages.get_mut(queue_id) { - // Cancel delivery for all recipients that match - for rcpt in &mut message.recipients { - if rcpt.address_lcase.contains(item) { - rcpt.flags |= RCPT_STATUS_CHANGED; - rcpt.status = Status::Completed(HostResponse { - hostname: String::new(), - response: Response { - code: 0, - esc: [0, 0, 0], - message: "Delivery canceled." - .to_string(), - }, - }); - found = true; - } - } - if found { - // Mark as completed domains without any pending deliveries - for (domain_idx, domain) in - message.domains.iter_mut().enumerate() - { - if matches!( - domain.status, - Status::TemporaryFailure(_) - | Status::Scheduled - ) { - let mut total_rcpt = 0; - let mut total_completed = 0; - - for rcpt in &message.recipients { - if rcpt.domain_idx == domain_idx { - total_rcpt += 1; - if matches!( - rcpt.status, - Status::PermanentFailure(_) - | Status::Completed(_) - ) { - total_completed += 1; - } - } - } - - if total_rcpt == total_completed { - domain.status = Status::Completed(()); - domain.changed = true; - } - } - } - - // Delete message if there are no pending deliveries - if message.domains.iter().any(|domain| { - matches!( - domain.status, - Status::TemporaryFailure(_) - | Status::Scheduled - ) - }) { - message.save_changes().await; - } else { - message.remove().await; - queue.messages.remove(queue_id); - } - } - } - } else if let Some(message) = queue.messages.remove(queue_id) { - message.remove().await; - found = true; - } - result.push(found); - } - let _ = result_tx.send(result); - } - management::QueueRequest::Retry { - queue_ids, - item, - time, - result_tx, - } => { - let mut result = Vec::with_capacity(queue_ids.len()); - for queue_id in &queue_ids { - let mut found = false; - if let Some(message) = queue.messages.get_mut(queue_id) { - for domain in &mut message.domains { - if matches!( - domain.status, - Status::Scheduled | Status::TemporaryFailure(_) - ) && item - .as_ref() - .map_or(true, |item| domain.domain.contains(item)) - { - domain.retry.due = time; - if domain.expires > time { - domain.expires = time + Duration::from_secs(10); - } - domain.changed = true; - found = true; - } - } - - if found { - queue.on_hold.retain(|oh| &oh.message != queue_id); - message.save_changes().await; - if let Some(next_event) = message.next_event() { - queue.scheduled.push(Schedule { - due: next_event, - inner: *queue_id, - }); - } - } - } - result.push(found); - } - let _ = result_tx.send(result); - } - }, - Event::Stop => break, - }, - Ok(None) => break, - Err(_) => (), + // Add message on hold + if let Some(on_hold) = on_hold { + queue.on_hold(on_hold); } } }); @@ -297,36 +107,16 @@ impl SpawnQueue for mpsc::Receiver { } impl Queue { - pub fn schedule(&mut self, message: Schedule>) { - self.scheduled.push(Schedule { - due: message.due, - inner: message.inner.id, - }); - self.messages.insert(message.inner.id, message.inner); - } - - pub fn on_hold(&mut self, message: OnHold>) { + pub fn on_hold(&mut self, message: OnHold) { self.on_hold.push(OnHold { next_due: message.next_due, limiters: message.limiters, - message: message.message.id, + message: message.message, }); - self.messages.insert(message.message.id, message.message); } - pub fn next_due(&mut self) -> Option> { - let item = self.scheduled.peek()?; - if item.due <= Instant::now() { - self.scheduled - .pop() - .and_then(|i| self.messages.remove(&i.inner)) - } else { - None - } - } - - pub fn next_on_hold(&mut self) -> Option> { - let now = Instant::now(); + pub fn next_on_hold(&mut self) -> Option { + let now = now(); self.on_hold .iter() .position(|o| { @@ -335,24 +125,13 @@ impl Queue { .any(|l| l.concurrent.load(Ordering::Relaxed) < l.max_concurrent) || o.next_due.map_or(false, |due| due <= now) }) - .and_then(|pos| self.messages.remove(&self.on_hold.remove(pos).message)) - } - - pub fn wake_up_time(&self) -> Duration { - self.scheduled - .peek() - .map(|item| { - item.due - .checked_duration_since(Instant::now()) - .unwrap_or(self.short_wait) - }) - .unwrap_or(self.long_wait) + .map(|pos| self.on_hold.remove(pos).message) } } impl Message { - pub fn next_event(&self) -> Option { - let mut next_event = Instant::now(); + pub fn next_event(&self) -> Option { + let mut next_event = now(); let mut has_events = false; for domain in &self.domains { @@ -380,8 +159,8 @@ impl Message { } } - pub fn next_delivery_event(&self) -> Instant { - let mut next_delivery = Instant::now(); + pub fn next_delivery_event(&self) -> u64 { + let mut next_delivery = now(); for (pos, domain) in self .domains @@ -397,7 +176,7 @@ impl Message { next_delivery } - pub fn next_event_after(&self, instant: Instant) -> Option { + pub fn next_event_after(&self, instant: u64) -> Option { let mut next_event = None; for domain in &self.domains { @@ -431,129 +210,14 @@ impl Message { } } -impl QueueCore { - pub async fn read_queue(&self) -> Queue { - let mut queue = Queue::default(); - let mut messages = Vec::new(); - - let mut dir = match tokio::fs::read_dir(&self.config.path).await { - Ok(dir) => dir, - Err(err) => { - tracing::warn!( - "Failed to read queue directory {}: {}", - self.config.path.display(), - err - ); - return queue; - } - }; - loop { - match dir.next_entry().await { - Ok(Some(file)) => { - let file = file.path(); - if file.is_dir() { - match tokio::fs::read_dir(&file).await { - Ok(mut dir) => { - let file_ = file; - loop { - match dir.next_entry().await { - Ok(Some(file)) => { - let file = file.path(); - if file.extension().map_or(false, |e| e == "msg") { - messages - .push(tokio::spawn(Message::from_path(file))); - } - } - Ok(None) => break, - Err(err) => { - tracing::warn!( - "Failed to read queue directory {}: {}", - file_.display(), - err - ); - break; - } - } - } - } - Err(err) => { - tracing::warn!( - "Failed to read queue directory {}: {}", - file.display(), - err - ) - } - }; - } else if file.extension().map_or(false, |e| e == "msg") { - messages.push(tokio::spawn(Message::from_path(file))); - } - } - Ok(None) => { - break; - } - Err(err) => { - tracing::warn!( - "Failed to read queue directory {}: {}", - self.config.path.display(), - err - ); - break; - } - } - } - - // Join all futures - for message in messages { - match message.await { - Ok(Ok(mut message)) => { - // Reserve quota - let todo = true; - //self.has_quota(&mut message).await; - - // Schedule message - queue.schedule(Schedule { - due: message.next_event().unwrap_or_else(|| { - tracing::warn!( - context = "queue", - event = "warn", - "No due events found for message {}", - message.path.display() - ); - Instant::now() - }), - inner: Box::new(message), - }); - } - Ok(Err(err)) => { - tracing::warn!( - context = "queue", - event = "error", - "Queue startup error: {}", - err - ); - } - Err(err) => { - tracing::error!("Join error while starting queue: {}", err); - } - } - } - - queue - } -} - impl Default for Queue { fn default() -> Self { Queue { - short_wait: Duration::from_millis(1), - long_wait: Duration::from_secs(86400 * 365), - scheduled: BinaryHeap::with_capacity(128), on_hold: Vec::with_capacity(128), - messages: AHashMap::with_capacity(128), } } } pub trait SpawnQueue { - fn spawn(self, core: Arc, queue: Queue); + fn spawn(self, core: Arc); } diff --git a/crates/smtp/src/queue/mod.rs b/crates/smtp/src/queue/mod.rs index 230e1892..eb621dc9 100644 --- a/crates/smtp/src/queue/mod.rs +++ b/crates/smtp/src/queue/mod.rs @@ -24,21 +24,22 @@ use std::{ fmt::Display, net::IpAddr, - path::PathBuf, - sync::{atomic::AtomicUsize, Arc}, time::{Duration, Instant, SystemTime}, }; use serde::{Deserialize, Serialize}; use smtp_proto::Response; -use utils::listener::limiter::{ConcurrencyLimiter, InFlight}; +use store::write::{now, QueueEvent}; +use utils::{ + listener::limiter::{ConcurrencyLimiter, InFlight}, + BlobHash, +}; -use crate::core::{eval::*, management, ResolveVariable}; +use crate::core::{eval::*, ResolveVariable}; pub mod dsn; pub mod manager; pub mod quota; -pub mod serialize; pub mod spool; pub mod throttle; @@ -46,37 +47,29 @@ pub type QueueId = u64; #[derive(Debug)] pub enum Event { - Queue(Schedule>), - Manage(management::QueueRequest), - Done(WorkerResult), + Reload, + OnHold(OnHold), Stop, } -#[derive(Debug)] -pub enum WorkerResult { - Done, - Retry(Schedule>), - OnHold(OnHold>), -} - #[derive(Debug)] pub struct OnHold { - pub next_due: Option, + pub next_due: Option, pub limiters: Vec, pub message: T, } -#[derive(Debug)] +#[derive(Debug, Serialize, Deserialize)] pub struct Schedule { - pub due: Instant, + pub due: u64, pub inner: T, } -#[derive(Debug)] +#[derive(Debug, serde::Serialize, serde::Deserialize)] pub struct Message { pub id: QueueId, pub created: u64, - pub path: PathBuf, + pub blob_hash: BlobHash, pub return_path: String, pub return_path_lcase: String, @@ -89,21 +82,26 @@ pub struct Message { pub priority: i16, pub size: usize, - pub queue_refs: Vec, + pub quota_keys: Vec, } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub enum QuotaKey { + Size { key: Vec, id: u64 }, + Count { key: Vec, id: u64 }, +} + +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct Domain { pub domain: String, pub retry: Schedule, pub notify: Schedule, - pub expires: Instant, + pub expires: u64, pub status: Status<(), Error>, pub disable_tls: bool, - pub changed: bool, } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct Recipient { pub domain_idx: usize, pub address: String, @@ -128,13 +126,13 @@ pub enum Status { PermanentFailure(E), } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct HostResponse { pub hostname: T, pub response: Response, } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum Error { DnsError(String), UnexpectedResponse(HostResponse), @@ -147,7 +145,7 @@ pub enum Error { Io(String), } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct ErrorDetails { pub entity: String, pub details: String, @@ -156,32 +154,10 @@ pub struct ErrorDetails { pub struct DeliveryAttempt { pub span: tracing::Span, pub in_flight: Vec, - pub message: Box, + pub message: Message, + pub event: QueueEvent, } -#[derive(Debug)] -pub struct QuotaLimiter { - pub max_size: usize, - pub max_messages: usize, - pub size: AtomicUsize, - pub messages: AtomicUsize, -} - -#[derive(Debug)] -pub struct UsedQuota { - id: u64, - size: usize, - limiter: Arc, -} - -impl PartialEq for UsedQuota { - fn eq(&self, other: &Self) -> bool { - self.id == other.id && self.size == other.size - } -} - -impl Eq for UsedQuota {} - impl Ord for Schedule { fn cmp(&self, other: &Self) -> std::cmp::Ordering { other.due.cmp(&self.due) @@ -205,14 +181,14 @@ impl Eq for Schedule {} impl Schedule { pub fn now() -> Self { Schedule { - due: Instant::now(), + due: now(), inner: T::default(), } } pub fn later(duration: Duration) -> Self { Schedule { - due: Instant::now() + duration, + due: now() + duration.as_secs(), inner: T::default(), } } diff --git a/crates/smtp/src/queue/quota.rs b/crates/smtp/src/queue/quota.rs index 8b6b6412..e3fa2268 100644 --- a/crates/smtp/src/queue/quota.rs +++ b/crates/smtp/src/queue/quota.rs @@ -21,25 +21,26 @@ * for more details. */ -use std::sync::{atomic::Ordering, Arc}; - -use dashmap::mapref::entry::Entry; +use store::{ + write::{BatchBuilder, QueueClass, ValueClass}, + ValueKey, +}; use crate::{ config::QueueQuota, core::{ResolveVariable, SMTP}, }; -use super::{Message, QuotaLimiter, SimpleEnvelope, Status, UsedQuota}; +use super::{Message, QuotaKey, SimpleEnvelope, Status}; impl SMTP { pub async fn has_quota(&self, message: &mut Message) -> bool { - let mut queue_refs = Vec::new(); + let mut quota_keys = Vec::new(); if !self.queue.config.quota.sender.is_empty() { for quota in &self.queue.config.quota.sender { if !self - .reserve_quota(quota, message, message.size, 0, &mut queue_refs) + .check_quota(quota, message, message.size, 0, &mut quota_keys) .await { return false; @@ -50,12 +51,12 @@ impl SMTP { for quota in &self.queue.config.quota.rcpt_domain { for (pos, domain) in message.domains.iter().enumerate() { if !self - .reserve_quota( + .check_quota( quota, &SimpleEnvelope::new(message, &domain.domain), message.size, ((pos + 1) << 32) as u64, - &mut queue_refs, + &mut quota_keys, ) .await { @@ -67,7 +68,7 @@ impl SMTP { for quota in &self.queue.config.quota.rcpt { for (pos, rcpt) in message.recipients.iter().enumerate() { if !self - .reserve_quota( + .check_quota( quota, &SimpleEnvelope::new_rcpt( message, @@ -76,7 +77,7 @@ impl SMTP { ), message.size, (pos + 1) as u64, - &mut queue_refs, + &mut quota_keys, ) .await { @@ -85,47 +86,65 @@ impl SMTP { } } - message.queue_refs = queue_refs; + message.quota_keys = quota_keys; true } - async fn reserve_quota( + async fn check_quota( &self, quota: &QueueQuota, envelope: &impl ResolveVariable, size: usize, id: u64, - refs: &mut Vec, + refs: &mut Vec, ) -> bool { if !quota.expr.is_empty() && self - .eval_expr("a.expr, envelope, "reserve_quota") + .eval_expr("a.expr, envelope, "check_quota") .await .unwrap_or(false) { - match self.queue.quota.entry(quota.new_key(envelope)) { - Entry::Occupied(e) => { - if let Some(qref) = e.get().is_allowed(id, size) { - refs.push(qref); - } else { - return false; - } - } - Entry::Vacant(e) => { - let limiter = Arc::new(QuotaLimiter { - max_size: quota.size.unwrap_or(0), - max_messages: quota.messages.unwrap_or(0), - size: 0.into(), - messages: 0.into(), + let key = quota.new_key(envelope); + if let Some(max_size) = quota.size { + if self + .shared + .default_data_store + .get_counter(ValueKey::from(ValueClass::Queue(QueueClass::QuotaSize( + key.as_ref().to_vec(), + )))) + .await + .unwrap_or(0) as usize + + size + > max_size + { + return false; + } else { + refs.push(QuotaKey::Size { + key: key.as_ref().to_vec(), + id, }); + } + } - if let Some(qref) = limiter.is_allowed(id, size) { - refs.push(qref); - e.insert(limiter); - } else { - return false; - } + if let Some(max_messages) = quota.messages { + if self + .shared + .default_data_store + .get_counter(ValueKey::from(ValueClass::Queue(QueueClass::QuotaCount( + key.as_ref().to_vec(), + )))) + .await + .unwrap_or(0) as usize + + 1 + > max_messages + { + return false; + } else { + refs.push(QuotaKey::Count { + key: key.as_ref().to_vec(), + id, + }); } } } @@ -134,7 +153,10 @@ impl SMTP { } impl Message { - pub fn release_quota(&mut self) { + pub fn release_quota(&mut self, batch: &mut BatchBuilder) { + if self.quota_keys.is_empty() { + return; + } let mut quota_ids = Vec::with_capacity(self.domains.len() + self.recipients.len()); for (pos, domain) in self.domains.iter().enumerate() { if matches!( @@ -153,48 +175,21 @@ impl Message { } } if !quota_ids.is_empty() { - self.queue_refs.retain(|q| !quota_ids.contains(&q.id)); - } - } -} - -trait QuotaLimiterAllowed { - fn is_allowed(&self, id: u64, size: usize) -> Option; -} - -impl QuotaLimiterAllowed for Arc { - fn is_allowed(&self, id: u64, size: usize) -> Option { - if self.max_messages > 0 { - if self.messages.load(Ordering::Relaxed) < self.max_messages { - self.messages.fetch_add(1, Ordering::Relaxed); - } else { - return None; + let mut quota_keys = Vec::new(); + for quota_key in std::mem::take(&mut self.quota_keys) { + match quota_key { + QuotaKey::Count { id, key } if quota_ids.contains(&id) => { + batch.clear(ValueClass::Queue(QueueClass::QuotaCount(key))); + } + QuotaKey::Size { id, key } if quota_ids.contains(&id) => { + batch.clear(ValueClass::Queue(QueueClass::QuotaSize(key))); + } + _ => { + quota_keys.push(quota_key); + } + } } - } - - if self.max_size > 0 { - if self.size.load(Ordering::Relaxed) + size < self.max_size { - self.size.fetch_add(size, Ordering::Relaxed); - } else { - return None; - } - } - - Some(UsedQuota { - id, - size, - limiter: self.clone(), - }) - } -} - -impl Drop for UsedQuota { - fn drop(&mut self) { - if self.limiter.max_messages > 0 { - self.limiter.messages.fetch_sub(1, Ordering::Relaxed); - } - if self.limiter.max_size > 0 { - self.limiter.size.fetch_sub(self.size, Ordering::Relaxed); + self.quota_keys = quota_keys; } } } diff --git a/crates/smtp/src/queue/serialize.rs b/crates/smtp/src/queue/serialize.rs deleted file mode 100644 index e056888b..00000000 --- a/crates/smtp/src/queue/serialize.rs +++ /dev/null @@ -1,565 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use mail_auth::common::base32::Base32Reader; -use smtp_proto::Response; -use std::io::SeekFrom; -use std::path::PathBuf; -use std::slice::Iter; -use std::{fmt::Write, time::Instant}; -use tokio::fs; -use tokio::fs::File; -use tokio::io::{AsyncReadExt, AsyncSeekExt}; - -use super::{ - instant_to_timestamp, Domain, DomainPart, Error, ErrorDetails, HostResponse, - InstantFromTimestamp, Message, Recipient, Schedule, Status, RCPT_STATUS_CHANGED, -}; - -pub trait QueueSerializer: Sized { - fn serialize(&self, buf: &mut String); - fn deserialize(bytes: &mut Iter<'_, u8>) -> Option; -} - -impl Message { - pub fn serialize(&self) -> Vec { - let mut buf = String::with_capacity( - self.return_path.len() - + self.env_id.as_ref().map_or(0, |e| e.len()) - + (self.domains.len() * 64) - + (self.recipients.len() * 64) - + 50, - ); - - // Serialize message properties - (self.created as usize).serialize(&mut buf); - self.return_path.serialize(&mut buf); - (self.env_id.as_deref().unwrap_or_default()).serialize(&mut buf); - (self.flags as usize).serialize(&mut buf); - self.priority.serialize(&mut buf); - - // Serialize domains - let now = Instant::now(); - self.domains.len().serialize(&mut buf); - for domain in &self.domains { - domain.domain.serialize(&mut buf); - (instant_to_timestamp(now, domain.expires) as usize).serialize(&mut buf); - } - - // Serialize recipients - self.recipients.len().serialize(&mut buf); - for rcpt in &self.recipients { - rcpt.domain_idx.serialize(&mut buf); - rcpt.address.serialize(&mut buf); - (rcpt.orcpt.as_deref().unwrap_or_default()).serialize(&mut buf); - } - - // Serialize domain status - for (idx, domain) in self.domains.iter().enumerate() { - domain.serialize(idx, now, &mut buf); - } - - // Serialize recipient status - for (idx, rcpt) in self.recipients.iter().enumerate() { - rcpt.serialize(idx, &mut buf); - } - - buf.into_bytes() - } - - pub fn serialize_changes(&mut self) -> Vec { - let now = Instant::now(); - let mut buf = String::with_capacity(128); - - for (idx, domain) in self.domains.iter_mut().enumerate() { - if domain.changed { - domain.changed = false; - domain.serialize(idx, now, &mut buf); - } - } - - for (idx, rcpt) in self.recipients.iter_mut().enumerate() { - if rcpt.has_flag(RCPT_STATUS_CHANGED) { - rcpt.flags &= !RCPT_STATUS_CHANGED; - rcpt.serialize(idx, &mut buf); - } - } - - buf.into_bytes() - } - - pub async fn from_path(path: PathBuf) -> Result { - let filename = path - .file_name() - .and_then(|f| f.to_str()) - .and_then(|f| f.rsplit_once('.')) - .map(|(f, _)| f) - .ok_or_else(|| format!("Invalid queue file name {}", path.display()))?; - - // Decode file name - let mut id = [0u8; std::mem::size_of::()]; - let mut size = [0u8; std::mem::size_of::()]; - - for (pos, byte) in Base32Reader::new(filename.as_bytes()).enumerate() { - match pos { - 0..=7 => { - id[pos] = byte; - } - 8..=11 => { - size[pos - 8] = byte; - } - _ => { - return Err(format!("Invalid queue file name {}", path.display())); - } - } - } - - let id = u64::from_le_bytes(id); - let size = u32::from_le_bytes(size) as u64; - - // Obtail file size - let file_size = fs::metadata(&path) - .await - .map_err(|err| { - format!( - "Failed to obtain file metadata for {}: {}", - path.display(), - err - ) - })? - .len(); - if size == 0 || size >= file_size { - return Err(format!( - "Invalid queue file name size {} for {}", - size, - path.display() - )); - } - let mut buf = Vec::with_capacity((file_size - size) as usize); - let mut file = File::open(&path) - .await - .map_err(|err| format!("Failed to open queue file {}: {}", path.display(), err))?; - file.seek(SeekFrom::Start(size)) - .await - .map_err(|err| format!("Failed to seek queue file {}: {}", path.display(), err))?; - file.read_to_end(&mut buf) - .await - .map_err(|err| format!("Failed to read queue file {}: {}", path.display(), err))?; - - let mut message = Self::deserialize(&buf) - .ok_or_else(|| format!("Failed to deserialize metadata for file {}", path.display()))?; - message.path = path; - message.size = size as usize; - message.id = id; - Ok(message) - } - - pub fn deserialize(bytes: &[u8]) -> Option { - let mut bytes = bytes.iter(); - let created = usize::deserialize(&mut bytes)? as u64; - let return_path = String::deserialize(&mut bytes)?; - let return_path_lcase = return_path.to_lowercase(); - let env_id = String::deserialize(&mut bytes)?; - - let mut message = Message { - id: 0, - path: PathBuf::new(), - created, - return_path_domain: return_path_lcase.domain_part().to_string(), - return_path_lcase, - return_path, - env_id: if !env_id.is_empty() { - env_id.into() - } else { - None - }, - flags: usize::deserialize(&mut bytes)? as u64, - priority: i16::deserialize(&mut bytes)?, - size: 0, - recipients: vec![], - domains: vec![], - queue_refs: vec![], - }; - - // Deserialize domains - let num_domains = usize::deserialize(&mut bytes)?; - message.domains = Vec::with_capacity(num_domains); - for _ in 0..num_domains { - message.domains.push(Domain { - domain: String::deserialize(&mut bytes)?, - expires: Instant::deserialize(&mut bytes)?, - retry: Schedule::now(), - notify: Schedule::now(), - status: Status::Scheduled, - disable_tls: false, - changed: false, - }); - } - - // Deserialize recipients - let num_recipients = usize::deserialize(&mut bytes)?; - message.recipients = Vec::with_capacity(num_recipients); - for _ in 0..num_recipients { - let domain_idx = usize::deserialize(&mut bytes)?; - let address = String::deserialize(&mut bytes)?; - let orcpt = String::deserialize(&mut bytes)?; - message.recipients.push(Recipient { - domain_idx, - address_lcase: address.to_lowercase(), - address, - status: Status::Scheduled, - flags: 0, - orcpt: if !orcpt.is_empty() { - orcpt.into() - } else { - None - }, - }); - } - - // Deserialize status - while let Some((ch, idx)) = bytes - .next() - .and_then(|ch| (ch, usize::deserialize(&mut bytes)?).into()) - { - match ch { - b'D' => { - if let (Some(domain), Some(retry), Some(notify), Some(status)) = ( - message.domains.get_mut(idx), - Schedule::deserialize(&mut bytes), - Schedule::deserialize(&mut bytes), - Status::deserialize(&mut bytes), - ) { - domain.retry = retry; - domain.notify = notify; - domain.status = status; - } else { - break; - } - } - b'R' => { - if let (Some(rcpt), Some(flags), Some(status)) = ( - message.recipients.get_mut(idx), - usize::deserialize(&mut bytes), - Status::deserialize(&mut bytes), - ) { - rcpt.flags = flags as u64; - rcpt.status = status; - } else { - break; - } - } - _ => break, - } - } - - message.into() - } -} - -impl QueueSerializer for Status { - fn serialize(&self, buf: &mut String) { - match self { - Status::Scheduled => buf.push('S'), - Status::Completed(s) => { - buf.push('C'); - s.serialize(buf); - } - Status::TemporaryFailure(s) => { - buf.push('T'); - s.serialize(buf); - } - Status::PermanentFailure(s) => { - buf.push('F'); - s.serialize(buf); - } - } - } - - fn deserialize(bytes: &mut Iter<'_, u8>) -> Option { - match bytes.next()? { - b'S' => Self::Scheduled.into(), - b'C' => Self::Completed(T::deserialize(bytes)?).into(), - b'T' => Self::TemporaryFailure(E::deserialize(bytes)?).into(), - b'F' => Self::PermanentFailure(E::deserialize(bytes)?).into(), - _ => None, - } - } -} - -impl QueueSerializer for Response { - fn serialize(&self, buf: &mut String) { - let _ = write!( - buf, - "{} {} {} {} {} {}", - self.code, - self.esc[0], - self.esc[1], - self.esc[2], - self.message.len(), - self.message - ); - } - - fn deserialize(bytes: &mut Iter<'_, u8>) -> Option { - Response { - code: usize::deserialize(bytes)? as u16, - esc: [ - usize::deserialize(bytes)? as u8, - usize::deserialize(bytes)? as u8, - usize::deserialize(bytes)? as u8, - ], - message: String::deserialize(bytes)?, - } - .into() - } -} - -impl QueueSerializer for usize { - fn serialize(&self, buf: &mut String) { - let _ = write!(buf, "{self} "); - } - - fn deserialize(bytes: &mut Iter<'_, u8>) -> Option { - let mut num = 0; - loop { - match bytes.next()? { - ch @ (b'0'..=b'9') => { - num = (num * 10) + (*ch - b'0') as usize; - } - b' ' => { - return num.into(); - } - _ => { - return None; - } - } - } - } -} - -impl QueueSerializer for i16 { - fn serialize(&self, buf: &mut String) { - let _ = write!(buf, "{self} "); - } - - fn deserialize(bytes: &mut Iter<'_, u8>) -> Option { - let mut num = 0; - let mut mul = 1; - loop { - match bytes.next()? { - ch @ (b'0'..=b'9') => { - num = (num * 10) + (*ch - b'0') as i16; - } - b' ' => { - return (num * mul).into(); - } - b'-' => { - mul = -1; - } - _ => { - return None; - } - } - } - } -} - -impl QueueSerializer for ErrorDetails { - fn serialize(&self, buf: &mut String) { - self.entity.serialize(buf); - self.details.serialize(buf); - } - - fn deserialize(bytes: &mut Iter<'_, u8>) -> Option { - ErrorDetails { - entity: String::deserialize(bytes)?, - details: String::deserialize(bytes)?, - } - .into() - } -} - -impl QueueSerializer for HostResponse { - fn serialize(&self, buf: &mut String) { - self.hostname.serialize(buf); - self.response.serialize(buf); - } - - fn deserialize(bytes: &mut Iter<'_, u8>) -> Option { - HostResponse { - hostname: T::deserialize(bytes)?, - response: Response::deserialize(bytes)?, - } - .into() - } -} - -impl QueueSerializer for String { - fn serialize(&self, buf: &mut String) { - if !self.is_empty() { - let _ = write!(buf, "{} {}", self.len(), self); - } else { - buf.push_str("0 "); - } - } - - fn deserialize(bytes: &mut Iter<'_, u8>) -> Option { - match usize::deserialize(bytes)? { - len @ (1..=4096) => { - String::from_utf8(bytes.take(len).copied().collect::>()).ok() - } - 0 => String::new().into(), - _ => None, - } - } -} - -impl QueueSerializer for &str { - fn serialize(&self, buf: &mut String) { - if !self.is_empty() { - let _ = write!(buf, "{} {}", self.len(), self); - } else { - buf.push_str("0 "); - } - } - - fn deserialize(_bytes: &mut Iter<'_, u8>) -> Option { - unimplemented!() - } -} - -impl QueueSerializer for Instant { - fn serialize(&self, buf: &mut String) { - let _ = write!(buf, "{} ", instant_to_timestamp(Instant::now(), *self),); - } - - fn deserialize(bytes: &mut Iter<'_, u8>) -> Option { - (usize::deserialize(bytes)? as u64).to_instant().into() - } -} - -impl QueueSerializer for Schedule { - fn serialize(&self, buf: &mut String) { - let _ = write!( - buf, - "{} {} ", - self.inner, - instant_to_timestamp(Instant::now(), self.due), - ); - } - - fn deserialize(bytes: &mut Iter<'_, u8>) -> Option { - Schedule { - inner: usize::deserialize(bytes)? as u32, - due: Instant::deserialize(bytes)?, - } - .into() - } -} - -impl QueueSerializer for Error { - fn serialize(&self, buf: &mut String) { - match self { - Error::DnsError(e) => { - buf.push('0'); - e.serialize(buf); - } - Error::UnexpectedResponse(e) => { - buf.push('1'); - e.serialize(buf); - } - Error::ConnectionError(e) => { - buf.push('2'); - e.serialize(buf); - } - Error::TlsError(e) => { - buf.push('3'); - e.serialize(buf); - } - Error::DaneError(e) => { - buf.push('4'); - e.serialize(buf); - } - Error::MtaStsError(e) => { - buf.push('5'); - e.serialize(buf); - } - Error::RateLimited => { - buf.push('6'); - } - Error::ConcurrencyLimited => { - buf.push('7'); - } - Error::Io(e) => { - buf.push('8'); - e.serialize(buf); - } - } - } - - fn deserialize(bytes: &mut Iter<'_, u8>) -> Option { - match bytes.next()? { - b'0' => Error::DnsError(String::deserialize(bytes)?).into(), - b'1' => Error::UnexpectedResponse(HostResponse::deserialize(bytes)?).into(), - b'2' => Error::ConnectionError(ErrorDetails::deserialize(bytes)?).into(), - b'3' => Error::TlsError(ErrorDetails::deserialize(bytes)?).into(), - b'4' => Error::DaneError(ErrorDetails::deserialize(bytes)?).into(), - b'5' => Error::MtaStsError(String::deserialize(bytes)?).into(), - b'6' => Error::RateLimited.into(), - b'7' => Error::ConcurrencyLimited.into(), - b'8' => Error::Io(String::deserialize(bytes)?).into(), - _ => None, - } - } -} - -impl QueueSerializer for () { - fn serialize(&self, _buf: &mut String) {} - - fn deserialize(_bytes: &mut Iter<'_, u8>) -> Option { - Some(()) - } -} - -impl Domain { - fn serialize(&self, idx: usize, now: Instant, buf: &mut String) { - let _ = write!( - buf, - "D{} {} {} {} {} ", - idx, - self.retry.inner, - instant_to_timestamp(now, self.retry.due), - self.notify.inner, - instant_to_timestamp(now, self.notify.due) - ); - self.status.serialize(buf); - } -} - -impl Recipient { - fn serialize(&self, idx: usize, buf: &mut String) { - let _ = write!(buf, "R{} {} ", idx, self.flags); - self.status.serialize(buf); - } -} diff --git a/crates/smtp/src/queue/spool.rs b/crates/smtp/src/queue/spool.rs index fde2171c..b21371c7 100644 --- a/crates/smtp/src/queue/spool.rs +++ b/crates/smtp/src/queue/spool.rs @@ -22,163 +22,32 @@ */ use crate::queue::DomainPart; -use mail_auth::common::base32::Base32Writer; -use mail_auth::common::headers::Writer; -use std::path::PathBuf; -use std::sync::atomic::Ordering; -use std::time::Instant; +use std::borrow::Cow; use std::time::{Duration, SystemTime}; -use tokio::fs::OpenOptions; -use tokio::{fs, io::AsyncWriteExt}; +use store::write::key::DeserializeBigEndian; +use store::write::{now, BatchBuilder, Bincode, BlobOp, QueueClass, QueueEvent, ValueClass}; +use store::{IterateParams, Serialize, ValueKey, U64_LEN}; +use utils::BlobHash; use crate::core::{QueueCore, SMTP}; -use super::{Domain, Event, Message, Recipient, Schedule, SimpleEnvelope, Status}; +use super::{ + Domain, Event, Message, QueueId, QuotaKey, Recipient, Schedule, SimpleEnvelope, Status, +}; impl QueueCore { - pub async fn queue_message( + pub fn new_message( &self, - mut message: Box, - raw_headers: Option<&[u8]>, - raw_message: &[u8], - span: &tracing::Span, - ) -> bool { - // Generate id - if message.id == 0 { - message.id = self.queue_id(); - } - if message.size == 0 { - message.size = raw_message.len() + raw_headers.as_ref().map_or(0, |h| h.len()); - } - - // Build path - let todo = 1; - message.path = self.config.path.clone(); - let hash = 1; - if hash > 0 { - message.path.push((message.id % hash).to_string()); - } - let _ = fs::create_dir(&message.path).await; - - // Encode file name - let mut encoder = Base32Writer::with_capacity(20); - encoder.write(&message.id.to_le_bytes()[..]); - encoder.write(&(message.size as u32).to_le_bytes()[..]); - let mut file = encoder.finalize(); - file.push_str(".msg"); - message.path.push(file); - - // Serialize metadata - let metadata = message.serialize(); - - // Save message - let mut file = match fs::File::create(&message.path).await { - Ok(file) => file, - Err(err) => { - tracing::error!( - parent: span, - context = "queue", - event = "error", - "Failed to create file {}: {}", - message.path.display(), - err - ); - return false; - } - }; - - let iter = if let Some(raw_headers) = raw_headers { - [raw_headers, raw_message, &metadata].into_iter() - } else { - [raw_message, &metadata, b""].into_iter() - }; - - for bytes in iter { - if !bytes.is_empty() { - if let Err(err) = file.write_all(bytes).await { - tracing::error!( - parent: span, - context = "queue", - event = "error", - "Failed to write to file {}: {}", - message.path.display(), - err - ); - return false; - } - } - } - if let Err(err) = file.flush().await { - tracing::error!( - parent: span, - context = "queue", - event = "error", - "Failed to flush file {}: {}", - message.path.display(), - err - ); - return false; - } - - tracing::info!( - parent: span, - context = "queue", - event = "scheduled", - id = message.id, - from = if !message.return_path.is_empty() { - message.return_path.as_str() - } else { - "<>" - }, - nrcpts = message.recipients.len(), - size = message.size, - "Message queued for delivery." - ); - - // Queue the message - if self - .tx - .send(Event::Queue(Schedule { - due: message.next_event().unwrap(), - inner: message, - })) - .await - .is_err() - { - tracing::warn!( - parent: span, - context = "queue", - event = "error", - "Queue channel closed: Message queued but won't be sent until next restart." - ); - } - - true - } - - pub fn queue_id(&self) -> u64 { - (SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .map_or(0, |d| d.as_secs()) - .saturating_sub(946684800) - & 0xFFFFFFFF) - | (self.id_seq.fetch_add(1, Ordering::Relaxed) as u64) << 32 - } -} - -impl Message { - pub fn new_boxed( return_path: impl Into, return_path_lcase: impl Into, return_path_domain: impl Into, - ) -> Box { - Box::new(Message { - id: 0, - path: PathBuf::new(), - created: SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0), + ) -> Message { + let created = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()); + Message { + id: self.snowflake_id.generate().unwrap_or(created), + created, return_path: return_path.into(), return_path_lcase: return_path_lcase.into(), return_path_domain: return_path_domain.into(), @@ -188,8 +57,189 @@ impl Message { env_id: None, priority: 0, size: 0, - queue_refs: vec![], - }) + blob_hash: Default::default(), + quota_keys: Vec::new(), + } + } +} + +impl SMTP { + pub async fn next_event(&self) -> Vec { + let from_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { + due: 0, + queue_id: 0, + }))); + let to_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { + due: u64::MAX, + queue_id: u64::MAX, + }))); + + let mut events = Vec::new(); + let now = now(); + let result = self + .shared + .default_data_store + .iterate( + IterateParams::new(from_key, to_key).ascending().no_values(), + |key, _| { + let event = QueueEvent { + due: key.deserialize_be_u64(1)?, + queue_id: key.deserialize_be_u64(U64_LEN + 1)?, + }; + let do_continue = event.due <= now; + events.push(event); + Ok(do_continue) + }, + ) + .await; + + if let Err(err) = result { + tracing::error!( + context = "queue", + event = "error", + "Failed to read from store: {}", + err + ); + } + + events + } + + pub async fn read_message(&self, id: QueueId) -> Option { + match self + .shared + .default_data_store + .get_value::>(ValueKey::from(ValueClass::Queue(QueueClass::Message( + id, + )))) + .await + { + Ok(Some(message)) => Some(message.inner), + Ok(None) => None, + Err(err) => { + tracing::error!( + context = "queue", + event = "error", + "Failed to read message from store: {}", + err + ); + None + } + } + } +} + +impl Message { + pub async fn queue( + mut self, + raw_headers: Option<&[u8]>, + raw_message: &[u8], + core: &SMTP, + span: &tracing::Span, + ) -> bool { + // Write blob + let message = if let Some(raw_headers) = raw_headers { + let mut message = Vec::with_capacity(raw_headers.len() + raw_message.len()); + message.extend_from_slice(raw_headers); + message.extend_from_slice(raw_message); + Cow::Owned(message) + } else { + raw_message.into() + }; + self.blob_hash = BlobHash::from(message.as_ref()); + + // Generate id + if self.size == 0 { + self.size = message.len(); + } + + // Reserve and write blob + let mut batch = BatchBuilder::new(); + batch.with_account_id(u32::MAX).set( + BlobOp::Reserve { + hash: self.blob_hash.clone(), + until: self.next_delivery_event() + 3600, + }, + 0u32.serialize(), + ); + if let Err(err) = core.shared.default_data_store.write(batch.build()).await { + tracing::error!( + parent: span, + context = "queue", + event = "error", + "Failed to write to data store: {}", + err + ); + return false; + } + if let Err(err) = core + .shared + .default_blob_store + .put_blob(self.blob_hash.as_slice(), message.as_ref()) + .await + { + tracing::error!( + parent: span, + context = "queue", + event = "error", + "Failed to write to blob store: {}", + err + ); + return false; + } + + tracing::info!( + parent: span, + context = "queue", + event = "scheduled", + id = self.id, + from = if !self.return_path.is_empty() { + self.return_path.as_str() + } else { + "<>" + }, + nrcpts = self.recipients.len(), + size = self.size, + "Message queued for delivery." + ); + + // Write message to queue + let mut batch = BatchBuilder::new(); + batch + .set( + ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { + due: self.next_event().unwrap_or_default(), + queue_id: self.id, + })), + vec![], + ) + .set( + ValueClass::Queue(QueueClass::Message(self.id)), + Bincode::new(self).serialize(), + ); + + if let Err(err) = core.shared.default_data_store.write(batch.build()).await { + tracing::error!( + parent: span, + context = "queue", + event = "error", + "Failed to write to store: {}", + err + ); + return false; + } + + // Queue the message + if core.queue.tx.send(Event::Reload).await.is_err() { + tracing::warn!( + parent: span, + context = "queue", + event = "error", + "Queue channel closed: Message queued but won't be sent until next restart." + ); + } + + true } pub async fn add_recipient_parts( @@ -216,10 +266,9 @@ impl Message { domain: rcpt_domain, retry: Schedule::now(), notify: Schedule::later(expires + Duration::from_secs(10)), - expires: Instant::now() + expires, + expires: now() + expires.as_secs(), status: Status::Scheduled, disable_tls: false, - changed: false, }); idx }; @@ -241,35 +290,95 @@ impl Message { .await; } - pub async fn save_changes(&mut self) { - let buf = self.serialize_changes(); - if !buf.is_empty() { - let err = match OpenOptions::new().append(true).open(&self.path).await { - Ok(mut file) => match file.write_all(&buf).await { - Ok(_) => return, - Err(err) => err, + pub async fn save_changes( + mut self, + core: &SMTP, + prev_event: Option, + next_event: Option, + ) -> bool { + debug_assert!(prev_event.is_some() == next_event.is_some()); + + let mut batch = BatchBuilder::new(); + + // Release quota for completed deliveries + self.release_quota(&mut batch); + + // Update message queue + let mut batch = BatchBuilder::new(); + if let Some(prev_event) = prev_event { + batch.clear(ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { + due: prev_event, + queue_id: self.id, + }))); + } + if let Some(next_event) = next_event { + batch.set( + ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { + due: next_event, + queue_id: self.id, + })), + vec![], + ); + } + batch + .with_account_id(u32::MAX) + .set( + BlobOp::Reserve { + hash: self.blob_hash.clone(), + until: self.next_delivery_event() + 3600, }, - Err(err) => err, - }; + 0u32.serialize(), + ) + .set( + ValueClass::Queue(QueueClass::Message(self.id)), + Bincode::new(self).serialize(), + ); + + if let Err(err) = core.shared.default_data_store.write(batch.build()).await { tracing::error!( context = "queue", event = "error", - "Failed to write to {}: {}", - self.path.display(), + "Failed to update queued message: {}", err ); + false + } else { + true } } - pub async fn remove(&self) { - if let Err(err) = fs::remove_file(&self.path).await { + pub async fn remove(self, core: &SMTP, prev_event: u64) -> bool { + let mut batch = BatchBuilder::new(); + + // Release all quotas + for quota_key in self.quota_keys { + match quota_key { + QuotaKey::Count { key, .. } => { + batch.clear(ValueClass::Queue(QueueClass::QuotaCount(key))); + } + QuotaKey::Size { key, .. } => { + batch.clear(ValueClass::Queue(QueueClass::QuotaSize(key))); + } + } + } + + batch + .clear(ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { + due: prev_event, + queue_id: self.id, + }))) + .clear(ValueClass::Queue(QueueClass::Message(self.id))); + + if let Err(err) = core.shared.default_data_store.write(batch.build()).await { tracing::error!( context = "queue", event = "error", - "Failed to delete queued message {}: {}", - self.path.display(), + "Failed to update queued message: {}", err ); + false + } else { + true } } } diff --git a/crates/smtp/src/queue/throttle.rs b/crates/smtp/src/queue/throttle.rs index 5f624719..17130cbf 100644 --- a/crates/smtp/src/queue/throttle.rs +++ b/crates/smtp/src/queue/throttle.rs @@ -21,14 +21,13 @@ * for more details. */ -use std::time::{Duration, Instant}; - use dashmap::mapref::entry::Entry; -use utils::listener::limiter::{ConcurrencyLimiter, InFlight, RateLimiter}; +use store::write::now; +use utils::listener::limiter::{ConcurrencyLimiter, InFlight}; use crate::{ config::Throttle, - core::{throttle::Limiter, ResolveVariable, SMTP}, + core::{ResolveVariable, SMTP}, }; use super::{Domain, Status}; @@ -36,7 +35,7 @@ use super::{Domain, Status}; #[derive(Debug)] pub enum Error { Concurrency { limiter: ConcurrencyLimiter }, - Rate { retry_at: Instant }, + Rate { retry_at: u64 }, } impl SMTP { @@ -53,10 +52,33 @@ impl SMTP { .await .unwrap_or(false) { - match self.queue.throttle.entry(throttle.new_key(envelope)) { - Entry::Occupied(mut e) => { - let limiter = e.get_mut(); - if let Some(limiter) = &limiter.concurrency { + let key = throttle.new_key(envelope); + + if let Some(rate) = &throttle.rate { + if let Ok(Some(next_refill)) = self + .shared + .default_lookup_store + .is_rate_allowed(key.as_ref(), rate, false) + .await + { + tracing::info!( + parent: span, + context = "throttle", + event = "rate-limit-exceeded", + max_requests = rate.requests, + max_interval = rate.period.as_secs(), + "Queue rate limit exceeded." + ); + return Err(Error::Rate { + retry_at: now() + next_refill, + }); + } + } + + if let Some(concurrency) = &throttle.concurrency { + match self.queue.throttle.entry(key) { + Entry::Occupied(mut e) => { + let limiter = e.get_mut(); if let Some(inflight) = limiter.is_allowed() { in_flight.push(inflight); } else { @@ -72,38 +94,13 @@ impl SMTP { }); } } - if let (Some(limiter), Some(rate)) = (&mut limiter.rate, &throttle.rate) { - if !limiter.is_allowed(rate) { - tracing::info!( - parent: span, - context = "throttle", - event = "rate-limit-exceeded", - max_requests = rate.requests, - max_interval = rate.period.as_secs(), - "Queue rate limit exceeded." - ); - return Err(Error::Rate { - retry_at: Instant::now() - + Duration::from_secs(limiter.secs_to_refill()), - }); - } - } - } - Entry::Vacant(e) => { - let concurrency = throttle.concurrency.map(|concurrency| { - let limiter = ConcurrencyLimiter::new(concurrency); + Entry::Vacant(e) => { + let limiter = ConcurrencyLimiter::new(*concurrency); if let Some(inflight) = limiter.is_allowed() { in_flight.push(inflight); } - limiter - }); - let rate = throttle.rate.as_ref().map(|rate| { - let r = RateLimiter::new(rate); - r.is_allowed(rate); - r - }); - - e.insert(Limiter { rate, concurrency }); + e.insert(limiter); + } } } } @@ -124,6 +121,5 @@ impl Domain { self.status = Status::TemporaryFailure(super::Error::RateLimited); } } - self.changed = true; } } diff --git a/crates/smtp/src/reporting/dkim.rs b/crates/smtp/src/reporting/dkim.rs index 8415eff8..23a78170 100644 --- a/crates/smtp/src/reporting/dkim.rs +++ b/crates/smtp/src/reporting/dkim.rs @@ -46,7 +46,7 @@ impl Session { }; // Throttle recipient - if !self.throttle_rcpt(rcpt, rate, "dkim") { + if !self.throttle_rcpt(rcpt, rate, "dkim").await { tracing::debug!( parent: &self.span, context = "report", diff --git a/crates/smtp/src/reporting/dmarc.rs b/crates/smtp/src/reporting/dmarc.rs index 2df9b32c..c1b7d626 100644 --- a/crates/smtp/src/reporting/dmarc.rs +++ b/crates/smtp/src/reporting/dmarc.rs @@ -21,7 +21,7 @@ * for more details. */ -use std::{collections::hash_map::Entry, path::PathBuf, sync::Arc}; +use std::collections::hash_map::Entry; use ahash::AHashMap; use mail_auth::{ @@ -31,28 +31,22 @@ use mail_auth::{ ArcOutput, AuthenticatedMessage, AuthenticationResults, DkimOutput, DkimResult, DmarcOutput, SpfResult, }; -use serde::{Deserialize, Serialize}; -use tokio::{ - io::{AsyncRead, AsyncWrite}, - runtime::Handle, +use store::{ + write::{now, BatchBuilder, Bincode, QueueClass, ReportEvent, ValueClass}, + Deserialize, IterateParams, Serialize, ValueKey, }; +use tokio::io::{AsyncRead, AsyncWrite}; use utils::config::Rate; use crate::{ config::AggregateFrequency, core::{Session, SMTP}, - queue::{DomainPart, InstantFromTimestamp, RecipientDomain, Schedule}, + queue::{DomainPart, RecipientDomain}, }; -use super::{ - scheduler::{ - json_append, json_read_blocking, json_write, ReportPath, ReportPolicy, ReportType, - Scheduler, ToHash, - }, - DmarcEvent, -}; +use super::{scheduler::ToHash, DmarcEvent, SerializedSize}; -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct DmarcFormat { pub rua: Vec, pub policy: PolicyPublished, @@ -88,16 +82,15 @@ impl Session { { Some(rcpts) => { if !rcpts.is_empty() { - rcpts - .into_iter() - .filter_map(|rcpt| { - if self.throttle_rcpt(rcpt.uri(), &failure_rate, "dmarc") { - rcpt.uri().into() - } else { - None - } - }) - .collect() + let mut new_rcpts = Vec::with_capacity(rcpts.len()); + + for rcpt in rcpts { + if self.throttle_rcpt(rcpt.uri(), &failure_rate, "dmarc").await { + new_rcpts.push(rcpt.uri()); + } + } + + new_rcpts } else { if !dmarc_record.ruf().is_empty() { tracing::debug!( @@ -306,224 +299,307 @@ impl Session { } } -pub trait GenerateDmarcReport { - fn generate_dmarc_report(&self, domain: ReportPolicy, path: ReportPath); -} +impl SMTP { + pub async fn generate_dmarc_report(&self, event: ReportEvent) { + let span = tracing::info_span!( + "dmarc-report", + domain = event.domain, + range_from = event.seq_id, + range_to = event.due, + ); -impl GenerateDmarcReport for Arc { - fn generate_dmarc_report(&self, domain: ReportPolicy, path: ReportPath) { - let core = self.clone(); - let handle = Handle::current(); - - self.worker_pool.spawn(move || { - let deliver_at = path.created + path.deliver_at.as_secs(); - let span = tracing::info_span!( - "dmarc-report", - domain = domain.inner, - range_from = path.created, - range_to = deliver_at, - size = path.size, - ); - - // Deserialize report - let dmarc = if let Some(dmarc) = json_read_blocking::(&path.path, &span) { - dmarc - } else { - return; - }; - - // Verify external reporting addresses - let rua = match handle.block_on( - core.resolvers - .dns - .verify_dmarc_report_address(&domain.inner, &dmarc.rua), - ) { - Some(rcpts) => { - if !rcpts.is_empty() { - rcpts - .into_iter() - .map(|u| u.uri().to_string()) - .collect::>() - } else { - tracing::info!( - parent: &span, - event = "failed", - reason = "unauthorized-rua", - rua = ?dmarc.rua, - "Unauthorized external reporting addresses" - ); - let _ = std::fs::remove_file(&path.path); - return; - } - } - None => { - tracing::info!( - parent: &span, - event = "failed", - reason = "dns-failure", - rua = ?dmarc.rua, - "Failed to validate external report addresses", - ); - let _ = std::fs::remove_file(&path.path); - return; - } - }; - - let config = &core.report.config.dmarc_aggregate; - - // Group duplicates - let mut record_map = AHashMap::with_capacity(dmarc.records.len()); - for record in dmarc.records { - match record_map.entry(record) { - Entry::Occupied(mut e) => { - *e.get_mut() += 1; - } - Entry::Vacant(e) => { - e.insert(1u32); - } - } - } - - // Create report - let mut report = Report::new() - .with_policy_published(dmarc.policy) - .with_date_range_begin(path.created) - .with_date_range_end(deliver_at) - .with_report_id(format!("{}_{}", domain.policy, path.created)) - .with_email( - handle - .block_on(core.eval_if( - &config.address, - &RecipientDomain::new(domain.inner.as_str()), - )) - .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()), - ); - if let Some(org_name) = handle.block_on(core.eval_if::( - &config.org_name, - &RecipientDomain::new(domain.inner.as_str()), - )) { - report = report.with_org_name(org_name); - } - if let Some(contact_info) = handle.block_on(core.eval_if::( - &config.contact_info, - &RecipientDomain::new(domain.inner.as_str()), - )) { - report = report.with_extra_contact_info(contact_info); - } - for (record, count) in record_map { - report.add_record(record.with_count(count)); - } - let from_addr = handle - .block_on(core.eval_if( - &config.address, - &RecipientDomain::new(domain.inner.as_str()), - )) - .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()); - let mut message = Vec::with_capacity(path.size); - let _ = - report.write_rfc5322( - &handle - .block_on(core.eval_if( - &core.report.config.submitter, - &RecipientDomain::new(domain.inner.as_str()), - )) - .unwrap_or_else(|| "localhost".to_string()), - ( - handle - .block_on(core.eval_if( - &config.name, - &RecipientDomain::new(domain.inner.as_str()), - )) - .unwrap_or_else(|| "Mail Delivery Subsystem".to_string()) - .as_str(), - from_addr.as_str(), - ), - rua.iter().map(|a| a.as_str()), - &mut message, - ); - - // Send report - handle.block_on(core.send_report( - &from_addr, - rua.iter(), - message, - &config.sign, - &span, - false, - )); - - if let Err(err) = std::fs::remove_file(&path.path) { + // Deserialize report + let dmarc = match self + .shared + .default_data_store + .get_value::>(ValueKey::from(ValueClass::Queue( + QueueClass::DmarcReportHeader(event.clone()), + ))) + .await + { + Ok(Some(dmarc)) => dmarc.inner, + Ok(None) => { tracing::warn!( - context = "report", + parent: &span, + event = "missing", + "Failed to read DMARC report: Report not found" + ); + return; + } + Err(err) => { + tracing::warn!( + parent: &span, event = "error", - "Failed to remove report file {}: {}", - path.path.display(), + "Failed to read DMARC report: {}", err ); - } - }); - } -} - -impl Scheduler { - pub async fn schedule_dmarc(&mut self, event: Box, core: &SMTP) { - let max_size = core - .eval_if( - &core.report.config.dmarc_aggregate.max_size, - &RecipientDomain::new(event.domain.as_str()), - ) - .await - .unwrap_or(25 * 1024 * 1024); - - let policy = event.dmarc_record.to_hash(); - let (create, path) = match self.reports.entry(ReportType::Dmarc(ReportPolicy { - inner: event.domain, - policy, - })) { - Entry::Occupied(e) => (None, e.into_mut().dmarc_path()), - Entry::Vacant(e) => { - let domain = e.key().domain_name().to_string(); - let created = event.interval.to_timestamp(); - let deliver_at = created + event.interval.as_secs(); - - self.main.push(Schedule { - due: deliver_at.to_instant(), - inner: e.key().clone(), - }); - let path = core - .build_report_path(ReportType::Dmarc(&domain), policy, created, event.interval) - .await; - let v = e.insert(ReportType::Dmarc(ReportPath { - path, - deliver_at: event.interval, - created, - size: 0, - })); - (domain.into(), v.dmarc_path()) + return; } }; - if let Some(domain) = create { + // Verify external reporting addresses + let rua = match self + .resolvers + .dns + .verify_dmarc_report_address(&event.domain, &dmarc.rua) + .await + { + Some(rcpts) => { + if !rcpts.is_empty() { + rcpts + .into_iter() + .map(|u| u.uri().to_string()) + .collect::>() + } else { + tracing::info!( + parent: &span, + event = "failed", + reason = "unauthorized-rua", + rua = ?dmarc.rua, + "Unauthorized external reporting addresses" + ); + self.delete_dmarc_report(event).await; + return; + } + } + None => { + tracing::info!( + parent: &span, + event = "failed", + reason = "dns-failure", + rua = ?dmarc.rua, + "Failed to validate external report addresses", + ); + self.delete_dmarc_report(event).await; + return; + } + }; + + let mut serialized_size = serde_json::Serializer::new(SerializedSize::new( + self.eval_if( + &self.report.config.dmarc_aggregate.max_size, + &RecipientDomain::new(event.domain.as_str()), + ) + .await + .unwrap_or(25 * 1024 * 1024), + )); + let _ = serde::Serialize::serialize(&dmarc, &mut serialized_size); + let config = &self.report.config.dmarc_aggregate; + + // Group duplicates + let from_key = ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportEvent( + ReportEvent { + due: event.due, + policy_hash: event.policy_hash, + seq_id: 0, + domain: event.domain.clone(), + }, + ))); + let to_key = ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportEvent( + ReportEvent { + due: event.due, + policy_hash: event.policy_hash, + seq_id: u64::MAX, + domain: event.domain.clone(), + }, + ))); + let mut record_map = AHashMap::with_capacity(dmarc.records.len()); + if let Err(err) = self + .shared + .default_data_store + .iterate( + IterateParams::new(from_key, to_key).ascending(), + |_, v| match record_map.entry(Bincode::::deserialize(v)?.inner) { + Entry::Occupied(mut e) => { + *e.get_mut() += 1; + Ok(true) + } + Entry::Vacant(e) => { + if serde::Serialize::serialize(e.key(), &mut serialized_size).is_ok() { + e.insert(1u32); + Ok(true) + } else { + Ok(false) + } + } + }, + ) + .await + { + tracing::warn!( + parent: &span, + event = "error", + "Failed to read DMARC report: {}", + err + ); + } + + // Create report + let mut report = Report::new() + .with_policy_published(dmarc.policy) + .with_date_range_begin(event.seq_id) + .with_date_range_end(event.due) + .with_report_id(format!("{}_{}", event.policy_hash, event.seq_id)) + .with_email( + self.eval_if( + &config.address, + &RecipientDomain::new(event.domain.as_str()), + ) + .await + .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()), + ); + if let Some(org_name) = self + .eval_if::( + &config.org_name, + &RecipientDomain::new(event.domain.as_str()), + ) + .await + { + report = report.with_org_name(org_name); + } + if let Some(contact_info) = self + .eval_if::( + &config.contact_info, + &RecipientDomain::new(event.domain.as_str()), + ) + .await + { + report = report.with_extra_contact_info(contact_info); + } + for (record, count) in record_map { + report.add_record(record.with_count(count)); + } + let from_addr = self + .eval_if( + &config.address, + &RecipientDomain::new(event.domain.as_str()), + ) + .await + .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()); + let mut message = Vec::with_capacity(2048); + let _ = report.write_rfc5322( + &self + .eval_if( + &self.report.config.submitter, + &RecipientDomain::new(event.domain.as_str()), + ) + .await + .unwrap_or_else(|| "localhost".to_string()), + ( + self.eval_if(&config.name, &RecipientDomain::new(event.domain.as_str())) + .await + .unwrap_or_else(|| "Mail Delivery Subsystem".to_string()) + .as_str(), + from_addr.as_str(), + ), + rua.iter().map(|a| a.as_str()), + &mut message, + ); + + // Send report + self.send_report(&from_addr, rua.iter(), message, &config.sign, &span, false) + .await; + + self.delete_dmarc_report(event).await; + } + + pub async fn delete_dmarc_report(&self, event: ReportEvent) { + let from_key = ReportEvent { + due: event.due, + policy_hash: event.policy_hash, + seq_id: 0, + domain: event.domain.clone(), + }; + let to_key = ReportEvent { + due: event.due, + policy_hash: event.policy_hash, + seq_id: u64::MAX, + domain: event.domain.clone(), + }; + + if let Err(err) = self + .shared + .default_data_store + .delete_range( + ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportEvent(from_key))), + ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportEvent(to_key))), + ) + .await + { + tracing::warn!( + context = "report", + event = "error", + "Failed to remove repors: {}", + err + ); + return; + } + + let mut batch = BatchBuilder::new(); + batch.clear(ValueClass::Queue(QueueClass::DmarcReportHeader(event))); + if let Err(err) = self.shared.default_data_store.write(batch.build()).await { + tracing::warn!( + context = "report", + event = "error", + "Failed to remove repors: {}", + err + ); + } + } + + pub async fn schedule_dmarc(&self, event: Box) { + let created = event.interval.to_timestamp(); + let deliver_at = created + event.interval.as_secs(); + let mut report_event = ReportEvent { + due: deliver_at, + policy_hash: event.dmarc_record.to_hash(), + seq_id: created, + domain: event.domain, + }; + + // Write policy if missing + let mut builder = BatchBuilder::new(); + if self + .shared + .default_data_store + .get_value::<()>(ValueKey::from(ValueClass::Queue( + QueueClass::DmarcReportHeader(report_event.clone()), + ))) + .await + .unwrap_or_default() + .is_none() + { // Serialize report let entry = DmarcFormat { rua: event.dmarc_record.rua().to_vec(), - policy: PolicyPublished::from_record(domain, &event.dmarc_record), - records: vec![event.report_record], + policy: PolicyPublished::from_record( + report_event.domain.to_string(), + &event.dmarc_record, + ), + records: vec![], }; - let bytes_written = json_write(&path.path, &entry).await; - if bytes_written > 0 { - path.size += bytes_written; - } else { - // Something went wrong, remove record - self.reports.remove(&ReportType::Dmarc(ReportPolicy { - inner: entry.policy.domain, - policy, - })); - } - } else if path.size < max_size { - // Append to existing report - path.size += json_append(&path.path, &event.report_record, max_size - path.size).await; + // Write report + builder.set( + ValueClass::Queue(QueueClass::DmarcReportHeader(report_event.clone())), + Bincode::new(entry).serialize(), + ); + } + + // Write entry + report_event.seq_id = self.queue.snowflake_id.generate().unwrap_or_else(now); + builder.set( + ValueClass::Queue(QueueClass::DmarcReportEvent(report_event)), + Bincode::new(event.report_record).serialize(), + ); + + if let Err(err) = self.shared.default_data_store.write(builder.build()).await { + tracing::error!( + context = "report", + event = "error", + "Failed to write DMARC report event: {}", + err + ); } } } diff --git a/crates/smtp/src/reporting/mod.rs b/crates/smtp/src/reporting/mod.rs index d2dd6b73..4dd2c6f8 100644 --- a/crates/smtp/src/reporting/mod.rs +++ b/crates/smtp/src/reporting/mod.rs @@ -21,7 +21,7 @@ * for more details. */ -use std::{sync::Arc, time::SystemTime}; +use std::{io, sync::Arc, time::SystemTime}; use mail_auth::{ common::headers::HeaderWriter, @@ -37,15 +37,13 @@ use tokio::io::{AsyncRead, AsyncWrite}; use utils::config::if_block::IfBlock; use crate::{ - config::{AddressMatch, AggregateFrequency, DkimSigner}, - core::{management, Session, SMTP}, + config::{AddressMatch, AggregateFrequency}, + core::{Session, SMTP}, outbound::{dane::Tlsa, mta_sts::Policy}, queue::{DomainPart, Message}, USER_AGENT, }; -use self::scheduler::{ReportKey, ReportValue}; - pub mod analysis; pub mod dkim; pub mod dmarc; @@ -57,7 +55,6 @@ pub mod tls; pub enum Event { Dmarc(Box), Tls(Box), - Manage(management::ReportRequest), Stop, } @@ -137,9 +134,11 @@ impl SMTP { // Build message let from_addr_lcase = from_addr.to_lowercase(); let from_addr_domain = from_addr_lcase.domain_part().to_string(); - let mut message = Message::new_boxed(from_addr, from_addr_lcase, from_addr_domain); + let mut message = self + .queue + .new_message(from_addr, from_addr_lcase, from_addr_domain); for rcpt_ in rcpts { - message.add_recipient(rcpt_.as_ref(), &self).await; + message.add_recipient(rcpt_.as_ref(), self).await; } // Sign message @@ -164,8 +163,8 @@ impl SMTP { } // Queue message - self.queue - .queue_message(message, signature.as_deref(), &report, span) + message + .queue(signature.as_deref(), &report, self, span) .await; } @@ -300,42 +299,28 @@ impl From<(&Option>, &Option>)> for PolicyType { } } -impl ReportKey { - pub fn domain(&self) -> &str { - match self { - scheduler::ReportType::Dmarc(p) => &p.inner, - scheduler::ReportType::Tls(d) => d, - } +pub(crate) struct SerializedSize { + bytes_left: usize, +} + +impl SerializedSize { + pub fn new(bytes_left: usize) -> Self { + Self { bytes_left } } } -impl ReportValue { - pub async fn delete(&self) { - match self { - scheduler::ReportType::Dmarc(path) => { - if let Err(err) = tokio::fs::remove_file(&path.path).await { - tracing::warn!( - context = "report", - event = "error", - "Failed to remove report file {}: {}", - path.path.display(), - err - ); - } - } - scheduler::ReportType::Tls(path) => { - for path in &path.path { - if let Err(err) = tokio::fs::remove_file(&path.inner).await { - tracing::warn!( - context = "report", - event = "error", - "Failed to remove report file {}: {}", - path.inner.display(), - err - ); - } - } - } +impl io::Write for SerializedSize { + fn write(&mut self, buf: &[u8]) -> io::Result { + let buf_len = buf.len(); + if buf_len <= self.bytes_left { + self.bytes_left -= buf_len; + Ok(buf_len) + } else { + Err(io::Error::new(io::ErrorKind::Other, "Size exceeded")) } } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } } diff --git a/crates/smtp/src/reporting/scheduler.rs b/crates/smtp/src/reporting/scheduler.rs index 30772c67..5cc29268 100644 --- a/crates/smtp/src/reporting/scheduler.rs +++ b/crates/smtp/src/reporting/scheduler.rs @@ -22,156 +22,82 @@ */ use ahash::{AHashMap, RandomState}; -use mail_auth::{ - common::{ - base32::{Base32Reader, Base32Writer}, - headers::Writer, - }, - dmarc::Dmarc, -}; +use mail_auth::dmarc::Dmarc; -use serde::{de::DeserializeOwned, Serialize}; use std::{ - collections::{hash_map::Entry, BinaryHeap}, - hash::Hash, - path::PathBuf, sync::Arc, time::{Duration, Instant, SystemTime}, }; -use tokio::{ - fs::{self, OpenOptions}, - io::AsyncWriteExt, - sync::mpsc, +use store::{ + write::{now, QueueClass, ReportEvent, ValueClass}, + Deserialize, IterateParams, ValueKey, }; +use tokio::sync::mpsc; use crate::{ - config::AggregateFrequency, - core::{management::ReportRequest, worker::SpawnCleanup, ReportCore, SMTP}, - queue::{InstantFromTimestamp, Schedule}, + core::{worker::SpawnCleanup, SMTP}, + queue::manager::LONG_WAIT, }; -use super::{dmarc::GenerateDmarcReport, tls::GenerateTlsReport, Event}; - -pub type ReportKey = ReportType, String>; -pub type ReportValue = ReportType, ReportPath>>>; - -pub struct Scheduler { - short_wait: Duration, - long_wait: Duration, - pub main: BinaryHeap>, - pub reports: AHashMap, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)] -pub enum ReportType { - Dmarc(T), - Tls(U), -} - -#[derive(Debug, PartialEq, Eq)] -pub struct ReportPath { - pub path: T, - pub size: usize, - pub created: u64, - pub deliver_at: AggregateFrequency, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct ReportPolicy { - pub inner: T, - pub policy: u64, -} +use super::Event; impl SpawnReport for mpsc::Receiver { - fn spawn(mut self, core: Arc, mut scheduler: Scheduler) { + fn spawn(mut self, core: Arc) { tokio::spawn(async move { let mut last_cleanup = Instant::now(); + let mut next_wake_up; loop { - match tokio::time::timeout(scheduler.wake_up_time(), self.recv()).await { + // Read events + let now = now(); + let events = core.next_report_event().await; + next_wake_up = events + .last() + .and_then(|e| match e { + QueueClass::DmarcReportHeader(e) | QueueClass::TlsReportHeader(e) + if e.due > now => + { + Duration::from_secs(e.due - now).into() + } + _ => None, + }) + .unwrap_or(LONG_WAIT); + + let core_ = core.clone(); + tokio::spawn(async move { + let mut tls_reports = AHashMap::new(); + for report_event in events { + match report_event { + QueueClass::DmarcReportHeader(event) if event.due <= now => { + core_.generate_dmarc_report(event).await; + } + QueueClass::TlsReportHeader(event) if event.due <= now => { + tls_reports + .entry(event.domain.clone()) + .or_insert_with(Vec::new) + .push(event); + } + _ => (), + } + } + + for (domain_name, tls_report) in tls_reports { + core_.generate_tls_report(domain_name, tls_report).await; + } + }); + + match tokio::time::timeout(next_wake_up, self.recv()).await { Ok(Some(event)) => match event { Event::Dmarc(event) => { - scheduler.schedule_dmarc(event, &core).await; + core.schedule_dmarc(event).await; } Event::Tls(event) => { - scheduler.schedule_tls(event, &core).await; + core.schedule_tls(event).await; } - Event::Manage(request) => match request { - ReportRequest::List { - type_, - domain, - result_tx, - } => { - let mut result = Vec::new(); - for key in scheduler.reports.keys() { - if domain - .as_ref() - .map_or(false, |domain| domain != key.domain()) - { - continue; - } - if let Some(type_) = &type_ { - if !matches!( - (key, type_), - (ReportType::Dmarc(_), ReportType::Dmarc(_)) - | (ReportType::Tls(_), ReportType::Tls(_)) - ) { - continue; - } - } - result.push(key.to_string()); - } - let _ = result_tx.send(result); - } - ReportRequest::Status { - report_ids, - result_tx, - } => { - let mut result = Vec::with_capacity(report_ids.len()); - for report_id in &report_ids { - result.push( - scheduler - .reports - .get(report_id) - .map(|report_value| (report_id, report_value).into()), - ); - } - let _ = result_tx.send(result); - } - ReportRequest::Cancel { - report_ids, - result_tx, - } => { - let mut result = Vec::with_capacity(report_ids.len()); - for report_id in &report_ids { - result.push( - if let Some(report) = scheduler.reports.remove(report_id) { - report.delete().await; - true - } else { - false - }, - ); - } - let _ = result_tx.send(result); - } - }, Event::Stop => break, }, Ok(None) => break, Err(_) => { - while let Some(report) = scheduler.next_due() { - match report { - (ReportType::Dmarc(domain), ReportType::Dmarc(path)) => { - core.generate_dmarc_report(domain, path); - } - (ReportType::Tls(domain), ReportType::Tls(path)) => { - core.generate_tls_report(domain, path); - } - _ => unreachable!(), - } - } - // Cleanup expired throttles if last_cleanup.elapsed().as_secs() >= 86400 { last_cleanup = Instant::now(); @@ -185,429 +111,54 @@ impl SpawnReport for mpsc::Receiver { } impl SMTP { - pub async fn build_report_path( - &self, - domain: ReportType<&str, &str>, - policy: u64, - created: u64, - interval: AggregateFrequency, - ) -> PathBuf { - let (ext, domain) = match domain { - ReportType::Dmarc(domain) => ("d", domain), - ReportType::Tls(domain) => ("t", domain), - }; - - // Build base path - let mut path = self.report.config.path.clone(); - let todo = "fix"; - let hash = 1; - if hash > 0 { - path.push((policy % hash).to_string()); - } - let _ = fs::create_dir(&path).await; - - // Build filename - let mut w = Base32Writer::with_capacity(domain.len() + 13); - w.write(&policy.to_le_bytes()[..]); - w.write(&(created.saturating_sub(946684800) as u32).to_le_bytes()[..]); - w.push_byte( - match interval { - AggregateFrequency::Hourly => 0, - AggregateFrequency::Daily => 1, - AggregateFrequency::Weekly => 2, - AggregateFrequency::Never => 3, + pub async fn next_report_event(&self) -> Vec { + let from_key = ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportHeader( + ReportEvent { + due: 0, + policy_hash: 0, + seq_id: 0, + domain: String::new(), }, - false, - ); - w.write(domain.as_bytes()); - let mut file = w.finalize(); - file.push('.'); - file.push_str(ext); - path.push(file); - path - } -} + ))); + let to_key = ValueKey::from(ValueClass::Queue(QueueClass::TlsReportHeader( + ReportEvent { + due: u64::MAX, + policy_hash: 0, + seq_id: 0, + domain: String::new(), + }, + ))); -impl ReportCore { - pub async fn read_reports(&self) -> Scheduler { - let mut scheduler = Scheduler::default(); - - let mut dir = match tokio::fs::read_dir(&self.config.path).await { - Ok(dir) => dir, - Err(_) => { - return scheduler; - } - }; - loop { - match dir.next_entry().await { - Ok(Some(file)) => { - let file = file.path(); - if file.is_dir() { - match tokio::fs::read_dir(&file).await { - Ok(mut dir) => { - let file_ = file; - loop { - match dir.next_entry().await { - Ok(Some(file)) => { - let file = file.path(); - if file - .extension() - .map_or(false, |e| e == "t" || e == "d") - { - if let Err(err) = scheduler.add_path(file).await { - tracing::warn!("{}", err); - } - } - } - Ok(None) => break, - Err(err) => { - tracing::warn!( - "Failed to read report directory {}: {}", - file_.display(), - err - ); - break; - } - } - } - } - Err(err) => { - tracing::warn!( - "Failed to read report directory {}: {}", - file.display(), - err - ) - } - }; - } else if file.extension().map_or(false, |e| e == "t" || e == "d") { - if let Err(err) = scheduler.add_path(file).await { - tracing::warn!("{}", err); - } - } - } - Ok(None) => { - break; - } - Err(err) => { - tracing::warn!( - "Failed to read report directory {}: {}", - self.config.path.display(), - err - ); - break; - } - } - } - - scheduler - } -} - -impl Scheduler { - pub fn next_due(&mut self) -> Option<(ReportKey, ReportValue)> { - let item = self.main.peek()?; - if item.due <= Instant::now() { - let item = self.main.pop().unwrap(); - self.reports - .remove(&item.inner) - .map(|policy| (item.inner, policy)) - } else { - None - } - } - - pub fn wake_up_time(&self) -> Duration { - self.main - .peek() - .map(|item| { - item.due - .checked_duration_since(Instant::now()) - .unwrap_or(self.short_wait) - }) - .unwrap_or(self.long_wait) - } - - pub async fn add_path(&mut self, path: PathBuf) -> Result<(), String> { - let (file, ext) = path - .file_name() - .and_then(|f| f.to_str()) - .and_then(|f| f.rsplit_once('.')) - .ok_or_else(|| format!("Invalid queue file name {}", path.display()))?; - let file_size = fs::metadata(&path) - .await - .map_err(|err| { - format!( - "Failed to obtain file metadata for {}: {}", - path.display(), - err - ) - })? - .len(); - if file_size == 0 { - let _ = fs::remove_file(&path).await; - return Err(format!( - "Removed zero length report file {}", - path.display() - )); - } - - // Decode domain name - let mut policy = [0u8; std::mem::size_of::()]; - let mut created = [0u8; std::mem::size_of::()]; - let mut deliver_at = AggregateFrequency::Never; - let mut domain = Vec::new(); - for (pos, byte) in Base32Reader::new(file.as_bytes()).enumerate() { - match pos { - 0..=7 => { - policy[pos] = byte; - } - 8..=11 => { - created[pos - 8] = byte; - } - 12 => { - deliver_at = match byte { - 0 => AggregateFrequency::Hourly, - 1 => AggregateFrequency::Daily, - 2 => AggregateFrequency::Weekly, - _ => { - return Err(format!( - "Failed to base32 decode report file {}", - path.display() - )); - } - }; - } - _ => { - domain.push(byte); - } - } - } - if domain.is_empty() { - return Err(format!( - "Failed to base32 decode report file {}", - path.display() - )); - } - let domain = String::from_utf8(domain).map_err(|err| { - format!( - "Failed to base32 decode report file {}: {}", - path.display(), - err - ) - })?; - - // Rebuild parts - let policy = u64::from_le_bytes(policy); - let created = u32::from_le_bytes(created) as u64 + 946684800; - - match ext { - "d" => { - let key = ReportType::Dmarc(ReportPolicy { - inner: domain, - policy, - }); - self.reports.insert( - key.clone(), - ReportType::Dmarc(ReportPath { - path, - size: file_size as usize, - created, - deliver_at, - }), - ); - self.main.push(Schedule { - due: (created + deliver_at.as_secs()).to_instant(), - inner: key, - }); - } - "t" => match self.reports.entry(ReportType::Tls(domain)) { - Entry::Occupied(mut e) => { - if let ReportType::Tls(tls) = e.get_mut() { - tls.size += file_size as usize; - tls.path.push(ReportPolicy { - inner: path, - policy, - }); - } - } - Entry::Vacant(e) => { - self.main.push(Schedule { - due: (created + deliver_at.as_secs()).to_instant(), - inner: e.key().clone(), + let mut events = Vec::new(); + let now = now(); + let result = self + .shared + .default_data_store + .iterate( + IterateParams::new(from_key, to_key).ascending().no_values(), + |key, _| { + let event = ReportEvent::deserialize(key)?; + let do_continue = event.due <= now; + events.push(if *key.last().unwrap() == 0 { + QueueClass::DmarcReportHeader(event) + } else { + QueueClass::TlsReportHeader(event) }); - e.insert(ReportType::Tls(ReportPath { - path: vec![ReportPolicy { - inner: path, - policy, - }], - size: file_size as usize, - created, - deliver_at, - })); - } - }, - _ => unreachable!(), - } + Ok(do_continue) + }, + ) + .await; - Ok(()) - } -} - -pub async fn json_write(path: &PathBuf, entry: &impl Serialize) -> usize { - if let Ok(bytes) = serde_json::to_vec(entry) { - // Save serialized report - let bytes_written = bytes.len() - 2; - match fs::File::create(&path).await { - Ok(mut file) => match file.write_all(&bytes[..bytes_written]).await { - Ok(_) => bytes_written, - Err(err) => { - tracing::error!( - context = "report", - event = "error", - "Failed to write to report file {}: {}", - path.display(), - err - ); - 0 - } - }, - Err(err) => { - tracing::error!( - context = "report", - event = "error", - "Failed to create report file {}: {}", - path.display(), - err - ); - 0 - } - } - } else { - 0 - } -} - -pub async fn json_append(path: &PathBuf, entry: &impl Serialize, bytes_left: usize) -> usize { - let mut bytes = Vec::with_capacity(128); - bytes.push(b','); - if serde_json::to_writer(&mut bytes, entry).is_ok() && bytes.len() <= bytes_left { - let err = match OpenOptions::new().append(true).open(&path).await { - Ok(mut file) => match file.write_all(&bytes).await { - Ok(_) => return bytes.len(), - Err(err) => err, - }, - Err(err) => err, - }; - tracing::error!( - context = "report", - event = "error", - "Failed to append report to {}: {}", - path.display(), - err - ); - } - 0 -} - -pub async fn json_read(path: &PathBuf, span: &tracing::Span) -> Option { - match fs::read_to_string(&path).await { - Ok(mut json) => { - json.push_str("]}"); - match serde_json::from_str(&json) { - Ok(report) => Some(report), - Err(err) => { - tracing::error!( - parent: span, - context = "deserialize", - event = "error", - "Failed to deserialize report file {}: {}", - path.display(), - err - ); - None - } - } - } - Err(err) => { + if let Err(err) = result { tracing::error!( - parent: span, - context = "io", + context = "queue", event = "error", - "Failed to read report file {}: {}", - path.display(), + "Failed to read from store: {}", err ); - None } - } -} -pub fn json_read_blocking(path: &PathBuf, span: &tracing::Span) -> Option { - match std::fs::read_to_string(path) { - Ok(mut json) => { - json.push_str("]}"); - match serde_json::from_str(&json) { - Ok(report) => Some(report), - Err(err) => { - tracing::error!( - parent: span, - context = "deserialize", - event = "error", - "Failed to deserialize report file {}: {}", - path.display(), - err - ); - None - } - } - } - Err(err) => { - tracing::error!( - parent: span, - context = "io", - event = "error", - "Failed to read report file {}: {}", - path.display(), - err - ); - None - } - } -} - -impl Default for Scheduler { - fn default() -> Self { - Self { - short_wait: Duration::from_millis(1), - long_wait: Duration::from_secs(86400 * 365), - main: BinaryHeap::with_capacity(128), - reports: AHashMap::with_capacity(128), - } - } -} - -impl ReportKey { - pub fn domain_name(&self) -> &str { - match self { - ReportType::Dmarc(domain) => domain.inner.as_str(), - ReportType::Tls(domain) => domain.as_str(), - } - } -} - -impl ReportValue { - pub fn dmarc_path(&mut self) -> &mut ReportPath { - match self { - ReportType::Dmarc(path) => path, - ReportType::Tls(_) => unreachable!(), - } - } - - pub fn tls_path(&mut self) -> &mut ReportPath>> { - match self { - ReportType::Tls(path) => path, - ReportType::Dmarc(_) => unreachable!(), - } + events } } @@ -641,5 +192,5 @@ impl ToTimestamp for Duration { } pub trait SpawnReport { - fn spawn(self, core: Arc, scheduler: Scheduler); + fn spawn(self, core: Arc); } diff --git a/crates/smtp/src/reporting/spf.rs b/crates/smtp/src/reporting/spf.rs index 1f5c37e5..ac2398d1 100644 --- a/crates/smtp/src/reporting/spf.rs +++ b/crates/smtp/src/reporting/spf.rs @@ -36,7 +36,7 @@ impl Session { output: &SpfOutput, ) { // Throttle recipient - if !self.throttle_rcpt(rcpt, rate, "spf") { + if !self.throttle_rcpt(rcpt, rate, "spf").await { tracing::debug!( parent: &self.span, context = "report", diff --git a/crates/smtp/src/reporting/tls.rs b/crates/smtp/src/reporting/tls.rs index 052d2757..a65e7e0e 100644 --- a/crates/smtp/src/reporting/tls.rs +++ b/crates/smtp/src/reporting/tls.rs @@ -21,7 +21,7 @@ * for more details. */ -use std::{collections::hash_map::Entry, path::PathBuf, sync::Arc, time::Duration}; +use std::{collections::hash_map::Entry, sync::Arc, time::Duration}; use ahash::AHashMap; use mail_auth::{ @@ -34,25 +34,21 @@ use mail_auth::{ use mail_parser::DateTime; use reqwest::header::CONTENT_TYPE; -use serde::{Deserialize, Serialize}; use std::fmt::Write; -use tokio::runtime::Handle; +use store::{ + write::{now, BatchBuilder, Bincode, QueueClass, ReportEvent, ValueClass}, + Deserialize, IterateParams, Serialize, ValueKey, +}; use crate::{ config::AggregateFrequency, core::SMTP, outbound::mta_sts::{Mode, MxPattern}, - queue::{InstantFromTimestamp, RecipientDomain, Schedule}, + queue::RecipientDomain, USER_AGENT, }; -use super::{ - scheduler::{ - json_append, json_read_blocking, json_write, ReportPath, ReportPolicy, ReportType, - Scheduler, ToHash, - }, - TlsEvent, -}; +use super::{scheduler::ToHash, SerializedSize, TlsEvent}; #[derive(Debug, Clone)] pub struct TlsRptOptions { @@ -60,310 +56,338 @@ pub struct TlsRptOptions { pub interval: AggregateFrequency, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, serde::Serialize, serde::Deserialize)] struct TlsFormat { rua: Vec, policy: PolicyDetails, records: Vec>, } -pub trait GenerateTlsReport { - fn generate_tls_report(&self, domain: String, paths: ReportPath>>); -} - #[cfg(feature = "test_mode")] pub static TLS_HTTP_REPORT: parking_lot::Mutex> = parking_lot::Mutex::new(Vec::new()); -impl GenerateTlsReport for Arc { - fn generate_tls_report(&self, domain: String, path: ReportPath>>) { - let core = self.clone(); - let handle = Handle::current(); +impl SMTP { + pub async fn generate_tls_report(&self, domain_name: String, events: Vec) { + let (event_from, event_to, policy) = events + .first() + .map(|e| (e.seq_id, e.due, e.policy_hash)) + .unwrap(); - self.worker_pool.spawn(move || { - let deliver_at = path.created + path.deliver_at.as_secs(); - let span = tracing::info_span!( - "tls-report", - domain = domain, - range_from = path.created, - range_to = deliver_at, - size = path.size, - ); + let span = tracing::info_span!( + "tls-report", + domain = domain_name, + range_from = event_from, + range_to = event_to, + ); - // Deserialize report - let config = &core.report.config.tls; - let mut report = TlsReport { - organization_name: handle - .block_on( - core.eval_if(&config.org_name, &RecipientDomain::new(domain.as_str())), - ) - .clone(), - date_range: DateRange { - start_datetime: DateTime::from_timestamp(path.created as i64), - end_datetime: DateTime::from_timestamp(deliver_at as i64), - }, - contact_info: handle - .block_on( - core.eval_if(&config.contact_info, &RecipientDomain::new(domain.as_str())), - ) - .clone(), - report_id: format!( - "{}_{}", - path.created, - path.path.first().map_or(0, |p| p.policy) - ), - policies: Vec::with_capacity(path.path.len()), - }; - let mut rua = Vec::new(); - for path in &path.path { - if let Some(tls) = json_read_blocking::(&path.inner, &span) { - // Group duplicates - let mut total_success = 0; - let mut total_failure = 0; - let mut record_map = AHashMap::with_capacity(tls.records.len()); - for record in tls.records { - if let Some(record) = record { - match record_map.entry(record) { - Entry::Occupied(mut e) => { - *e.get_mut() += 1; - } - Entry::Vacant(e) => { - e.insert(1u32); - } - } - total_failure += 1; - } else { - total_success += 1; - } - } - report.policies.push(Policy { - policy: tls.policy, - summary: Summary { - total_success, - total_failure, - }, - failure_details: record_map - .into_iter() - .map(|(mut r, count)| { - r.failed_session_count = count; - r - }) - .collect(), - }); - - rua = tls.rua; - } - } - - if report.policies.is_empty() { - // This should not happen - tracing::warn!( - parent: &span, - event = "empty-report", - "No policies found in report" - ); - path.cleanup_blocking(); - return; - } - - // Compress and serialize report - let json = report.to_json(); - let mut e = GzEncoder::new(Vec::with_capacity(json.len()), Compression::default()); - let json = - match std::io::Write::write_all(&mut e, json.as_bytes()).and_then(|_| e.finish()) { - Ok(report) => report, - Err(err) => { - tracing::error!( - parent: &span, - event = "error", - "Failed to compress report: {}", - err - ); - return; - } - }; - - // Try delivering report over HTTP - let mut rcpts = Vec::with_capacity(rua.len()); - for uri in &rua { - match uri { - ReportUri::Http(uri) => { - if let Ok(client) = reqwest::blocking::Client::builder() - .user_agent(USER_AGENT) - .timeout(Duration::from_secs(2 * 60)) - .build() - { - #[cfg(feature = "test_mode")] - if uri == "https://127.0.0.1/tls" { - TLS_HTTP_REPORT.lock().extend_from_slice(&json); - path.cleanup_blocking(); - return; - } - - match client - .post(uri) - .header(CONTENT_TYPE, "application/tlsrpt+gzip") - .body(json.to_vec()) - .send() - { - Ok(response) => { - if response.status().is_success() { - tracing::info!( - parent: &span, - context = "http", - event = "success", - url = uri, - ); - path.cleanup_blocking(); - return; - } else { - tracing::debug!( - parent: &span, - context = "http", - event = "invalid-response", - url = uri, - status = %response.status() - ); - } - } - Err(err) => { - tracing::debug!( - parent: &span, - context = "http", - event = "error", - url = uri, - reason = %err - ); - } - } - } - } - ReportUri::Mail(mailto) => { - rcpts.push(mailto.as_str()); - } - } - } - - // Deliver report over SMTP - if !rcpts.is_empty() { - let from_addr = handle - .block_on(core.eval_if(&config.address, &RecipientDomain::new(domain.as_str()))) - .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()); - let mut message = Vec::with_capacity(path.size); - let _ = report.write_rfc5322_from_bytes( - &domain, - &handle - .block_on(core.eval_if( - &core.report.config.submitter, - &RecipientDomain::new(domain.as_str()), - )) - .unwrap_or_else(|| "localhost".to_string()), - ( - handle - .block_on( - core.eval_if(&config.name, &RecipientDomain::new(domain.as_str())), - ) - .unwrap_or_else(|| "Mail Delivery Subsystem".to_string()) - .as_str(), - from_addr.as_str(), - ), - rcpts.iter().copied(), - &json, - &mut message, - ); - - // Send report - handle.block_on(core.send_report( - &from_addr, - rcpts.iter(), - message, - &config.sign, - &span, - false, - )); - } else { - tracing::info!( - parent: &span, - event = "delivery-failed", - "No valid recipients found to deliver report to." - ); - } - path.cleanup_blocking(); - }); - } -} - -impl Scheduler { - pub async fn schedule_tls(&mut self, event: Box, core: &SMTP) { - let max_size = core - .eval_if( - &core.report.config.tls.max_size, - &RecipientDomain::new(event.domain.as_str()), + // Deserialize report + let config = &self.report.config.tls; + let mut report = TlsReport { + organization_name: self + .eval_if( + &config.org_name, + &RecipientDomain::new(domain_name.as_str()), + ) + .await + .clone(), + date_range: DateRange { + start_datetime: DateTime::from_timestamp(event_from as i64), + end_datetime: DateTime::from_timestamp(event_to as i64), + }, + contact_info: self + .eval_if( + &config.contact_info, + &RecipientDomain::new(domain_name.as_str()), + ) + .await + .clone(), + report_id: format!("{}_{}", event_from, policy), + policies: Vec::with_capacity(events.len()), + }; + let mut rua = Vec::new(); + let mut serialized_size = serde_json::Serializer::new(SerializedSize::new( + self.eval_if( + &self.report.config.tls.max_size, + &RecipientDomain::new(domain_name.as_str()), ) .await - .unwrap_or(25 * 1024 * 1024); - let policy_hash = event.policy.to_hash(); + .unwrap_or(25 * 1024 * 1024), + )); + let _ = serde::Serialize::serialize(&report, &mut serialized_size); - let (path, pos, create) = match self.reports.entry(ReportType::Tls(event.domain)) { - Entry::Occupied(e) => { - if let ReportType::Tls(path) = e.get() { - if let Some(pos) = path.path.iter().position(|p| p.policy == policy_hash) { - (e.into_mut().tls_path(), pos, None) - } else { - let pos = path.path.len(); - let domain = e.key().domain_name().to_string(); - let path = e.into_mut().tls_path(); - path.path.push(ReportPolicy { - inner: core - .build_report_path( - ReportType::Tls(&domain), - policy_hash, - path.created, - path.deliver_at, - ) - .await, - policy: policy_hash, - }); - (path, pos, domain.into()) - } - } else { - unreachable!() + for event in &events { + // Deserialize report + let tls = match self + .shared + .default_data_store + .get_value::>(ValueKey::from(ValueClass::Queue( + QueueClass::TlsReportHeader(event.clone()), + ))) + .await + { + Ok(Some(dmarc)) => dmarc.inner, + Ok(None) => { + tracing::warn!( + parent: &span, + event = "missing", + "Failed to read DMARC report: Report not found" + ); + continue; } - } - Entry::Vacant(e) => { - let created = event.interval.to_timestamp(); - let deliver_at = created + event.interval.as_secs(); + Err(err) => { + tracing::warn!( + parent: &span, + event = "error", + "Failed to read DMARC report: {}", + err + ); + continue; + } + }; + let _ = serde::Serialize::serialize(&tls, &mut serialized_size); - self.main.push(Schedule { - due: deliver_at.to_instant(), - inner: e.key().clone(), - }); - let domain = e.key().domain_name().to_string(); - let path = core - .build_report_path( - ReportType::Tls(&domain), - policy_hash, - created, - event.interval, - ) - .await; - let v = e.insert(ReportType::Tls(ReportPath { - path: vec![ReportPolicy { - inner: path, - policy: policy_hash, - }], - size: 0, - created, - deliver_at: event.interval, - })); - (v.tls_path(), 0, domain.into()) + // Group duplicates + let mut total_success = 0; + let mut total_failure = 0; + + let from_key = + ValueKey::from(ValueClass::Queue(QueueClass::TlsReportEvent(ReportEvent { + due: event.due, + policy_hash: event.policy_hash, + seq_id: 0, + domain: event.domain.clone(), + }))); + let to_key = + ValueKey::from(ValueClass::Queue(QueueClass::TlsReportEvent(ReportEvent { + due: event.due, + policy_hash: event.policy_hash, + seq_id: u64::MAX, + domain: event.domain.clone(), + }))); + let mut record_map = AHashMap::with_capacity(tls.records.len()); + if let Err(err) = self + .shared + .default_data_store + .iterate(IterateParams::new(from_key, to_key).ascending(), |_, v| { + if let Some(failure_details) = + Bincode::>::deserialize(v)?.inner + { + total_failure += 1; + + match record_map.entry(failure_details) { + Entry::Occupied(mut e) => { + *e.get_mut() += 1; + Ok(true) + } + Entry::Vacant(e) => { + if serde::Serialize::serialize(e.key(), &mut serialized_size) + .is_ok() + { + e.insert(1u32); + Ok(true) + } else { + Ok(false) + } + } + } + } else { + total_success += 1; + Ok(true) + } + }) + .await + { + tracing::warn!( + parent: &span, + event = "error", + "Failed to read TLS report: {}", + err + ); + } + + report.policies.push(Policy { + policy: tls.policy, + summary: Summary { + total_success, + total_failure, + }, + failure_details: record_map + .into_iter() + .map(|(mut r, count)| { + r.failed_session_count = count; + r + }) + .collect(), + }); + + rua = tls.rua; + } + + if report.policies.is_empty() { + // This should not happen + tracing::warn!( + parent: &span, + event = "empty-report", + "No policies found in report" + ); + self.delete_tls_report(events).await; + return; + } + + // Compress and serialize report + let json = report.to_json(); + let mut e = GzEncoder::new(Vec::with_capacity(json.len()), Compression::default()); + let json = match std::io::Write::write_all(&mut e, json.as_bytes()).and_then(|_| e.finish()) + { + Ok(report) => report, + Err(err) => { + tracing::error!( + parent: &span, + event = "error", + "Failed to compress report: {}", + err + ); + self.delete_tls_report(events).await; + return; } }; - if let Some(domain) = create { + // Try delivering report over HTTP + let mut rcpts = Vec::with_capacity(rua.len()); + for uri in &rua { + match uri { + ReportUri::Http(uri) => { + if let Ok(client) = reqwest::blocking::Client::builder() + .user_agent(USER_AGENT) + .timeout(Duration::from_secs(2 * 60)) + .build() + { + #[cfg(feature = "test_mode")] + if uri == "https://127.0.0.1/tls" { + TLS_HTTP_REPORT.lock().extend_from_slice(&json); + self.delete_tls_report(events).await; + return; + } + + match client + .post(uri) + .header(CONTENT_TYPE, "application/tlsrpt+gzip") + .body(json.to_vec()) + .send() + { + Ok(response) => { + if response.status().is_success() { + tracing::info!( + parent: &span, + context = "http", + event = "success", + url = uri, + ); + self.delete_tls_report(events).await; + return; + } else { + tracing::debug!( + parent: &span, + context = "http", + event = "invalid-response", + url = uri, + status = %response.status() + ); + } + } + Err(err) => { + tracing::debug!( + parent: &span, + context = "http", + event = "error", + url = uri, + reason = %err + ); + } + } + } + } + ReportUri::Mail(mailto) => { + rcpts.push(mailto.as_str()); + } + } + } + + // Deliver report over SMTP + if !rcpts.is_empty() { + let from_addr = self + .eval_if(&config.address, &RecipientDomain::new(domain_name.as_str())) + .await + .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()); + let mut message = Vec::with_capacity(2048); + let _ = report.write_rfc5322_from_bytes( + &domain_name, + &self + .eval_if( + &self.report.config.submitter, + &RecipientDomain::new(domain_name.as_str()), + ) + .await + .unwrap_or_else(|| "localhost".to_string()), + ( + self.eval_if(&config.name, &RecipientDomain::new(domain_name.as_str())) + .await + .unwrap_or_else(|| "Mail Delivery Subsystem".to_string()) + .as_str(), + from_addr.as_str(), + ), + rcpts.iter().copied(), + &json, + &mut message, + ); + + // Send report + self.send_report( + &from_addr, + rcpts.iter(), + message, + &config.sign, + &span, + false, + ) + .await; + } else { + tracing::info!( + parent: &span, + event = "delivery-failed", + "No valid recipients found to deliver report to." + ); + } + self.delete_tls_report(events).await; + } + + pub async fn schedule_tls(&self, event: Box) { + let created = event.interval.to_timestamp(); + let deliver_at = created + event.interval.as_secs(); + let mut report_event = ReportEvent { + due: deliver_at, + policy_hash: event.policy.to_hash(), + seq_id: created, + domain: event.domain, + }; + + // Write policy if missing + let mut builder = BatchBuilder::new(); + if self + .shared + .default_data_store + .get_value::<()>(ValueKey::from(ValueClass::Queue( + QueueClass::TlsReportHeader(report_event.clone()), + ))) + .await + .unwrap_or_default() + .is_none() + { + // Serialize report let mut policy = PolicyDetails { policy_type: PolicyType::NoPolicyFound, policy_string: vec![], - policy_domain: domain, + policy_domain: report_event.domain.clone(), mx_host: vec![], }; @@ -420,47 +444,78 @@ impl Scheduler { let entry = TlsFormat { rua: event.tls_record.rua.clone(), policy, - records: vec![event.failure], + records: vec![], }; - let bytes_written = json_write(&path.path[pos].inner, &entry).await; - if bytes_written > 0 { - path.size += bytes_written; - } else { - // Something went wrong, remove record - if let Entry::Occupied(mut e) = self - .reports - .entry(ReportType::Tls(entry.policy.policy_domain)) - { - if let ReportType::Tls(path) = e.get_mut() { - path.path.retain(|p| p.policy != policy_hash); - if path.path.is_empty() { - e.remove_entry(); - } - } - } - } - } else if path.size < max_size { - // Append to existing report - path.size += - json_append(&path.path[pos].inner, &event.failure, max_size - path.size).await; + // Write report + builder.set( + ValueClass::Queue(QueueClass::TlsReportHeader(report_event.clone())), + Bincode::new(entry).serialize(), + ); + } + + // Write entry + report_event.seq_id = self.queue.snowflake_id.generate().unwrap_or_else(now); + builder.set( + ValueClass::Queue(QueueClass::TlsReportEvent(report_event)), + Bincode::new(event.failure).serialize(), + ); + + if let Err(err) = self.shared.default_data_store.write(builder.build()).await { + tracing::error!( + context = "report", + event = "error", + "Failed to write DMARC report event: {}", + err + ); } } -} -impl ReportPath>> { - fn cleanup_blocking(&self) { - for path in &self.path { - if let Err(err) = std::fs::remove_file(&path.inner) { - tracing::error!( + pub async fn delete_tls_report(&self, events: Vec) { + let mut batch = BatchBuilder::new(); + + for event in events { + let from_key = ReportEvent { + due: event.due, + policy_hash: event.policy_hash, + seq_id: 0, + domain: event.domain.clone(), + }; + let to_key = ReportEvent { + due: event.due, + policy_hash: event.policy_hash, + seq_id: u64::MAX, + domain: event.domain.clone(), + }; + + if let Err(err) = self + .shared + .default_data_store + .delete_range( + ValueKey::from(ValueClass::Queue(QueueClass::TlsReportEvent(from_key))), + ValueKey::from(ValueClass::Queue(QueueClass::TlsReportEvent(to_key))), + ) + .await + { + tracing::warn!( context = "report", - report = "tls", event = "error", - "Failed to delete file {}: {}", - path.inner.display(), + "Failed to remove repors: {}", err ); + return; } + + batch.clear(ValueClass::Queue(QueueClass::TlsReportHeader(event))); + } + + if let Err(err) = self.shared.default_data_store.write(batch.build()).await { + tracing::warn!( + context = "report", + event = "error", + "Failed to remove repors: {}", + err + ); } } } diff --git a/crates/smtp/src/scripts/event_loop.rs b/crates/smtp/src/scripts/event_loop.rs index 845430f7..4bfaefaf 100644 --- a/crates/smtp/src/scripts/event_loop.rs +++ b/crates/smtp/src/scripts/event_loop.rs @@ -21,7 +21,7 @@ * for more details. */ -use std::{sync::Arc, time::Duration}; +use std::sync::Arc; use mail_auth::common::headers::HeaderWriter; use sieve::{ @@ -32,18 +32,12 @@ use smtp_proto::{ MAIL_BY_TRACE, MAIL_RET_FULL, MAIL_RET_HDRS, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, }; -use store::{backend::memory::MemoryStore, LookupKey, LookupStore, LookupValue}; +use store::{backend::memory::MemoryStore, LookupStore}; use tokio::runtime::Handle; -use crate::{ - core::SMTP, - queue::{DomainPart, InstantFromTimestamp, Message}, -}; +use crate::{core::SMTP, queue::DomainPart}; -use super::{ - plugins::{lookup::VariableExists, PluginContext}, - ScriptModification, ScriptParameters, ScriptResult, -}; +use super::{plugins::PluginContext, ScriptModification, ScriptParameters, ScriptResult}; impl SMTP { pub fn run_script_blocking( @@ -97,15 +91,15 @@ impl SMTP { 'outer: for list in lists { if let Some(store) = self.shared.lookup_stores.get(&list) { for value in &values { - if let Ok(LookupValue::Value { .. }) = handle.block_on( - store.key_get::(LookupKey::Key( + if let Ok(true) = handle.block_on( + store.key_exists( if !matches!(match_as, MatchAs::Lowercase) { value.clone() } else { value.to_lowercase() } .into_bytes(), - )), + ), ) { input = true.into(); break 'outer; @@ -156,7 +150,7 @@ impl SMTP { // Build message let return_path_lcase = self.sieve.return_path.to_lowercase(); let return_path_domain = return_path_lcase.domain_part().to_string(); - let mut message = Message::new_boxed( + let mut message = self.queue.new_message( self.sieve.return_path.clone(), return_path_lcase, return_path_domain, @@ -223,7 +217,6 @@ impl SMTP { if trace { message.flags |= MAIL_BY_TRACE; } - let rlimit = Duration::from_secs(rlimit); match mode { ByMode::Notify => { for domain in &mut message.domains { @@ -246,16 +239,15 @@ impl SMTP { if trace { message.flags |= MAIL_BY_TRACE; } - let alimit = (alimit as u64).to_instant(); match mode { ByMode::Notify => { for domain in &mut message.domains { - domain.notify.due = alimit; + domain.notify.due = alimit as u64; } } ByMode::Return => { for domain in &mut message.domains { - domain.expires = alimit; + domain.expires = alimit as u64; } } ByMode::Default => (), @@ -302,10 +294,10 @@ impl SMTP { None }; - handle.block_on(self.queue.queue_message( - message, + handle.block_on(message.queue( headers.as_deref(), raw_message, + self, &span, )); } diff --git a/crates/smtp/src/scripts/plugins/bayes.rs b/crates/smtp/src/scripts/plugins/bayes.rs index ab5a147b..8b035ceb 100644 --- a/crates/smtp/src/scripts/plugins/bayes.rs +++ b/crates/smtp/src/scripts/plugins/bayes.rs @@ -29,12 +29,12 @@ use nlp::{ tokenizers::osb::{OsbToken, OsbTokenizer}, }; use sieve::{runtime::Variable, FunctionMap}; -use store::{write::key::KeySerializer, LookupKey, LookupStore, LookupValue, U64_LEN}; +use store::{write::key::KeySerializer, LookupStore, U64_LEN}; use tokio::runtime::Handle; use crate::config::scripts::SieveContext; -use super::{lookup::VariableExists, PluginContext}; +use super::PluginContext; pub fn register_train(plugin_id: u32, fnc_map: &mut FunctionMap) { fnc_map.set_external_function("bayes_train", plugin_id, 3); @@ -110,14 +110,13 @@ fn train(ctx: PluginContext<'_>, is_train: bool) -> Variable { for (hash, weights) in model.weights { if handle .block_on( - store.key_set( + store.counter_incr( KeySerializer::new(U64_LEN) .write(hash.h1) .write(hash.h2) .finalize(), - LookupValue::Counter { - num: weights.into(), - }, + weights.into(), + None, ), ) .is_err() @@ -135,14 +134,13 @@ fn train(ctx: PluginContext<'_>, is_train: bool) -> Variable { }; if handle .block_on( - store.key_set( + store.counter_incr( KeySerializer::new(U64_LEN) .write(0u64) .write(0u64) .finalize(), - LookupValue::Counter { - num: weights.into(), - }, + weights.into(), + None, ), ) .is_err() @@ -337,15 +335,15 @@ impl LookupOrInsert for BayesTokenCache { ) -> Option { if let Some(weights) = self.get(&hash) { weights.unwrap_or_default().into() - } else if let Ok(result) = handle.block_on( - get_token.key_get::(LookupKey::Counter( + } else if let Ok(num) = handle.block_on( + get_token.counter_get( KeySerializer::new(U64_LEN) .write(hash.h1) .write(hash.h2) .finalize(), - )), + ), ) { - if let LookupValue::Counter { num } = result { + if num != 0 { let weights = Weights::from(num); self.insert_positive(hash, weights); weights diff --git a/crates/smtp/src/scripts/plugins/lookup.rs b/crates/smtp/src/scripts/plugins/lookup.rs index 27978d8a..913a761b 100644 --- a/crates/smtp/src/scripts/plugins/lookup.rs +++ b/crates/smtp/src/scripts/plugins/lookup.rs @@ -29,7 +29,7 @@ use std::{ use mail_auth::flate2; use sieve::{runtime::Variable, FunctionMap}; -use store::{Deserialize, LookupKey, LookupValue, Value}; +use store::{Deserialize, Value}; use crate::{ config::scripts::{RemoteList, SieveContext}, @@ -72,10 +72,7 @@ pub fn exec(ctx: PluginContext<'_>) -> Variable { if !item.is_empty() && ctx .handle - .block_on(store.key_get::(LookupKey::Key( - item.to_string().into_owned().into_bytes(), - ))) - .map(|v| v != LookupValue::None) + .block_on(store.key_exists(item.to_string().into_owned().into_bytes())) .unwrap_or(false) { return true.into(); @@ -85,10 +82,7 @@ pub fn exec(ctx: PluginContext<'_>) -> Variable { } v if !v.is_empty() => ctx .handle - .block_on(store.key_get::(LookupKey::Key( - v.to_string().into_owned().into_bytes(), - ))) - .map(|v| v != LookupValue::None) + .block_on(store.key_exists(v.to_string().into_owned().into_bytes())) .unwrap_or(false), _ => false, } @@ -113,14 +107,13 @@ pub fn exec_get(ctx: PluginContext<'_>) -> Variable { if let Some(store) = store { ctx.handle - .block_on(store.key_get::(LookupKey::Key( - ctx.arguments[1].to_string().into_owned().into_bytes(), - ))) - .map(|v| match v { - LookupValue::Value { value, .. } => value.into_inner(), - LookupValue::Counter { num } => num.into(), - LookupValue::None => Variable::default(), - }) + .block_on( + store.key_get::( + ctx.arguments[1].to_string().into_owned().into_bytes(), + ), + ) + .unwrap_or_default() + .map(|v| v.into_inner()) .unwrap_or_default() } else { tracing::warn!( @@ -142,22 +135,20 @@ pub fn exec_set(ctx: PluginContext<'_>) -> Variable { if let Some(store) = store { let expires = match &ctx.arguments[3] { - Variable::Integer(v) => *v as u64, - Variable::Float(v) => *v as u64, - _ => 0, + Variable::Integer(v) => Some(*v as u64), + Variable::Float(v) => Some(*v as u64), + _ => None, }; ctx.handle .block_on(store.key_set( ctx.arguments[1].to_string().into_owned().into_bytes(), - LookupValue::Value { - value: if !ctx.arguments[2].is_empty() { - bincode::serialize(&ctx.arguments[2]).unwrap_or_default() - } else { - vec![] - }, - expires, + if !ctx.arguments[2].is_empty() { + bincode::serialize(&ctx.arguments[2]).unwrap_or_default() + } else { + vec![] }, + expires, )) .is_ok() .into() @@ -426,9 +417,6 @@ pub fn exec_local_domain(ctx: PluginContext<'_>) -> Variable { #[derive(Debug, PartialEq, Eq)] pub struct VariableWrapper(Variable); -#[derive(Debug, PartialEq, Eq)] -pub struct VariableExists; - impl Deserialize for VariableWrapper { fn deserialize(bytes: &[u8]) -> store::Result { Ok(VariableWrapper( @@ -439,9 +427,9 @@ impl Deserialize for VariableWrapper { } } -impl Deserialize for VariableExists { - fn deserialize(_: &[u8]) -> store::Result { - Ok(VariableExists) +impl From for VariableWrapper { + fn from(value: i64) -> Self { + VariableWrapper(value.into()) } } @@ -451,12 +439,6 @@ impl VariableWrapper { } } -impl From> for VariableExists { - fn from(_: Value<'static>) -> Self { - VariableExists - } -} - impl From> for VariableWrapper { fn from(value: Value<'static>) -> Self { VariableWrapper(into_sieve_value(value)) diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index d550d677..f53ff273 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -44,6 +44,7 @@ flate2 = "1.0" async-trait = "0.1.68" redis = { version = "0.24.0", features = [ "tokio-comp", "tokio-rustls-comp", "tls-rustls-insecure", "tls-rustls-webpki-roots", "cluster-async"], optional = true } deadpool = { version = "0.10.0", features = ["managed"], optional = true } +bincode = "1.3.3" [dev-dependencies] tokio = { version = "1.23", features = ["full"] } diff --git a/crates/store/src/backend/redis/lookup.rs b/crates/store/src/backend/redis/lookup.rs index 493ad4e7..431405fa 100644 --- a/crates/store/src/backend/redis/lookup.rs +++ b/crates/store/src/backend/redis/lookup.rs @@ -23,68 +23,145 @@ use redis::AsyncCommands; -use crate::{Deserialize, LookupKey, LookupValue}; +use crate::Deserialize; use super::{RedisPool, RedisStore}; impl RedisStore { - pub async fn key_set(&self, key: Vec, value: LookupValue>) -> crate::Result<()> { + pub async fn key_set( + &self, + key: Vec, + value: Vec, + expires: Option, + ) -> crate::Result<()> { match &self.pool { - RedisPool::Single(pool) => self.key_set_(pool.get().await?.as_mut(), key, value).await, - RedisPool::Cluster(pool) => self.key_set_(pool.get().await?.as_mut(), key, value).await, + RedisPool::Single(pool) => { + self.key_set_(pool.get().await?.as_mut(), key, value, expires) + .await + } + RedisPool::Cluster(pool) => { + self.key_set_(pool.get().await?.as_mut(), key, value, expires) + .await + } + } + } + + pub async fn key_incr( + &self, + key: Vec, + value: i64, + expires: Option, + ) -> crate::Result { + match &self.pool { + RedisPool::Single(pool) => { + self.key_incr_(pool.get().await?.as_mut(), key, value, expires) + .await + } + RedisPool::Cluster(pool) => { + self.key_incr_(pool.get().await?.as_mut(), key, value, expires) + .await + } + } + } + + pub async fn key_delete(&self, key: Vec) -> crate::Result<()> { + match &self.pool { + RedisPool::Single(pool) => self.key_delete_(pool.get().await?.as_mut(), key).await, + RedisPool::Cluster(pool) => self.key_delete_(pool.get().await?.as_mut(), key).await, } } pub async fn key_get( &self, - key: LookupKey, - ) -> crate::Result> { + key: Vec, + ) -> crate::Result> { match &self.pool { RedisPool::Single(pool) => self.key_get_(pool.get().await?.as_mut(), key).await, RedisPool::Cluster(pool) => self.key_get_(pool.get().await?.as_mut(), key).await, } } + pub async fn counter_get(&self, key: Vec) -> crate::Result { + match &self.pool { + RedisPool::Single(pool) => self.counter_get_(pool.get().await?.as_mut(), key).await, + RedisPool::Cluster(pool) => self.counter_get_(pool.get().await?.as_mut(), key).await, + } + } + + pub async fn key_exists(&self, key: Vec) -> crate::Result { + match &self.pool { + RedisPool::Single(pool) => self.key_exists_(pool.get().await?.as_mut(), key).await, + RedisPool::Cluster(pool) => self.key_exists_(pool.get().await?.as_mut(), key).await, + } + } + async fn key_get_( &self, conn: &mut impl AsyncCommands, - key: LookupKey, - ) -> crate::Result> { - match key { - LookupKey::Key(key) => { - if let Some(value) = conn.get::<_, Option>>(key).await? { - T::deserialize(&value).map(|value| LookupValue::Value { value, expires: 0 }) - } else { - Ok(LookupValue::None) - } - } - LookupKey::Counter(key) => { - let value: Option = conn.get(key).await?; - Ok(LookupValue::Counter { - num: value.unwrap_or(0), - }) - } + key: Vec, + ) -> crate::Result> { + if let Some(value) = conn.get::<_, Option>>(key).await? { + T::deserialize(&value).map(Some) + } else { + Ok(None) } } + async fn counter_get_( + &self, + conn: &mut impl AsyncCommands, + key: Vec, + ) -> crate::Result { + conn.get::<_, Option>(key) + .await + .map(|x| x.unwrap_or(0)) + .map_err(Into::into) + } + + async fn key_exists_( + &self, + conn: &mut impl AsyncCommands, + key: Vec, + ) -> crate::Result { + conn.exists(key).await.map_err(Into::into) + } + async fn key_set_( &self, conn: &mut impl AsyncCommands, key: Vec, - value: LookupValue>, + value: Vec, + expires: Option, ) -> crate::Result<()> { - match value { - LookupValue::Value { value, expires } => { - if expires > 0 { - conn.set_ex(key, value, expires).await?; - } else { - conn.set(key, value).await?; - } - } - LookupValue::Counter { num } => conn.incr(key, num).await?, - LookupValue::None => (), + if let Some(expires) = expires { + conn.set_ex(key, value, expires).await.map_err(Into::into) + } else { + conn.set(key, value).await.map_err(Into::into) } + } - Ok(()) + async fn key_incr_( + &self, + conn: &mut impl AsyncCommands, + key: Vec, + value: i64, + expires: Option, + ) -> crate::Result { + if let Some(expires) = expires { + redis::pipe() + .atomic() + .incr(&key, value) + .expire(&key, expires as i64) + .ignore() + .query_async(conn) + .await + .map_err(Into::into) + } else { + conn.incr(&key, value).await.map_err(Into::into) + } + } + + async fn key_delete_(&self, conn: &mut impl AsyncCommands, key: Vec) -> crate::Result<()> { + conn.del(key).await.map_err(Into::into) } } diff --git a/crates/store/src/dispatch/lookup.rs b/crates/store/src/dispatch/lookup.rs index 004f8484..74ae894f 100644 --- a/crates/store/src/dispatch/lookup.rs +++ b/crates/store/src/dispatch/lookup.rs @@ -21,17 +21,16 @@ * for more details. */ -use utils::expr; +use utils::{config::Rate, expr}; -use crate::{backend::memory::MemoryStore, Row}; +use crate::{backend::memory::MemoryStore, write::LookupClass, Row}; #[allow(unused_imports)] use crate::{ write::{ key::{DeserializeBigEndian, KeySerializer}, now, BatchBuilder, Operation, ValueClass, ValueOp, }, - Deserialize, IterateParams, LookupKey, LookupStore, LookupValue, QueryResult, Store, Value, - ValueKey, U64_LEN, + Deserialize, IterateParams, LookupStore, QueryResult, Store, Value, ValueKey, U64_LEN, }; impl LookupStore { @@ -59,33 +58,28 @@ impl LookupStore { result } - pub async fn key_set(&self, key: Vec, value: LookupValue>) -> crate::Result<()> { + pub async fn key_set( + &self, + key: Vec, + value: Vec, + expires: Option, + ) -> crate::Result<()> { match self { LookupStore::Store(store) => { - let (class, op) = match value { - LookupValue::Value { value, expires } => ( - ValueClass::Key(key), - ValueOp::Set( - KeySerializer::new(value.len() + U64_LEN) - .write(if expires > 0 { - now() + expires - } else { - u64::MAX - }) - .write(value.as_slice()) - .finalize(), - ), - ), - LookupValue::Counter { num } => (ValueClass::Key(key), ValueOp::Add(num)), - LookupValue::None => return Ok(()), - }; - let mut batch = BatchBuilder::new(); - batch.ops.push(Operation::Value { class, op }); + batch.ops.push(Operation::Value { + class: ValueClass::Lookup(LookupClass::Key(key)), + op: ValueOp::Set( + KeySerializer::new(value.len() + U64_LEN) + .write(expires.map_or(u64::MAX, |expires| now() + expires)) + .write(value.as_slice()) + .finalize(), + ), + }); store.write(batch.build()).await } #[cfg(feature = "redis")] - LookupStore::Redis(store) => store.key_set(key, value).await, + LookupStore::Redis(store) => store.key_set(key, value, expires).await, LookupStore::Query(lookup) => lookup .store .query::( @@ -100,83 +94,209 @@ impl LookupStore { } } + pub async fn counter_incr( + &self, + key: Vec, + value: i64, + expires: Option, + ) -> crate::Result { + match self { + LookupStore::Store(store) => { + let mut batch = BatchBuilder::new(); + + if let Some(expires) = expires { + batch.ops.push(Operation::Value { + class: ValueClass::Lookup(LookupClass::CounterExpiry(key.clone())), + op: ValueOp::Set( + KeySerializer::new(U64_LEN) + .write(now() + expires) + .finalize(), + ), + }); + } + + batch.ops.push(Operation::Value { + class: ValueClass::Lookup(LookupClass::Counter(key)), + op: ValueOp::Add(value), + }); + + store.write(batch.build()).await?; + + Ok(0) + } + #[cfg(feature = "redis")] + LookupStore::Redis(store) => store.key_incr(key, value, expires).await, + LookupStore::Query(_) | LookupStore::Memory(_) => Err(crate::Error::InternalError( + "This store does not support counter_incr".into(), + )), + } + } + + pub async fn key_delete(&self, key: Vec) -> crate::Result<()> { + match self { + LookupStore::Store(store) => { + let mut batch = BatchBuilder::new(); + batch.ops.push(Operation::Value { + class: ValueClass::Lookup(LookupClass::Key(key)), + op: ValueOp::Clear, + }); + store.write(batch.build()).await + } + #[cfg(feature = "redis")] + LookupStore::Redis(store) => store.key_delete(key).await, + LookupStore::Query(_) | LookupStore::Memory(_) => Err(crate::Error::InternalError( + "This store does not support key_set".into(), + )), + } + } + + pub async fn counter_delete(&self, key: Vec) -> crate::Result<()> { + match self { + LookupStore::Store(store) => { + let mut batch = BatchBuilder::new(); + batch.ops.push(Operation::Value { + class: ValueClass::Lookup(LookupClass::Counter(key)), + op: ValueOp::Clear, + }); + store.write(batch.build()).await + } + #[cfg(feature = "redis")] + LookupStore::Redis(store) => store.key_delete(key).await, + LookupStore::Query(_) | LookupStore::Memory(_) => Err(crate::Error::InternalError( + "This store does not support key_set".into(), + )), + } + } + pub async fn key_get> + std::fmt::Debug + 'static>( &self, - key: LookupKey, - ) -> crate::Result> { + key: Vec, + ) -> crate::Result> { match self { - LookupStore::Store(store) => match key { - LookupKey::Key(key) => store - .get_value::>(ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::Key(key), - }) - .await - .map(|value| value.unwrap_or(LookupValue::None)), - LookupKey::Counter(key) => store - .get_counter(ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::Key(key), - }) - .await - .map(|num| LookupValue::Counter { num }), - }, + LookupStore::Store(store) => store + .get_value::>(ValueKey::from(ValueClass::Lookup(LookupClass::Key( + key, + )))) + .await + .map(|value| value.and_then(|v| v.into())), #[cfg(feature = "redis")] LookupStore::Redis(store) => store.key_get(key).await, LookupStore::Memory(store) => { - let key = String::from(key); + let key = String::from_utf8(key).unwrap_or_default(); match store.as_ref() { MemoryStore::List(list) => Ok(if list.contains(&key) { - LookupValue::Value { - value: T::from(Value::Bool(true)), - expires: 0, - } + Some(T::from(Value::Bool(true))) } else { - LookupValue::None + None }), - MemoryStore::Map(map) => Ok(map - .get(&key) - .map(|value| LookupValue::Value { - value: T::from(value.to_owned()), - expires: 0, - }) - .unwrap_or(LookupValue::None)), + MemoryStore::Map(map) => { + Ok(map.get(&key).map(|value| T::from(value.to_owned()))) + } } } LookupStore::Query(lookup) => lookup .store - .query::>(&lookup.query, vec![String::from(key).into()]) + .query::>( + &lookup.query, + vec![String::from_utf8(key).unwrap_or_default().into()], + ) .await .map(|row| { row.and_then(|row| row.values.into_iter().next()) - .map(|value| LookupValue::Value { - value: T::from(value), - expires: 0, - }) - .unwrap_or(LookupValue::None) + .map(|value| T::from(value)) }), } } + pub async fn counter_get(&self, key: Vec) -> crate::Result { + match self { + LookupStore::Store(store) => { + store + .get_counter(ValueKey::from(ValueClass::Lookup(LookupClass::Counter( + key, + )))) + .await + } + #[cfg(feature = "redis")] + LookupStore::Redis(store) => store.counter_get(key).await, + LookupStore::Query(_) | LookupStore::Memory(_) => Err(crate::Error::InternalError( + "This store does not support counter_get".into(), + )), + } + } + + pub async fn key_exists(&self, key: Vec) -> crate::Result { + match self { + LookupStore::Store(store) => store + .get_value::>(ValueKey::from(ValueClass::Lookup(LookupClass::Key( + key, + )))) + .await + .map(|value| matches!(value, Some(LookupValue::Value(())))), + #[cfg(feature = "redis")] + LookupStore::Redis(store) => store.key_exists(key).await, + LookupStore::Memory(store) => { + let key = String::from_utf8(key).unwrap_or_default(); + match store.as_ref() { + MemoryStore::List(list) => Ok(list.contains(&key)), + MemoryStore::Map(map) => Ok(map.contains_key(&key)), + } + } + LookupStore::Query(lookup) => lookup + .store + .query::>( + &lookup.query, + vec![String::from_utf8(key).unwrap_or_default().into()], + ) + .await + .map(|row| row.is_some()), + } + } + + pub async fn is_rate_allowed( + &self, + key: &[u8], + rate: &Rate, + soft_check: bool, + ) -> crate::Result> { + let now = now(); + let range_start = now / rate.period.as_secs(); + let range_end = (range_start * rate.period.as_secs()) + rate.period.as_secs(); + let expires_in = range_end - now; + + let mut bucket = Vec::with_capacity(key.len() + U64_LEN); + bucket.extend_from_slice(key); + bucket.extend_from_slice(range_start.to_be_bytes().as_slice()); + + let requests = if !soft_check { + let requests = self.counter_incr(bucket, 1, expires_in.into()).await?; + if requests > 0 { + requests - 1 + } else { + // Increment and get not supported by store, fetch counter + let mut bucket = Vec::with_capacity(key.len() + U64_LEN); + bucket.extend_from_slice(key); + bucket.extend_from_slice(range_start.to_be_bytes().as_slice()); + self.counter_get(bucket).await?.saturating_sub(1) + } + } else { + self.counter_get(bucket).await? + }; + + if requests < rate.requests as i64 { + Ok(None) + } else { + Ok(Some(expires_in)) + } + } + pub async fn purge_expired(&self) -> crate::Result<()> { match self { LookupStore::Store(store) => { - let from_key = ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::Key(vec![0u8]), - }; - let to_key = ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::Key(vec![u8::MAX; 10]), - }; + // Delete expired keys + let from_key = ValueKey::from(ValueClass::Lookup(LookupClass::Key(vec![0u8]))); + let to_key = + ValueKey::from(ValueClass::Lookup(LookupClass::Key(vec![u8::MAX; 10]))); let current_time = now(); let mut expired_keys = Vec::new(); @@ -192,7 +312,46 @@ impl LookupStore { let mut batch = BatchBuilder::new(); for key in expired_keys { batch.ops.push(Operation::Value { - class: ValueClass::Key(key), + class: ValueClass::Lookup(LookupClass::Key(key)), + op: ValueOp::Clear, + }); + if batch.ops.len() >= 1000 { + store.write(batch.build()).await?; + batch = BatchBuilder::new(); + } + } + if !batch.ops.is_empty() { + store.write(batch.build()).await?; + } + } + + // Delete expired counters + let from_key = + ValueKey::from(ValueClass::Lookup(LookupClass::CounterExpiry(vec![0u8]))); + let to_key = ValueKey::from(ValueClass::Lookup(LookupClass::CounterExpiry(vec![ + u8::MAX; + 10 + ]))); + + let current_time = now(); + let mut expired_keys = Vec::new(); + store + .iterate(IterateParams::new(from_key, to_key), |key, value| { + if value.deserialize_be_u64(0)? < current_time { + expired_keys.push(key.get(1..).unwrap_or_default().to_vec()); + } + Ok(true) + }) + .await?; + if !expired_keys.is_empty() { + let mut batch = BatchBuilder::new(); + for key in expired_keys { + batch.ops.push(Operation::Value { + class: ValueClass::Lookup(LookupClass::Counter(key.clone())), + op: ValueOp::Clear, + }); + batch.ops.push(Operation::Value { + class: ValueClass::Lookup(LookupClass::CounterExpiry(key)), op: ValueOp::Clear, }); if batch.ops.len() >= 1000 { @@ -214,14 +373,16 @@ impl LookupStore { } } +enum LookupValue { + Value(T), + None, +} + impl Deserialize for LookupValue { fn deserialize(bytes: &[u8]) -> crate::Result { bytes.deserialize_be_u64(0).and_then(|expires| { Ok(if expires > now() { - LookupValue::Value { - value: T::deserialize(bytes.get(U64_LEN..).unwrap_or_default())?, - expires, - } + LookupValue::Value(T::deserialize(bytes.get(U64_LEN..).unwrap_or_default())?) } else { LookupValue::None }) @@ -229,6 +390,15 @@ impl Deserialize for LookupValue { } } +impl From> for Option { + fn from(value: LookupValue) -> Self { + match value { + LookupValue::Value(value) => Some(value), + LookupValue::None => None, + } + } +} + impl From> for String { fn from(value: Value<'static>) -> Self { match value { diff --git a/crates/store/src/dispatch/store.rs b/crates/store/src/dispatch/store.rs index 9d5b4f25..c2928624 100644 --- a/crates/store/src/dispatch/store.rs +++ b/crates/store/src/dispatch/store.rs @@ -255,7 +255,8 @@ impl Store { Self::RocksDb(store) => store.purge_bitmaps().await, } } - pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> crate::Result<()> { + + pub async fn delete_range(&self, from: impl Key, to: impl Key) -> crate::Result<()> { match self { #[cfg(feature = "sqlite")] Self::SQLite(store) => store.delete_range(from, to).await, @@ -395,9 +396,11 @@ impl Store { #[cfg(feature = "test_mode")] pub async fn blob_expire_all(&self) { + use utils::{BlobHash, BLOB_HASH_LEN}; + use crate::{ write::{key::DeserializeBigEndian, BatchBuilder, BlobOp, Operation, ValueOp}, - BlobHash, BLOB_HASH_LEN, U64_LEN, + U64_LEN, }; // Delete all temporary hashes diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 96e6ea40..33101bda 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -119,13 +119,9 @@ pub struct LogKey { pub change_id: u64, } -pub const BLOB_HASH_LEN: usize = 32; pub const U64_LEN: usize = std::mem::size_of::(); pub const U32_LEN: usize = std::mem::size_of::(); -#[derive(Clone, Debug, Default, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] -pub struct BlobHash([u8; BLOB_HASH_LEN]); - pub type Result = std::result::Result; #[derive(Clone, Debug, PartialEq, Eq, Hash)] @@ -325,19 +321,6 @@ impl From for LookupStore { } } -#[derive(Clone, Debug)] -pub enum LookupKey { - Key(Vec), - Counter(Vec), -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum LookupValue { - Value { value: T, expires: u64 }, - Counter { num: i64 }, - None, -} - #[derive(Clone, Debug, PartialEq)] pub enum Value<'x> { Integer(i64), @@ -363,16 +346,6 @@ impl<'x> Value<'x> { } } -impl From for String { - fn from(value: LookupKey) -> Self { - let key = match value { - LookupKey::Key(key) | LookupKey::Counter(key) => key, - }; - String::from_utf8(key) - .unwrap_or_else(|err| String::from_utf8_lossy(&err.into_bytes()).into_owned()) - } -} - #[derive(Clone, Debug)] pub struct Row { pub values: Vec>, @@ -558,6 +531,16 @@ impl<'x> From for Value<'x> { } } +impl From> for i64 { + fn from(value: Value<'static>) -> Self { + if let Value::Integer(value) = value { + value + } else { + 0 + } + } +} + impl<'x> From for Value<'x> { fn from(value: u64) -> Self { Self::Integer(value as i64) diff --git a/crates/store/src/write/blob.rs b/crates/store/src/write/blob.rs index 9e385728..8f78218e 100644 --- a/crates/store/src/write/blob.rs +++ b/crates/store/src/write/blob.rs @@ -22,10 +22,11 @@ */ use ahash::AHashSet; +use utils::{BlobHash, BLOB_HASH_LEN}; use crate::{ - write::BatchBuilder, BlobClass, BlobHash, BlobStore, Deserialize, IterateParams, Store, - ValueKey, BLOB_HASH_LEN, U32_LEN, U64_LEN, + write::BatchBuilder, BlobClass, BlobStore, Deserialize, IterateParams, Store, ValueKey, + U32_LEN, U64_LEN, }; use super::{key::DeserializeBigEndian, now, BlobOp, Operation, ValueClass, ValueOp}; diff --git a/crates/store/src/write/key.rs b/crates/store/src/write/key.rs index 89e9cd3e..1aa586dc 100644 --- a/crates/store/src/write/key.rs +++ b/crates/store/src/write/key.rs @@ -22,15 +22,18 @@ */ use std::convert::TryInto; -use utils::codec::leb128::Leb128_; +use utils::{codec::leb128::Leb128_, BLOB_HASH_LEN}; use crate::{ - BitmapKey, IndexKey, IndexKeyPrefix, Key, LogKey, ValueKey, BLOB_HASH_LEN, SUBSPACE_BITMAPS, - SUBSPACE_INDEXES, SUBSPACE_LOGS, SUBSPACE_VALUES, U32_LEN, U64_LEN, WITHOUT_BLOCK_NUM, - WITH_SUBSPACE, + BitmapKey, Deserialize, IndexKey, IndexKeyPrefix, Key, LogKey, ValueKey, SUBSPACE_BITMAPS, + SUBSPACE_COUNTERS, SUBSPACE_INDEXES, SUBSPACE_LOGS, SUBSPACE_VALUES, U32_LEN, U64_LEN, + WITHOUT_BLOCK_NUM, WITH_SUBSPACE, }; -use super::{AnyKey, BitmapClass, BlobOp, DirectoryClass, TagValue, ValueClass}; +use super::{ + AnyKey, BitmapClass, BlobOp, DirectoryClass, LookupClass, QueueClass, ReportEvent, TagValue, + ValueClass, +}; pub struct KeySerializer { pub buf: Vec, @@ -217,7 +220,16 @@ impl Key for LogKey { impl + Sync + Send> Key for ValueKey { fn subspace(&self) -> u8 { - SUBSPACE_VALUES + if !matches!( + self.class.as_ref(), + ValueClass::Directory(DirectoryClass::UsedQuota(_)) + | ValueClass::Lookup(LookupClass::Counter(_)) + | ValueClass::Queue(QueueClass::QuotaCount(_) | QueueClass::QuotaSize(_)) + ) { + SUBSPACE_VALUES + } else { + SUBSPACE_COUNTERS + } } fn serialize(&self, flags: u32) -> Vec { @@ -250,7 +262,6 @@ impl + Sync + Send> Key for ValueKey { .write(self.account_id) .write(self.collection) .write(self.document_id), - ValueClass::Key(key) => serializer.write(4u8).write(key.as_slice()), ValueClass::IndexEmail(seq) => serializer .write(5u8) .write(*seq) @@ -276,6 +287,11 @@ impl + Sync + Send> Key for ValueKey { .write(self.document_id), }, ValueClass::Config(key) => serializer.write(8u8).write(key.as_slice()), + ValueClass::Lookup(lookup) => match lookup { + LookupClass::Key(key) => serializer.write(4u8).write(key.as_slice()), + LookupClass::Counter(key) => serializer.write(9u8).write(key.as_slice()), + LookupClass::CounterExpiry(key) => serializer.write(10u8).write(key.as_slice()), + }, ValueClass::Directory(directory) => match directory { DirectoryClass::NameToId(name) => serializer.write(20u8).write(name.as_slice()), DirectoryClass::EmailToId(email) => serializer.write(21u8).write(email.as_slice()), @@ -297,6 +313,41 @@ impl + Sync + Send> Key for ValueKey { .write(*principal_id) .write(*has_member), }, + ValueClass::Queue(queue) => match queue { + QueueClass::Message(queue_id) => serializer.write(50u8).write(*queue_id), + QueueClass::MessageEvent(event) => serializer + .write(51u8) + .write(event.due) + .write(event.queue_id), + QueueClass::DmarcReportHeader(event) => serializer + .write(52u8) + .write(event.due) + .write(event.domain.as_bytes()) + .write(event.policy_hash) + .write(event.seq_id) + .write(0u8), + QueueClass::TlsReportHeader(event) => serializer + .write(52u8) + .write(event.due) + .write(event.domain.as_bytes()) + .write(event.policy_hash) + .write(event.seq_id) + .write(1u8), + QueueClass::DmarcReportEvent(event) => serializer + .write(53u8) + .write(event.due) + .write(event.domain.as_bytes()) + .write(event.policy_hash) + .write(event.seq_id), + QueueClass::TlsReportEvent(event) => serializer + .write(54u8) + .write(event.due) + .write(event.domain.as_bytes()) + .write(event.policy_hash) + .write(event.seq_id), + QueueClass::QuotaCount(key) => serializer.write(55u8).write(key.as_slice()), + QueueClass::QuotaSize(key) => serializer.write(56u8).write(key.as_slice()), + }, } .finalize() } @@ -425,7 +476,10 @@ impl ValueClass { U32_LEN * 2 + 3 } ValueClass::Acl(_) => U32_LEN * 3 + 2, - ValueClass::Key(v) | ValueClass::Config(v) => v.len(), + ValueClass::Lookup( + LookupClass::Counter(v) | LookupClass::CounterExpiry(v) | LookupClass::Key(v), + ) + | ValueClass::Config(v) => v.len(), ValueClass::Directory(d) => match d { DirectoryClass::NameToId(v) | DirectoryClass::EmailToId(v) @@ -438,6 +492,17 @@ impl ValueClass { BlobOp::Commit { .. } | BlobOp::Link { .. } => BLOB_HASH_LEN + U32_LEN * 2 + 2, }, ValueClass::IndexEmail { .. } => U64_LEN * 2, + ValueClass::Queue(q) => match q { + QueueClass::Message(_) => U64_LEN, + QueueClass::MessageEvent(_) => U64_LEN * 2, + QueueClass::DmarcReportEvent(event) | QueueClass::TlsReportEvent(event) => { + event.domain.len() + U64_LEN * 3 + } + QueueClass::DmarcReportHeader(event) | QueueClass::TlsReportHeader(event) => { + event.domain.len() + (U64_LEN * 3) + 1 + } + QueueClass::QuotaCount(v) | QueueClass::QuotaSize(v) => v.len(), + }, } } } @@ -475,3 +540,20 @@ impl From for ValueClass { ValueClass::Blob(value) } } + +impl Deserialize for ReportEvent { + fn deserialize(key: &[u8]) -> crate::Result { + Ok(ReportEvent { + due: key.deserialize_be_u64(1)?, + policy_hash: key.deserialize_be_u64(key.len() - (U64_LEN * 2 + 1))?, + seq_id: key.deserialize_be_u64(key.len() - (U64_LEN + 1))?, + domain: key + .get(U64_LEN + 1..key.len() - (U64_LEN * 2 + 1)) + .and_then(|domain| std::str::from_utf8(domain).ok()) + .map(|s| s.to_string()) + .ok_or_else(|| { + crate::Error::InternalError("Failed to deserialize report domain".into()) + })?, + }) + } +} diff --git a/crates/store/src/write/mod.rs b/crates/store/src/write/mod.rs index 47b2cadb..a08b514e 100644 --- a/crates/store/src/write/mod.rs +++ b/crates/store/src/write/mod.rs @@ -29,12 +29,13 @@ use std::{ }; use nlp::tokenizers::word::WordTokenizer; -use utils::codec::leb128::{Leb128Iterator, Leb128Vec}; - -use crate::{ - backend::MAX_TOKEN_LENGTH, BlobClass, BlobHash, Deserialize, Serialize, BLOB_HASH_LEN, +use utils::{ + codec::leb128::{Leb128Iterator, Leb128Vec}, + BlobHash, }; +use crate::{backend::MAX_TOKEN_LENGTH, BlobClass, Deserialize, Serialize}; + use self::assert::AssertValue; pub mod assert; @@ -131,13 +132,21 @@ pub enum TagValue { pub enum ValueClass { Property(u8), Acl(u32), - Key(Vec), + Lookup(LookupClass), TermIndex, ReservedId, Directory(DirectoryClass), Blob(BlobOp), IndexEmail(u64), Config(Vec), + Queue(QueueClass), +} + +#[derive(Debug, PartialEq, Clone, Eq, Hash)] +pub enum LookupClass { + Key(Vec), + Counter(Vec), + CounterExpiry(Vec), } #[derive(Debug, PartialEq, Clone, Eq, Hash)] @@ -151,6 +160,32 @@ pub enum DirectoryClass { UsedQuota(u32), } +#[derive(Debug, PartialEq, Clone, Eq, Hash)] +pub enum QueueClass { + Message(u64), + MessageEvent(QueueEvent), + DmarcReportHeader(ReportEvent), + DmarcReportEvent(ReportEvent), + TlsReportHeader(ReportEvent), + TlsReportEvent(ReportEvent), + QuotaCount(Vec), + QuotaSize(Vec), +} + +#[derive(Debug, PartialEq, Clone, Eq, Hash)] +pub struct QueueEvent { + pub due: u64, + pub queue_id: u64, +} + +#[derive(Debug, PartialEq, Clone, Eq, Hash)] +pub struct ReportEvent { + pub due: u64, + pub policy_hash: u64, + pub seq_id: u64, + pub domain: String, +} + #[derive(Debug, PartialEq, Eq, Hash, Default)] pub enum ValueOp { Set(Vec), @@ -264,6 +299,14 @@ impl Deserialize for u64 { } } +impl Deserialize for i64 { + fn deserialize(bytes: &[u8]) -> crate::Result { + Ok(i64::from_be_bytes(bytes.try_into().map_err(|_| { + crate::Error::InternalError("Failed to deserialize i64".to_string()) + })?)) + } +} + impl Deserialize for u32 { fn deserialize(bytes: &[u8]) -> crate::Result { Ok(u32::from_be_bytes(bytes.try_into().map_err(|_| { @@ -527,68 +570,12 @@ impl BitmapClass { } } -impl BlobHash { - pub fn new_max() -> Self { - BlobHash([u8::MAX; BLOB_HASH_LEN]) - } - - pub fn try_from_hash_slice(value: &[u8]) -> Result { - value.try_into().map(BlobHash) - } - - pub fn as_slice(&self) -> &[u8] { - self.0.as_ref() - } -} - -impl From<&[u8]> for BlobHash { - fn from(value: &[u8]) -> Self { - BlobHash(blake3::hash(value).into()) - } -} - -impl From> for BlobHash { - fn from(value: Vec) -> Self { - value.as_slice().into() - } -} - -impl From<&Vec> for BlobHash { - fn from(value: &Vec) -> Self { - value.as_slice().into() - } -} - -impl AsRef for BlobHash { - fn as_ref(&self) -> &BlobHash { - self - } -} - -impl AsRef<[u8]> for BlobHash { - fn as_ref(&self) -> &[u8] { - self.0.as_ref() - } -} - -impl AsMut<[u8]> for BlobHash { - fn as_mut(&mut self) -> &mut [u8] { - self.0.as_mut() - } -} - impl AsRef for BlobClass { fn as_ref(&self) -> &BlobClass { self } } -impl From for Vec { - fn from(value: BlobHash) -> Self { - value.0.to_vec() - } -} - impl BlobClass { pub fn account_id(&self) -> u32 { match self { @@ -605,3 +592,57 @@ impl BlobClass { } } } + +pub struct Bincode { + pub inner: T, +} + +impl Bincode { + pub fn new(inner: T) -> Self { + Self { inner } + } +} + +impl Serialize for &Bincode { + fn serialize(self) -> Vec { + lz4_flex::compress_prepend_size(&bincode::serialize(&self.inner).unwrap_or_default()) + } +} + +impl Serialize for Bincode { + fn serialize(self) -> Vec { + lz4_flex::compress_prepend_size(&bincode::serialize(&self.inner).unwrap_or_default()) + } +} + +impl Deserialize + for Bincode +{ + fn deserialize(bytes: &[u8]) -> crate::Result { + lz4_flex::decompress_size_prepended(bytes) + .map_err(|err| { + crate::Error::InternalError(format!("Bincode decompression failed: {err:?}")) + }) + .and_then(|result| { + bincode::deserialize(&result).map_err(|err| { + crate::Error::InternalError(format!( + "Bincode deserialization failed (len {}): {err:?}", + result.len() + )) + }) + }) + .map(|inner| Self { inner }) + } +} + +impl ToBitmaps for Bincode { + fn to_bitmaps(&self, _ops: &mut Vec, _field: u8, _set: bool) { + unreachable!() + } +} + +impl ToBitmaps for &Bincode { + fn to_bitmaps(&self, _ops: &mut Vec, _field: u8, _set: bool) { + unreachable!() + } +} diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index cd25bbb3..34cd574b 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -39,6 +39,7 @@ arc-swap = "1.6.0" futures = "0.3" proxy-header = { version = "0.1.0", features = ["tokio"] } regex = "1.7.0" +blake3 = "1.3.3" [target.'cfg(unix)'.dependencies] privdrop = "0.5.3" diff --git a/crates/utils/src/ipc.rs b/crates/utils/src/ipc.rs index 774776af..b36d810a 100644 --- a/crates/utils/src/ipc.rs +++ b/crates/utils/src/ipc.rs @@ -21,9 +21,11 @@ * for more details. */ -use std::{borrow::Cow, path::PathBuf}; +use std::borrow::Cow; -use tokio::{fs, io::AsyncReadExt, sync::oneshot}; +use tokio::sync::oneshot; + +use crate::BlobHash; #[derive(Debug)] pub enum DeliveryEvent { @@ -38,7 +40,7 @@ pub enum DeliveryEvent { pub struct IngestMessage { pub sender_address: String, pub recipients: Vec, - pub message_path: PathBuf, + pub message_blob: BlobHash, pub message_size: usize, } @@ -53,29 +55,3 @@ pub enum DeliveryResult { reason: Cow<'static, str>, }, } - -impl IngestMessage { - pub async fn read_message(&self) -> Result, ()> { - let mut raw_message = vec![0u8; self.message_size]; - let mut file = fs::File::open(&self.message_path).await.map_err(|err| { - tracing::error!( - context = "read_message", - event = "error", - "Failed to open message file {}: {}", - self.message_path.display(), - err - ); - })?; - file.read_exact(&mut raw_message).await.map_err(|err| { - tracing::error!( - context = "read_message", - event = "error", - "Failed to read {} bytes file {} from disk: {}", - self.message_size, - self.message_path.display(), - err - ); - })?; - Ok(raw_message) - } -} diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 961fac56..64e15067 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -50,6 +50,66 @@ use rustls_pki_types::TrustAnchor; use tracing_appender::non_blocking::WorkerGuard; use tracing_subscriber::{prelude::__tracing_subscriber_SubscriberExt, EnvFilter}; +pub const BLOB_HASH_LEN: usize = 32; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub struct BlobHash([u8; BLOB_HASH_LEN]); + +impl BlobHash { + pub fn new_max() -> Self { + BlobHash([u8::MAX; BLOB_HASH_LEN]) + } + + pub fn try_from_hash_slice(value: &[u8]) -> Result { + value.try_into().map(BlobHash) + } + + pub fn as_slice(&self) -> &[u8] { + self.0.as_ref() + } +} + +impl From<&[u8]> for BlobHash { + fn from(value: &[u8]) -> Self { + BlobHash(blake3::hash(value).into()) + } +} + +impl From> for BlobHash { + fn from(value: Vec) -> Self { + value.as_slice().into() + } +} + +impl From<&Vec> for BlobHash { + fn from(value: &Vec) -> Self { + value.as_slice().into() + } +} + +impl AsRef for BlobHash { + fn as_ref(&self) -> &BlobHash { + self + } +} + +impl From for Vec { + fn from(value: BlobHash) -> Self { + value.0.to_vec() + } +} + +impl AsRef<[u8]> for BlobHash { + fn as_ref(&self) -> &[u8] { + self.0.as_ref() + } +} + +impl AsMut<[u8]> for BlobHash { + fn as_mut(&mut self) -> &mut [u8] { + self.0.as_mut() + } +} pub trait UnwrapFailure { fn failed(self, action: &str) -> T; } diff --git a/tests/Cargo.toml b/tests/Cargo.toml index f5329a84..3f554ca4 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -6,7 +6,7 @@ resolver = "2" [features] #default = ["sqlite", "foundationdb", "postgres", "mysql", "rocks", "elastic", "s3", "redis"] -default = ["sqlite", "postgres", "mysql"] +default = ["sqlite", "postgres", "mysql", "redis"] sqlite = ["store/sqlite"] foundationdb = ["store/foundation"] postgres = ["store/postgres"] diff --git a/tests/resources/scripts/create_test_env.sh b/tests/resources/scripts/create_test_env.sh index 4ee7628e..fb95e781 100644 --- a/tests/resources/scripts/create_test_env.sh +++ b/tests/resources/scripts/create_test_env.sh @@ -7,14 +7,14 @@ DOMAIN="example.org" #STORE="foundationdb" #FTS_STORE="foundationdb" #BLOB_STORE="foundationdb" -STORE="rocksdb" -FTS_STORE="rocksdb" -BLOB_STORE="rocksdb" -#STORE="sqlite" -#FTS_STORE="sqlite" -#BLOB_STORE="sqlite" -FEATURES="foundationdb postgres mysql rocks elastic s3 redis" -#FEATURES="sqlite" +#STORE="rocksdb" +#FTS_STORE="rocksdb" +#BLOB_STORE="rocksdb" +STORE="sqlite" +FTS_STORE="sqlite" +BLOB_STORE="sqlite" +#FEATURES="foundationdb postgres mysql rocks elastic s3 redis" +FEATURES="sqlite" # Directories DIRECTORY="internal"