diff --git a/Cargo.lock b/Cargo.lock index 71224739..2bffcef4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -30,7 +30,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ "crypto-common", - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -110,9 +110,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45862d1c77f2228b9e10bc609d5bc203d86ebc9b87ad8d5d5167a6c9abf739d9" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android-tzdata" @@ -180,9 +180,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.93" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" +checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" [[package]] name = "arbitrary" @@ -284,7 +284,7 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", "synstructure 0.13.1", ] @@ -307,7 +307,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -353,7 +353,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -375,7 +375,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -386,7 +386,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -397,13 +397,13 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "attohttpc" -version = "0.28.0" +version = "0.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a13149d0cf3f7f9b9261fad4ec63b2efbf9a80665f52def86282d26255e6331" +checksum = "412b79ce053cef36eda52c25664b45ec92a21769488e20d5a8bf0b3c9e1a28cb" dependencies = [ - "http 1.1.0", + "http 1.2.0", "log", - "rustls 0.22.4", + "rustls 0.23.20", "serde", "serde_json", "url", @@ -452,7 +452,7 @@ dependencies = [ "axum-core", "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "itoa", @@ -464,7 +464,7 @@ dependencies = [ "rustversion", "serde", "sync_wrapper 1.0.2", - "tower 0.5.1", + "tower 0.5.2", "tower-layer", "tower-service", ] @@ -478,7 +478,7 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "mime", @@ -634,9 +634,9 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "bigdecimal" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f850665a0385e070b64c38d2354e6c104c8479c59868d1e48a0c13ee2c7a1c1" +checksum = "7f31f3af01c5c65a07985c804d3366560e6fa7883d640a122819b14ec327482c" dependencies = [ "autocfg", "libm", @@ -671,7 +671,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -691,7 +691,25 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.89", + "syn 2.0.90", +] + +[[package]] +name = "bindgen" +version = "0.71.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" +dependencies = [ + "bitflags 2.6.0", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "proc-macro2", + "quote", + "regex", + "rustc-hash 2.1.0", + "shlex", + "syn 2.0.90", ] [[package]] @@ -811,7 +829,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -820,7 +838,7 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -829,7 +847,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -873,7 +891,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -887,9 +905,9 @@ dependencies = [ [[package]] name = "buffered-reader" -version = "1.3.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd098763fdb64579407a8c83cf0d751e6d4a7e161d0114c89cc181a2ca760ec8" +checksum = "fabd1c5e55587a8e8526172d63ad2ba665fa18c8acb39ec9a77af1708c982b9b" dependencies = [ "lazy_static", "libc", @@ -937,9 +955,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" [[package]] name = "bzip2" @@ -992,9 +1010,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.1" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47" +checksum = "9157bbaa6b165880c27a4293a474c91cdcf265cc68cc829bf10be0964a391caf" dependencies = [ "jobserver", "libc", @@ -1053,9 +1071,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1072,7 +1090,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12f8e7987cbd042a63249497f41aed09f8e65add917ea6566effbc56578d6801" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -1099,9 +1117,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.21" +version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f" +checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" dependencies = [ "clap_builder", "clap_derive", @@ -1109,9 +1127,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.21" +version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec" +checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" dependencies = [ "anstream", "anstyle", @@ -1128,14 +1146,14 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] name = "clap_lex" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "cmac" @@ -1194,8 +1212,8 @@ dependencies = [ "dns-update", "futures", "hostname 0.4.0", - "hyper 1.5.1", - "idna 1.0.3", + "hyper 1.5.2", + "idna", "imagesize", "imap_proto", "infer 0.16.0", @@ -1225,7 +1243,7 @@ dependencies = [ "reqwest 0.12.9", "ring 0.17.8", "rsa", - "rustls 0.23.19", + "rustls 0.23.20", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", @@ -1236,7 +1254,7 @@ dependencies = [ "smtp-proto", "store", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "trc", "unicode-security", "utils", @@ -1257,15 +1275,15 @@ dependencies = [ [[package]] name = "console" -version = "0.15.8" +version = "0.15.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +checksum = "ea3c6ecd8059b57859df5c69830340ed3c41d30e3da0c1cbed90a96ac853041b" dependencies = [ - "encode_unicode 0.3.6", - "lazy_static", + "encode_unicode", "libc", - "unicode-width 0.1.14", - "windows-sys 0.52.0", + "once_cell", + "unicode-width 0.2.0", + "windows-sys 0.59.0", ] [[package]] @@ -1296,9 +1314,9 @@ dependencies = [ [[package]] name = "const_panic" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "013b6c2c3a14d678f38cd23994b02da3a1a1b6a5d1eedddfe63a5a5f11b13a81" +checksum = "53857514f72ee4a2b583de67401e3ff63a5472ca4acf289d09a9ea7636dfec17" [[package]] name = "constant_time_eq" @@ -1382,18 +1400,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.13" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -1410,18 +1428,18 @@ dependencies = [ [[package]] name = "crossbeam-queue" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" @@ -1435,7 +1453,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ - "generic-array", + "generic-array 0.14.7", "rand_core 0.6.4", "subtle", "zeroize", @@ -1447,7 +1465,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "generic-array", + "generic-array 0.14.7", "rand_core 0.6.4", "typenum", ] @@ -1458,7 +1476,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6" dependencies = [ - "generic-array", + "generic-array 0.14.7", "subtle", ] @@ -1516,7 +1534,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -1564,7 +1582,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -1586,7 +1604,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core 0.20.10", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -1615,7 +1633,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd2735a791158376708f9347fe8faba9667589d82427ef3aed6794a8981de3d9" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -1738,7 +1756,7 @@ checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -1759,7 +1777,7 @@ dependencies = [ "darling 0.20.10", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -1769,7 +1787,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -1787,7 +1805,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -1826,7 +1844,7 @@ dependencies = [ "pwhash", "regex", "reqwest 0.12.9", - "rustls 0.23.19", + "rustls 0.23.20", "rustls-pki-types", "scrypt", "serde", @@ -1836,7 +1854,7 @@ dependencies = [ "smtp-proto", "store", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "totp-rs", "trc", "utils", @@ -1871,7 +1889,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -2040,7 +2058,7 @@ dependencies = [ "crypto-bigint", "digest 0.10.7", "ff", - "generic-array", + "generic-array 0.14.7", "group", "hkdf", "pem-rfc7468", @@ -2060,12 +2078,6 @@ dependencies = [ "log", ] -[[package]] -name = "encode_unicode" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" - [[package]] name = "encode_unicode" version = "1.0.0" @@ -2096,7 +2108,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -2124,9 +2136,9 @@ dependencies = [ [[package]] name = "event-listener-strategy" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" +checksum = "3c3e4e0dd3673c1139bf041f3008816d9cf2946bbfac2945c09e523b8d7b05b2" dependencies = [ "event-listener 5.3.1", "pin-project-lite", @@ -2138,7 +2150,7 @@ version = "0.10.7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -2258,7 +2270,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c9f1255b81bd56192c67475eefa322f44b85440030547cea7664fd5dc248964" dependencies = [ "bytes", - "http 1.1.0", + "http 1.2.0", "httparse", "memchr", "mime", @@ -2314,7 +2326,7 @@ checksum = "afc84a5ff0dba78222551017f5625f3365aa09551c78cfaa44136fc6818c2611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", "try_map", ] @@ -2357,7 +2369,7 @@ checksum = "e99b8b3c28ae0e84b604c75f721c21dc77afb3706076af5e8216d15fd1deaae3" dependencies = [ "frunk_proc_macro_helpers", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -2369,7 +2381,7 @@ dependencies = [ "frunk_core", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -2381,7 +2393,7 @@ dependencies = [ "frunk_core", "frunk_proc_macro_helpers", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -2461,7 +2473,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -2514,6 +2526,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "generic-array" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cb8bc4c28d15ade99c7e90b219f30da4be5c88e586277e8cbe886beeb868ab2" +dependencies = [ + "typenum", +] + [[package]] name = "gethostname" version = "0.4.3" @@ -2593,7 +2614,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.6.0", + "indexmap 2.7.0", "slab", "tokio", "tokio-util", @@ -2611,8 +2632,8 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.1.0", - "indexmap 2.6.0", + "http 1.2.0", + "indexmap 2.7.0", "slab", "tokio", "tokio-util", @@ -2690,9 +2711,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hickory-client" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab9683b08d8f8957a857b0236455d80e1886eaa8c6178af556aa7871fb61b55" +checksum = "949d2fef0bbdd31a0f6affc6bf390b4a0017492903eff6f7516cb382d9e85536" dependencies = [ "cfg-if", "data-encoding", @@ -2710,9 +2731,9 @@ dependencies = [ [[package]] name = "hickory-proto" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07698b8420e2f0d6447a436ba999ec85d8fbf2a398bbd737b82cac4a2e96e512" +checksum = "447afdcdb8afb9d0a852af6dc65d9b285ce720ed7a59e42a8bf2e931c67bc1b5" dependencies = [ "async-trait", "bytes", @@ -2724,7 +2745,7 @@ dependencies = [ "futures-util", "h2 0.3.26", "http 0.2.12", - "idna 0.4.0", + "idna", "ipnet", "once_cell", "rand 0.8.5", @@ -2741,9 +2762,9 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28757f23aa75c98f254cf0405e6d8c25b831b32921b050a66692427679b1f243" +checksum = "0a2e2aba9c389ce5267d31cf1e4dace82390ae276b0b364ea55630b1fa1b44b4" dependencies = [ "cfg-if", "futures-util", @@ -2792,11 +2813,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2834,9 +2855,9 @@ dependencies = [ [[package]] name = "http" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" dependencies = [ "bytes", "fnv", @@ -2861,7 +2882,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.1.0", + "http 1.2.0", ] [[package]] @@ -2872,7 +2893,7 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "pin-project-lite", ] @@ -2917,9 +2938,9 @@ checksum = "9994b79e8c1a39b3166c63ae7823bb2b00831e2a96a31399c50fe69df408eaeb" [[package]] name = "hyper" -version = "0.14.31" +version = "0.14.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c08302e8fa335b151b788c775ff56e7a03ae64ff85c548ee820fecb70356e85" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" dependencies = [ "bytes", "futures-channel", @@ -2941,15 +2962,15 @@ dependencies = [ [[package]] name = "hyper" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" +checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" dependencies = [ "bytes", "futures-channel", "futures-util", "h2 0.4.7", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "httparse", "httpdate", @@ -2968,7 +2989,7 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http 0.2.12", - "hyper 0.14.31", + "hyper 0.14.32", "rustls 0.21.12", "tokio", "tokio-rustls 0.24.1", @@ -2976,18 +2997,18 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.3" +version = "0.27.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" +checksum = "f6884a48c6826ec44f524c7456b163cebe9e55a18d7b5e307cb4f100371cc767" dependencies = [ "futures-util", - "http 1.1.0", - "hyper 1.5.1", + "http 1.2.0", + "hyper 1.5.2", "hyper-util", - "rustls 0.23.19", + "rustls 0.23.20", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "tower-service", "webpki-roots 0.26.7", ] @@ -2998,7 +3019,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", "pin-project-lite", "tokio", @@ -3014,9 +3035,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", - "hyper 1.5.1", + "hyper 1.5.2", "pin-project-lite", "socket2", "tokio", @@ -3162,7 +3183,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -3180,16 +3201,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "idna" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "idna" version = "1.0.3" @@ -3234,11 +3245,11 @@ dependencies = [ "nlp", "parking_lot", "rand 0.8.5", - "rustls 0.23.19", + "rustls 0.23.20", "rustls-pemfile 2.2.0", "store", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "trc", "utils", ] @@ -3268,9 +3279,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -3311,7 +3322,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" dependencies = [ "block-padding", - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -3456,7 +3467,7 @@ dependencies = [ "futures-util", "hkdf", "http-body-util", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", "jmap_proto", "lz4_flex", @@ -3542,10 +3553,11 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.72" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -3555,7 +3567,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.7.0", ] [[package]] @@ -3662,15 +3674,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.166" +version = "0.2.168" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2ccc108bbc0b1331bd061864e7cd823c0cab660bbe6970e66e2c0614decde36" +checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" [[package]] name = "libloading" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", "windows-targets 0.52.6", @@ -3810,9 +3822,9 @@ dependencies = [ [[package]] name = "mail-auth" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaee4c38f4df428c6732f3d5472a013fa248d2772f48c8932295b32c683a23c4" +checksum = "01e36ed6613c4c43f2d3e4ff3af1888b3612c74d56fe0eac2cf25706f6a85a6b" dependencies = [ "ahash 0.8.11", "flate2", @@ -3821,7 +3833,7 @@ dependencies = [ "mail-builder", "mail-parser", "parking_lot", - "quick-xml 0.36.2", + "quick-xml 0.37.1", "rand 0.8.5", "ring 0.17.8", "rsa", @@ -3859,11 +3871,11 @@ dependencies = [ "base64 0.22.1", "gethostname", "md5", - "rustls 0.23.19", + "rustls 0.23.20", "rustls-pki-types", "smtp-proto", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "webpki-roots 0.26.7", ] @@ -3903,12 +3915,12 @@ dependencies = [ "mail-send", "md5", "parking_lot", - "rustls 0.23.19", + "rustls 0.23.20", "rustls-pemfile 2.2.0", "sieve-rs", "store", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "trc", "utils", ] @@ -3939,7 +3951,7 @@ checksum = "5cf92c10c7e361d6b99666ec1c6f9805b0bea2c3bd8c78dc6fe98ac5bd78db11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -4024,9 +4036,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394" dependencies = [ "adler2", ] @@ -4056,7 +4068,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", "termcolor", "thiserror 1.0.69", ] @@ -4106,7 +4118,7 @@ checksum = "478b0ff3f7d67b79da2b96f56f334431aef65e15ba4b29dd74a4236e29582bdc" dependencies = [ "base64 0.21.7", "bigdecimal", - "bindgen 0.70.1", + "bindgen 0.71.1", "bitflags 2.6.0", "bitvec", "btoi", @@ -4345,7 +4357,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -4388,7 +4400,7 @@ checksum = "88d8c2b76e5f7848a289aa9666dbe56b16f8a22a4c5246ef37a14941818d2913" dependencies = [ "async-trait", "bytes", - "http 1.1.0", + "http 1.2.0", "opentelemetry", "reqwest 0.12.9", ] @@ -4401,7 +4413,7 @@ checksum = "596b1719b3cab83addb20bcbffdf21575279d9436d9ccccfe651a3bf0ab5ab06" dependencies = [ "async-trait", "futures-core", - "http 1.1.0", + "http 1.2.0", "opentelemetry", "opentelemetry-http", "opentelemetry-proto", @@ -4588,7 +4600,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.6.0", + "indexmap 2.7.0", ] [[package]] @@ -4631,7 +4643,7 @@ dependencies = [ "phf_shared 0.11.2", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -4669,7 +4681,7 @@ checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -4734,10 +4746,10 @@ dependencies = [ "jmap_proto", "mail-parser", "mail-send", - "rustls 0.23.19", + "rustls 0.23.20", "store", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "trc", "utils", ] @@ -4805,7 +4817,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -4815,7 +4827,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eea25e07510aa6ab6547308ebe3c036016d162b8da920dbb079e3ba8acf3d95a" dependencies = [ "csv", - "encode_unicode 1.0.0", + "encode_unicode", "is-terminal", "lazy_static", "term", @@ -4869,7 +4881,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -4887,7 +4899,7 @@ version = "0.10.7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -4906,9 +4918,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" +checksum = "2c0fef6c4230e4ccf618a35c59d7ede15dea37de8427500f50aff708806e42ec" dependencies = [ "bytes", "prost-derive", @@ -4916,15 +4928,15 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" +checksum = "157c5a9d7ea5c2ed2d9fb8f495b64759f7816c7eaea54ba3978f0d63000162e3" dependencies = [ "anyhow", "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -4939,9 +4951,9 @@ dependencies = [ [[package]] name = "psl" -version = "2.1.62" +version = "2.1.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68dcd6202774d99d3d1558ab22f3618bd94e70c9f029d9c03d7af582ea8f8846" +checksum = "699f79b15ea465a0a1ef3c7f0ef2b6b6782a2f720f3587428f0eea37f5c1d3b9" dependencies = [ "psl-types", ] @@ -5022,6 +5034,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "quick-xml" +version = "0.37.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f22f29bdff3987b4d8632ef95fd6424ec7e4e0a57e2f4fc63e489e75357f6a03" +dependencies = [ + "memchr", +] + [[package]] name = "quinn" version = "0.11.6" @@ -5032,10 +5053,10 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.0.0", - "rustls 0.23.19", + "rustc-hash 2.1.0", + "rustls 0.23.20", "socket2", - "thiserror 2.0.3", + "thiserror 2.0.8", "tokio", "tracing", ] @@ -5050,11 +5071,11 @@ dependencies = [ "getrandom 0.2.15", "rand 0.8.5", "ring 0.17.8", - "rustc-hash 2.0.0", - "rustls 0.23.19", + "rustc-hash 2.1.0", + "rustls 0.23.20", "rustls-pki-types", "slab", - "thiserror 2.0.3", + "thiserror 2.0.8", "tinyvec", "tracing", "web-time", @@ -5062,9 +5083,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.7" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5a626c6807713b15cac82a6acaccd6043c9a5408c24baae07611fec3f243da" +checksum = "1c40286217b4ba3a71d644d752e6a0b71f13f1b6a2c5311acfcbe0c2418ed904" dependencies = [ "cfg_aliases", "libc", @@ -5301,7 +5322,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rand 0.8.5", - "rustls 0.23.19", + "rustls 0.23.20", "rustls-native-certs 0.7.3", "rustls-pemfile 2.2.0", "rustls-pki-types", @@ -5309,7 +5330,7 @@ dependencies = [ "sha1_smol", "socket2", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "tokio-util", "url", "webpki-roots 0.26.7", @@ -5317,9 +5338,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ "bitflags 2.6.0", ] @@ -5388,7 +5409,7 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.31", + "hyper 0.14.32", "hyper-rustls 0.24.2", "ipnet", "js-sys", @@ -5429,11 +5450,11 @@ dependencies = [ "futures-core", "futures-util", "h2 0.4.7", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.5.1", - "hyper-rustls 0.27.3", + "hyper 1.5.2", + "hyper-rustls 0.27.4", "hyper-util", "ipnet", "js-sys", @@ -5444,7 +5465,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.19", + "rustls 0.23.20", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", @@ -5452,7 +5473,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.2", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "tokio-util", "tower-service", "url", @@ -5563,9 +5584,9 @@ dependencies = [ [[package]] name = "roaring" -version = "0.10.7" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f81dc953b2244ddd5e7860cb0bb2a790494b898ef321d4aff8e260efab60cc88" +checksum = "41589aba99537475bf697f2118357cad1c31590c5a1b9f6d9fc4ad6d07503661" dependencies = [ "bytemuck", "byteorder", @@ -5669,7 +5690,7 @@ dependencies = [ "hex", "hmac 0.12.1", "http 0.2.12", - "hyper 0.14.31", + "hyper 0.14.32", "hyper-rustls 0.24.2", "log", "maybe-async", @@ -5730,9 +5751,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-hash" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" +checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" [[package]] name = "rustc_version" @@ -5749,7 +5770,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "semver 1.0.23", + "semver 1.0.24", ] [[package]] @@ -5789,9 +5810,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.19" +version = "0.23.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "934b404430bb06b3fae2cba809eb45a1ab1aecd64491213d7c3301b88393f8d1" +checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" dependencies = [ "once_cell", "ring 0.17.8", @@ -5846,9 +5867,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" dependencies = [ "web-time", ] @@ -5912,9 +5933,9 @@ checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71" [[package]] name = "scc" -version = "2.2.5" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66b202022bb57c049555430e11fc22fea12909276a80a4c3d368da36ac1d88ed" +checksum = "94b13f8ea6177672c49d12ed964cca44836f59621981b04a3e26b87e675181de" dependencies = [ "sdd", ] @@ -5967,9 +5988,9 @@ dependencies = [ [[package]] name = "sdd" -version = "3.0.4" +version = "3.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49c1eeaf4b6a87c7479688c6d52b9f1153cedd3c489300564f932b065c6eab95" +checksum = "478f121bb72bbf63c52c93011ea1791dca40140dfe13f8336c4c5ac952c33aa9" [[package]] name = "seahash" @@ -5985,7 +6006,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct", "der", - "generic-array", + "generic-array 0.14.7", "pkcs8", "subtle", "zeroize", @@ -6006,9 +6027,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" +checksum = "1863fd3768cd83c56a7f60faa4dc0d403f1b6df0a38c3c25f44b7894e45370d5" dependencies = [ "core-foundation-sys", "libc", @@ -6025,9 +6046,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.23" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" [[package]] name = "semver-parser" @@ -6037,9 +6058,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "sequoia-openpgp" -version = "1.21.2" +version = "1.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13261ee216b44d932ef93b2d4a75d45199bef77864bcc5b77ecfc7bc0ecb02d6" +checksum = "e858e4e9e48ff079cede92e1b45c942a5466ce9a4e3cc0c2a7e66586a718ef59" dependencies = [ "aes", "aes-gcm", @@ -6064,7 +6085,7 @@ dependencies = [ "ed25519-dalek", "getrandom 0.2.15", "idea", - "idna 1.0.3", + "idna", "lalrpop", "lalrpop-util", "lazy_static", @@ -6093,9 +6114,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.215" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" +checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" dependencies = [ "serde_derive", ] @@ -6111,13 +6132,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.215" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" +checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -6126,7 +6147,7 @@ version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.7.0", "itoa", "memchr", "ryu", @@ -6200,7 +6221,7 @@ checksum = "5d69265a08751de7844521fd15003ae0a888e035773ba05695c5c759a6f89eef" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -6241,7 +6262,7 @@ checksum = "1f606421e4a6012877e893c399822a4ed4b089164c5969424e1b9d1e66e6964b" dependencies = [ "const-oid", "digest 0.10.7", - "generic-array", + "generic-array 1.1.1", ] [[package]] @@ -6360,7 +6381,7 @@ dependencies = [ "directory", "form_urlencoded", "http-body-util", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", "lru-cache", "mail-auth", @@ -6375,7 +6396,7 @@ dependencies = [ "rayon", "regex", "reqwest 0.12.9", - "rustls 0.23.19", + "rustls 0.23.20", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", @@ -6384,9 +6405,10 @@ dependencies = [ "sha2 0.10.8", "sieve-rs", "smtp-proto", + "spam-filter", "store", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "trc", "utils", "webpki-roots 0.26.7", @@ -6427,9 +6449,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" dependencies = [ "libc", "windows-sys 0.52.0", @@ -6441,8 +6463,8 @@ version = "0.10.7" dependencies = [ "common", "decancer", - "hyper 1.5.1", - "idna 1.0.3", + "hyper 1.5.2", + "idna", "infer 0.16.0", "mail-auth", "mail-builder", @@ -6558,13 +6580,13 @@ dependencies = [ "rocksdb", "rusqlite", "rust-s3", - "rustls 0.23.19", + "rustls 0.23.20", "rustls-pki-types", "serde", "serde_json", "tokio", "tokio-postgres", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "trc", "utils", "xxhash-rust", @@ -6635,9 +6657,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.89" +version = "2.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d46482f1c1c87acd84dea20c1bf5ebff4c757009ed6bf19cfd36fb10e92c4e" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" dependencies = [ "proc-macro2", "quote", @@ -6679,7 +6701,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -6748,7 +6770,7 @@ dependencies = [ "form_urlencoded", "futures", "http-body-util", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", "imap", "imap_proto", @@ -6766,7 +6788,7 @@ dependencies = [ "rayon", "reqwest 0.12.9", "ring 0.17.8", - "rustls 0.23.19", + "rustls 0.23.20", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", @@ -6778,7 +6800,7 @@ dependencies = [ "spam-filter", "store", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "trc", "utils", ] @@ -6794,11 +6816,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.3" +version = "2.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa" +checksum = "08f5383f3e0071702bf93ab5ee99b52d26936be9dedd9413067cbdcddcb6141a" dependencies = [ - "thiserror-impl 2.0.3", + "thiserror-impl 2.0.8", ] [[package]] @@ -6809,25 +6831,25 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] name = "thiserror-impl" -version = "2.0.3" +version = "2.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" +checksum = "f2f357fcec90b3caef6623a099691be676d033b40a058ac95d2a6ade6fa0c943" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] name = "time" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" dependencies = [ "deranged", "itoa", @@ -6847,9 +6869,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" dependencies = [ "num-conv", "time-core", @@ -6916,7 +6938,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -6968,20 +6990,19 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ - "rustls 0.23.19", - "rustls-pki-types", + "rustls 0.23.20", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" dependencies = [ "futures-core", "pin-project-lite", @@ -7018,9 +7039,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" +checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" dependencies = [ "bytes", "futures-core", @@ -7041,7 +7062,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.7.0", "toml_datetime", "winnow", ] @@ -7058,10 +7079,10 @@ dependencies = [ "base64 0.22.1", "bytes", "h2 0.4.7", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-timeout", "hyper-util", "percent-encoding", @@ -7113,14 +7134,14 @@ dependencies = [ [[package]] name = "tower" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", "pin-project-lite", - "sync_wrapper 0.1.2", + "sync_wrapper 1.0.2", "tower-layer", "tower-service", ] @@ -7156,7 +7177,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -7213,7 +7234,7 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http 1.1.0", + "http 1.2.0", "httparse", "log", "rand 0.8.5", @@ -7234,7 +7255,7 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http 1.1.0", + "http 1.2.0", "httparse", "log", "rand 0.8.5", @@ -7292,9 +7313,9 @@ checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" [[package]] name = "unicode-bidi" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" @@ -7380,7 +7401,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", - "idna 1.0.3", + "idna", "percent-encoding", "serde", ] @@ -7439,14 +7460,14 @@ dependencies = [ "regex", "reqwest 0.12.9", "ring 0.17.8", - "rustls 0.23.19", + "rustls 0.23.20", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", "serde_json", "smtp-proto", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "trc", "webpki-roots 0.26.7", "x509-parser 0.16.0", @@ -7525,9 +7546,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" dependencies = [ "cfg-if", "once_cell", @@ -7536,36 +7557,36 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.45" +version = "0.4.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" +checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" dependencies = [ "cfg-if", "js-sys", + "once_cell", "wasm-bindgen", "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -7573,22 +7594,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" [[package]] name = "wasm-streams" @@ -7605,9 +7626,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.72" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" +checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" dependencies = [ "js-sys", "wasm-bindgen", @@ -7990,9 +8011,9 @@ dependencies = [ [[package]] name = "xml-rs" -version = "0.8.23" +version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af310deaae937e48a26602b730250b4949e125f468f11e6990be3e5304ddd96f" +checksum = "ea8b391c9a790b496184c29f7f93b9ed5b16abb306c05415b68bcc16e4d06432" [[package]] name = "xxhash-rust" @@ -8029,7 +8050,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", "synstructure 0.13.1", ] @@ -8051,7 +8072,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -8071,7 +8092,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", "synstructure 0.13.1", ] @@ -8092,7 +8113,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -8114,14 +8135,14 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] name = "zip" -version = "2.2.1" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d52293fc86ea7cf13971b3bb81eb21683636e7ae24c729cdaf1b7c4157a352" +checksum = "ae9c1ea7b3a5e1f4b922ff856a129881167511563dc219869afe3787fc0c1a45" dependencies = [ "aes", "arbitrary", @@ -8133,13 +8154,13 @@ dependencies = [ "displaydoc", "flate2", "hmac 0.12.1", - "indexmap 2.6.0", + "indexmap 2.7.0", "lzma-rs", "memchr", "pbkdf2", "rand 0.8.5", "sha1", - "thiserror 2.0.3", + "thiserror 2.0.8", "time", "zeroize", "zopfli", diff --git a/crates/common/src/config/smtp/queue.rs b/crates/common/src/config/smtp/queue.rs index f471ba04..34257bf0 100644 --- a/crates/common/src/config/smtp/queue.rs +++ b/crates/common/src/config/smtp/queue.rs @@ -84,6 +84,7 @@ pub struct QueueOutboundTimeout { #[derive(Debug, Clone)] pub struct QueueThrottle { + pub outbound_concurrency: u64, pub sender: Vec, pub rcpt: Vec, pub host: Vec, @@ -205,6 +206,7 @@ impl Default for QueueConfig { mta_sts: IfBlock::new::<()>("queue.outbound.timeouts.mta-sts", [], "10m"), }, throttle: QueueThrottle { + outbound_concurrency: 25, sender: Default::default(), rcpt: Default::default(), host: Default::default(), @@ -388,6 +390,9 @@ fn parse_queue_throttle(config: &mut Config) -> QueueThrottle { sender: Vec::new(), rcpt: Vec::new(), host: Vec::new(), + outbound_concurrency: config + .property_or_default::("queue.outbound.concurrency", "25") + .unwrap_or(25), }; let all_throttles = parse_throttle( diff --git a/crates/common/src/expr/if_block.rs b/crates/common/src/expr/if_block.rs index 5254cbf1..921a0c17 100644 --- a/crates/common/src/expr/if_block.rs +++ b/crates/common/src/expr/if_block.rs @@ -14,15 +14,13 @@ use super::{ ConstantValue, ExpressionItem, }; -#[derive(Debug, Clone)] -#[cfg_attr(feature = "test_mode", derive(PartialEq, Eq))] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct IfThen { pub expr: Expression, pub then: Expression, } -#[derive(Debug, Clone)] -#[cfg_attr(feature = "test_mode", derive(PartialEq, Eq))] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct IfBlock { pub key: String, pub if_then: Vec, diff --git a/crates/common/src/ipc.rs b/crates/common/src/ipc.rs index 113a01fd..b21e4b5a 100644 --- a/crates/common/src/ipc.rs +++ b/crates/common/src/ipc.rs @@ -120,8 +120,10 @@ pub struct EncryptionKeys { #[derive(Debug)] pub enum QueueEvent { - Reload, - OnHold(OnHold), + Refresh(Option), + WorkerDone(u64), + OnHold(OnHold), + Paused(bool), Stop, } @@ -132,11 +134,10 @@ pub struct OnHold { pub message: T, } -#[derive(Debug)] -pub struct QueueEventLock { +#[derive(Debug, Clone, Copy)] +pub struct QueuedMessage { pub due: u64, pub queue_id: u64, - pub lock_expiry: u64, } #[derive(Debug)] diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 53b59d12..36e16942 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -87,6 +87,10 @@ pub const KV_BAYES_MODEL_GLOBAL: u8 = 17; pub const KV_BAYES_MODEL_USER: u8 = 18; pub const KV_TRUSTED_REPLY: u8 = 19; pub const KV_LOCK_PURGE_ACCOUNT: u8 = 20; +pub const KV_LOCK_QUEUE_MESSAGE: u8 = 21; +pub const KV_LOCK_QUEUE_REPORT: u8 = 22; +pub const KV_LOCK_FTS: u8 = 23; +pub const KV_LOCK_HOUSEKEEPER: u8 = 24; #[derive(Clone)] pub struct Server { diff --git a/crates/common/src/scripts/plugins/mod.rs b/crates/common/src/scripts/plugins/mod.rs index 9643f655..4cdfd164 100644 --- a/crates/common/src/scripts/plugins/mod.rs +++ b/crates/common/src/scripts/plugins/mod.rs @@ -66,7 +66,7 @@ impl RegisterSievePlugins for FunctionMap { } fn register_plugins_untrusted(mut self) -> Self { - llm_prompt::register(18, &mut self); + llm_prompt::register(12, &mut self); self } } diff --git a/crates/jmap/src/api/management/queue.rs b/crates/jmap/src/api/management/queue.rs index 35042a8c..56ba0c58 100644 --- a/crates/jmap/src/api/management/queue.rs +++ b/crates/jmap/src/api/management/queue.rs @@ -343,7 +343,12 @@ impl QueueManagement for Server { message .save_changes(self, prev_event.into(), next_event.into()) .await; - let _ = self.inner.ipc.queue_tx.send(QueueEvent::Reload).await; + let _ = self + .inner + .ipc + .queue_tx + .send(QueueEvent::Refresh(None)) + .await; } Ok(JsonResponse::new(json!({ diff --git a/crates/jmap/src/auth/oauth/token.rs b/crates/jmap/src/auth/oauth/token.rs index bdfc943c..8eef9ec5 100644 --- a/crates/jmap/src/auth/oauth/token.rs +++ b/crates/jmap/src/auth/oauth/token.rs @@ -9,11 +9,11 @@ use common::{ oauth::{oidc::StandardClaims, GrantType}, AccessToken, }, - Server, + Server, KV_OAUTH, }; use hyper::StatusCode; use std::future::Future; -use store::write::Bincode; +use store::{dispatch::lookup::KeyValue, write::Bincode}; use trc::AddContext; use crate::api::{ @@ -79,7 +79,10 @@ impl TokenHandler for Server { .core .storage .lookup - .key_get::>(format!("oauth:{code}").into_bytes()) + .key_get::>(KeyValue::<()>::build_key( + KV_OAUTH, + code.as_bytes(), + )) .await? { Some(auth_code) => { @@ -102,7 +105,10 @@ impl TokenHandler for Server { self.core .storage .lookup - .key_delete(format!("oauth:{code}").into_bytes()) + .key_delete(KeyValue::<()>::build_key( + KV_OAUTH, + code.as_bytes(), + )) .await?; // Issue token @@ -143,7 +149,10 @@ impl TokenHandler for Server { .core .storage .lookup - .key_get::>(format!("oauth:{device_code}").into_bytes()) + .key_get::>(KeyValue::<()>::build_key( + KV_OAUTH, + device_code.as_bytes(), + )) .await? { let oauth = auth_code.inner; @@ -162,7 +171,10 @@ impl TokenHandler for Server { self.core .storage .lookup - .key_delete(format!("oauth:{device_code}").into_bytes()) + .key_delete(KeyValue::<()>::build_key( + KV_OAUTH, + device_code.as_bytes(), + )) .await?; // Issue token diff --git a/crates/jmap/src/email/delete.rs b/crates/jmap/src/email/delete.rs index 3ad55545..712e1521 100644 --- a/crates/jmap/src/email/delete.rs +++ b/crates/jmap/src/email/delete.rs @@ -298,10 +298,8 @@ impl EmailDeletion for Server { // Delete lock if let Err(err) = self - .core - .storage - .lookup - .counter_delete(format!("purge:{account_id}").into_bytes()) + .in_memory_store() + .remove_lock(KV_LOCK_PURGE_ACCOUNT, &account_id.to_be_bytes()) .await { trc::error!(err.details("Failed to delete lock.").account_id(account_id)); diff --git a/crates/jmap/src/services/gossip/ping.rs b/crates/jmap/src/services/gossip/ping.rs index dc05a7e3..ba2b6346 100644 --- a/crates/jmap/src/services/gossip/ping.rs +++ b/crates/jmap/src/services/gossip/ping.rs @@ -75,7 +75,12 @@ impl Gossiper { trc::event!(Cluster(ClusterEvent::OneOrMorePeersOffline)); server.request_fts_index(); - let _ = server.inner.ipc.queue_tx.send(QueueEvent::Reload).await; + let _ = server + .inner + .ipc + .queue_tx + .send(QueueEvent::Refresh(None)) + .await; }); } diff --git a/crates/jmap/src/services/housekeeper.rs b/crates/jmap/src/services/housekeeper.rs index 0be886e0..d27605a5 100644 --- a/crates/jmap/src/services/housekeeper.rs +++ b/crates/jmap/src/services/housekeeper.rs @@ -6,6 +6,7 @@ use std::{ collections::BinaryHeap, + future::Future, sync::{atomic::Ordering, Arc}, time::{Duration, Instant, SystemTime}, }; @@ -14,7 +15,7 @@ use common::{ config::telemetry::OtelMetrics, core::BuildServer, ipc::{HousekeeperEvent, PurgeType}, - Inner, + Inner, Server, KV_LOCK_HOUSEKEEPER, }; #[cfg(feature = "enterprise")] @@ -229,94 +230,12 @@ pub fn spawn_housekeeper(inner: Arc, mut rx: mpsc::Receiver match purge { - PurgeType::Data(store) => { - // SPDX-SnippetBegin - // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd - // SPDX-License-Identifier: LicenseRef-SEL - #[cfg(feature = "enterprise")] - let trace_retention = inner - .shared_core - .load() - .enterprise - .as_ref() - .and_then(|e| e.trace_store.as_ref()) - .and_then(|t| t.retention); - #[cfg(feature = "enterprise")] - let metrics_retention = inner - .shared_core - .load() - .enterprise - .as_ref() - .and_then(|e| e.metrics_store.as_ref()) - .and_then(|m| m.retention); - // SPDX-SnippetEnd - - tokio::spawn(async move { - trc::event!( - Housekeeper(trc::HousekeeperEvent::PurgeStore), - Type = "data" - ); - if let Err(err) = store.purge_store().await { - trc::error!(err.details("Failed to purge data store")); - } - - // SPDX-SnippetBegin - // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd - // SPDX-License-Identifier: LicenseRef-SEL - #[cfg(feature = "enterprise")] - if let Some(trace_retention) = trace_retention { - if let Err(err) = store.purge_spans(trace_retention).await { - trc::error!(err.details("Failed to purge tracing spans")); - } - } - - #[cfg(feature = "enterprise")] - if let Some(metrics_retention) = metrics_retention { - if let Err(err) = store.purge_metrics(metrics_retention).await { - trc::error!(err.details("Failed to purge metrics")); - } - } - // SPDX-SnippetEnd - }); - } - PurgeType::Blobs { store, blob_store } => { - trc::event!( - Housekeeper(trc::HousekeeperEvent::PurgeStore), - Type = "blob" - ); - - tokio::spawn(async move { - if let Err(err) = store.purge_blobs(blob_store).await { - trc::error!(err.details("Failed to purge blob store")); - } - }); - } - PurgeType::Lookup(store) => { - trc::event!( - Housekeeper(trc::HousekeeperEvent::PurgeStore), - Type = "lookup" - ); - - tokio::spawn(async move { - if let Err(err) = store.purge_in_memory_store().await { - trc::error!(err.details("Failed to purge lookup store")); - } - }); - } - PurgeType::Account(account_id) => { - let server = inner.build_server(); - tokio::spawn(async move { - trc::event!(Housekeeper(trc::HousekeeperEvent::PurgeAccounts)); - - if let Some(account_id) = account_id { - server.purge_account(account_id).await; - } else { - server.purge_accounts().await; - } - }); - } - }, + HousekeeperEvent::Purge(purge) => { + let server = inner.build_server(); + tokio::spawn(async move { + server.purge(purge).await; + }); + } HousekeeperEvent::Exit => { trc::event!(Housekeeper(trc::HousekeeperEvent::Stop)); @@ -663,6 +582,133 @@ pub fn spawn_housekeeper(inner: Arc, mut rx: mpsc::Receiver impl Future + Send; +} + +impl Purge for Server { + async fn purge(&self, purge: PurgeType) { + // Lock task + let lock_name = match &purge { + PurgeType::Data(_) => "data".into(), + PurgeType::Blobs { .. } => "blob".into(), + PurgeType::Lookup(_) => "lookup".into(), + PurgeType::Account(_) => None, + }; + if let Some(lock_name) = lock_name { + match self + .core + .storage + .lookup + .try_lock(KV_LOCK_HOUSEKEEPER, lock_name.as_bytes(), 3600) + .await + { + Ok(true) => (), + Ok(false) => { + trc::event!(Purge(trc::PurgeEvent::PurgeActive), Details = lock_name); + return; + } + Err(err) => { + trc::error!(err.details("Failed to lock task.").details(lock_name)); + return; + } + } + } + + match purge { + PurgeType::Data(store) => { + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + let trace_retention = self + .core + .enterprise + .as_ref() + .and_then(|e| e.trace_store.as_ref()) + .and_then(|t| t.retention); + #[cfg(feature = "enterprise")] + let metrics_retention = self + .core + .enterprise + .as_ref() + .and_then(|e| e.metrics_store.as_ref()) + .and_then(|m| m.retention); + // SPDX-SnippetEnd + + trc::event!( + Housekeeper(trc::HousekeeperEvent::PurgeStore), + Type = "data" + ); + + if let Err(err) = store.purge_store().await { + trc::error!(err.details("Failed to purge data store")); + } + + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + if let Some(trace_retention) = trace_retention { + if let Err(err) = store.purge_spans(trace_retention).await { + trc::error!(err.details("Failed to purge tracing spans")); + } + } + + #[cfg(feature = "enterprise")] + if let Some(metrics_retention) = metrics_retention { + if let Err(err) = store.purge_metrics(metrics_retention).await { + trc::error!(err.details("Failed to purge metrics")); + } + } + // SPDX-SnippetEnd + } + PurgeType::Blobs { store, blob_store } => { + trc::event!( + Housekeeper(trc::HousekeeperEvent::PurgeStore), + Type = "blob" + ); + + if let Err(err) = store.purge_blobs(blob_store).await { + trc::error!(err.details("Failed to purge blob store")); + } + } + PurgeType::Lookup(store) => { + trc::event!( + Housekeeper(trc::HousekeeperEvent::PurgeStore), + Type = "lookup" + ); + + if let Err(err) = store.purge_in_memory_store().await { + trc::error!(err.details("Failed to purge lookup store")); + } + } + PurgeType::Account(account_id) => { + trc::event!(Housekeeper(trc::HousekeeperEvent::PurgeAccounts)); + + if let Some(account_id) = account_id { + self.purge_account(account_id).await; + } else { + self.purge_accounts().await; + } + } + } + + // Remove lock + if let Some(lock_name) = lock_name { + if let Err(err) = self + .in_memory_store() + .remove_lock(KV_LOCK_HOUSEKEEPER, lock_name.as_bytes()) + .await + { + trc::error!(err + .details("Failed to delete task lock.") + .details(lock_name)); + } + } + } +} + impl Queue { pub fn schedule(&mut self, due: Instant, event: ActionClass) { trc::event!( diff --git a/crates/jmap/src/services/index.rs b/crates/jmap/src/services/index.rs index 4381856e..fa5516f5 100644 --- a/crates/jmap/src/services/index.rs +++ b/crates/jmap/src/services/index.rs @@ -6,7 +6,7 @@ use std::{sync::Arc, time::Instant}; -use common::{core::BuildServer, Inner, Server}; +use common::{core::BuildServer, Inner, Server, KV_LOCK_FTS}; use directory::{ backend::internal::{manage::ManageDirectory, PrincipalField}, Type, @@ -17,10 +17,10 @@ use store::{ fts::index::FtsDocument, roaring::RoaringBitmap, write::{ - key::DeserializeBigEndian, now, BatchBuilder, Bincode, BlobOp, FtsQueueClass, - MaybeDynamicId, ValueClass, + key::DeserializeBigEndian, BatchBuilder, Bincode, BlobOp, FtsQueueClass, MaybeDynamicId, + ValueClass, }, - Deserialize, IterateParams, Serialize, ValueKey, U32_LEN, U64_LEN, + IterateParams, Serialize, ValueKey, U32_LEN, U64_LEN, }; use std::future::Future; @@ -39,7 +39,6 @@ pub struct IndexEmail { account_id: u32, document_id: u32, seq: u64, - lock_expiry: u64, insert_hash: BlobHash, } @@ -48,9 +47,13 @@ const INDEX_LOCK_EXPIRY: u64 = 60 * 5; pub fn spawn_index_task(inner: Arc) { tokio::spawn(async move { let rx = inner.ipc.index_tx.clone(); + let mut locked_seq_ids = AHashMap::new(); loop { // Index any queued messages - inner.build_server().fts_index_queued().await; + inner + .build_server() + .fts_index_queued(&mut locked_seq_ids) + .await; // Wait for a signal to index more messages rx.notified().await; @@ -59,8 +62,12 @@ pub fn spawn_index_task(inner: Arc) { } pub trait Indexer: Sync + Send { - fn fts_index_queued(&self) -> impl Future + Send; + fn fts_index_queued( + &self, + locked_seq_ids: &mut AHashMap, + ) -> impl Future + Send; fn try_lock_index(&self, event: &IndexEmail) -> impl Future + Send; + fn remove_index_lock(&self, seq_id: u64) -> impl Future + Send; fn reindex( &self, account_id: Option, @@ -70,7 +77,7 @@ pub trait Indexer: Sync + Send { } impl Indexer for Server { - async fn fts_index_queued(&self) { + async fn fts_index_queued(&self, locked_seq_ids: &mut AHashMap) { let from_key = ValueKey::> { account_id: 0, collection: 0, @@ -92,25 +99,20 @@ impl Indexer for Server { // Retrieve entries pending to be indexed let mut entries = Vec::new(); - let now = now(); + let now = Instant::now(); let _ = self .core .storage .data .iterate( - IterateParams::new(from_key, to_key).ascending(), - |key, value| { - let event = IndexEmail::deserialize(key, value)?; - - if event.lock_expiry < now { - entries.push(event); - } else { - trc::event!( - FtsIndex(FtsIndexEvent::Locked), - AccountId = event.account_id, - DocumentId = event.document_id, - Expires = trc::Value::Timestamp(event.lock_expiry), - ); + IterateParams::new(from_key, to_key).ascending().no_values(), + |key, _| { + let entry = IndexEmail::deserialize(key)?; + if locked_seq_ids + .get(&entry.seq) + .map_or(true, |expires| now >= *expires) + { + entries.push(entry); } Ok(true) @@ -124,12 +126,18 @@ impl Indexer for Server { }); // Add entries to the index + let mut unlock_seq_ids = Vec::with_capacity(entries.len()); for event in entries { let op_start = Instant::now(); // Lock index if !self.try_lock_index(&event).await { + locked_seq_ids.insert( + event.seq, + Instant::now() + std::time::Duration::from_secs(INDEX_LOCK_EXPIRY + 1), + ); continue; } + unlock_seq_ids.push(event.seq); match self .get_property::>( @@ -227,27 +235,33 @@ impl Indexer for Server { break; } } + + // Unlock entries + for seq_id in unlock_seq_ids { + self.remove_index_lock(seq_id).await; + } + + // Delete expired locks + let now = Instant::now(); + locked_seq_ids.retain(|_, expires| *expires > now); } async fn try_lock_index(&self, event: &IndexEmail) -> bool { - let mut batch = BatchBuilder::new(); - batch - .with_account_id(event.account_id) - .with_collection(Collection::Email) - .update_document(event.document_id) - .assert_value(event.value_class(), event.lock_expiry) - .set(event.value_class(), (now() + INDEX_LOCK_EXPIRY).serialize()); - match self.core.storage.data.write(batch.build()).await { - Ok(_) => true, - Err(err) if err.is_assertion_failure() => { - trc::event!( - FtsIndex(FtsIndexEvent::LockBusy), - AccountId = event.account_id, - DocumentId = event.document_id, - CausedBy = err, - ); - - false + match self + .in_memory_store() + .try_lock(KV_LOCK_FTS, &event.seq.to_be_bytes(), INDEX_LOCK_EXPIRY) + .await + { + Ok(result) => { + if !result { + trc::event!( + FtsIndex(FtsIndexEvent::Locked), + AccountId = event.account_id, + DocumentId = event.document_id, + Expires = trc::Value::Timestamp(INDEX_LOCK_EXPIRY), + ); + } + result } Err(err) => { trc::error!(err @@ -260,6 +274,19 @@ impl Indexer for Server { } } + async fn remove_index_lock(&self, seq_id: u64) { + if let Err(err) = self + .in_memory_store() + .remove_lock(KV_LOCK_FTS, &seq_id.to_be_bytes()) + .await + { + trc::error!(err + .details("Failed to unlock FTS index") + .ctx(trc::Key::Key, seq_id) + .caused_by(trc::location!())); + } + } + fn request_fts_index(&self) { self.inner.ipc.index_tx.notify_one(); } @@ -383,19 +410,18 @@ impl IndexEmail { }) } - fn deserialize(key: &[u8], value: &[u8]) -> trc::Result { + fn deserialize(key: &[u8]) -> trc::Result { Ok(IndexEmail { seq: key.deserialize_be_u64(0)?, account_id: key.deserialize_be_u32(U64_LEN)?, document_id: key.deserialize_be_u32(U64_LEN + U32_LEN + 1)?, - lock_expiry: u64::deserialize(value)?, insert_hash: key .get( U64_LEN + U32_LEN + U32_LEN + 1 ..U64_LEN + U32_LEN + U32_LEN + BLOB_HASH_LEN + 1, ) .and_then(|bytes| BlobHash::try_from_hash_slice(bytes).ok()) - .ok_or_else(|| trc::Error::corrupted_key(key, value.into(), trc::location!()))?, + .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?, }) } } diff --git a/crates/smtp/src/outbound/delivery.rs b/crates/smtp/src/outbound/delivery.rs index f8f00ad0..691d004b 100644 --- a/crates/smtp/src/outbound/delivery.rs +++ b/crates/smtp/src/outbound/delivery.rs @@ -11,14 +11,14 @@ use crate::outbound::mta_sts::lookup::MtaStsLookup; use crate::outbound::mta_sts::verify::VerifyPolicy; use crate::outbound::{client::StartTlsResult, dane::verify::TlsaVerify}; use crate::queue::dsn::SendDsn; -use crate::queue::spool::SmtpSpool; +use crate::queue::spool::{SmtpSpool, LOCK_EXPIRY}; use crate::queue::throttle::IsAllowed; use crate::reporting::SmtpReporting; use common::config::{ server::ServerProtocol, smtp::{queue::RequireOptional, report::AggregateFrequency}, }; -use common::ipc::{OnHold, PolicyType, QueueEvent, TlsEvent}; +use common::ipc::{OnHold, PolicyType, QueueEvent, QueuedMessage, TlsEvent}; use common::Server; use mail_auth::{ mta_sts::TlsRpt, @@ -41,14 +41,21 @@ use super::{lookup::ToNextHop, mta_sts, session::SessionParams, NextHop, TlsStra use crate::queue::{throttle, DeliveryAttempt, Domain, Error, QueueEnvelope, Status}; impl DeliveryAttempt { - pub async fn try_deliver(mut self, server: Server) { - tokio::spawn(async move { - // Lock message - if let Some(event) = server.try_lock_event(self.event).await { - self.event = event; + pub fn try_deliver(mut self, server: Server) -> Option> { + // Global concurrency limiter + if let Err(limiter) = server.is_outbound_allowed(&mut self.in_flight) { + return Some(OnHold { + next_due: None, + limiters: vec![limiter], + message: self.event, + }); + } - // Fetch message - if let Some(mut message) = server.read_message(self.event.queue_id).await { + tokio::spawn(async move { + // Lock queue event + let queue_id = self.event.queue_id; + let queue_event = if server.try_lock_event(queue_id).await { + if let Some(mut message) = server.read_message(queue_id).await { // Generate span id message.span_id = server.inner.data.span_id_gen.generate().unwrap_or_else(now); let span_id = message.span_id; @@ -82,13 +89,18 @@ impl DeliveryAttempt { // Attempt delivery let start_time = Instant::now(); - self.deliver_task(server, message).await; + let queue_event = self.deliver_task(server.clone(), message).await; trc::event!( Delivery(DeliveryEvent::AttemptEnd), SpanId = span_id, Elapsed = start_time.elapsed(), ); + + // Unlock event + server.unlock_event(queue_id).await; + + queue_event } else { // Message no longer exists, delete queue event. let mut batch = BatchBuilder::new(); @@ -104,15 +116,38 @@ impl DeliveryAttempt { .details("Failed to delete queue event.") .caused_by(trc::location!())); } + + // Unlock event + server.unlock_event(queue_id).await; + + QueueEvent::WorkerDone(queue_id) } + } else { + QueueEvent::OnHold(OnHold { + next_due: Some(LOCK_EXPIRY + 1), + limiters: vec![], + message: self.event, + }) + }; + + // Notify queue manager + if server.inner.ipc.queue_tx.send(queue_event).await.is_err() { + trc::event!( + Server(ServerEvent::ThreadError), + Reason = "Channel closed.", + CausedBy = trc::location!(), + ); } }); + + None } - async fn deliver_task(mut self, server: Server, mut message: Message) { + async fn deliver_task(mut self, server: Server, mut message: Message) -> QueueEvent { // Check that the message still has recipients to be delivered let has_pending_delivery = message.has_pending_delivery(); let span_id = message.span_id; + let queue_id = message.queue_id; // Send any due Delivery Status Notifications server.send_dsn(&mut message).await; @@ -125,22 +160,7 @@ impl DeliveryAttempt { message .save_changes(&server, self.event.due.into(), due.into()) .await; - if server - .inner - .ipc - .queue_tx - .send(QueueEvent::Reload) - .await - .is_err() - { - trc::event!( - Server(ServerEvent::ThreadError), - Reason = "Channel closed.", - CausedBy = trc::location!(), - SpanId = span_id - ); - } - return; + return QueueEvent::Refresh(queue_id.into()); } } else { trc::event!( @@ -151,23 +171,8 @@ impl DeliveryAttempt { // All message recipients expired, do not re-queue. (DSN has been already sent) message.remove(&server, self.event.due).await; - if server - .inner - .ipc - .queue_tx - .send(QueueEvent::Reload) - .await - .is_err() - { - trc::event!( - Server(ServerEvent::ThreadError), - Reason = "Channel closed.", - CausedBy = trc::location!(), - SpanId = span_id - ); - } - return; + return QueueEvent::WorkerDone(queue_id); } // Throttle sender @@ -212,19 +217,11 @@ impl DeliveryAttempt { .save_changes(&server, self.event.due.into(), next_event.into()) .await; - QueueEvent::Reload + QueueEvent::Refresh(queue_id.into()) } }; - if server.inner.ipc.queue_tx.send(event).await.is_err() { - trc::event!( - Server(ServerEvent::ThreadError), - Reason = "Channel closed.", - CausedBy = trc::location!(), - SpanId = span_id - ); - } - return; + return event; } } @@ -1329,7 +1326,7 @@ impl DeliveryAttempt { server.send_dsn(&mut message).await; // Notify queue manager - let result = if !on_hold.is_empty() { + if !on_hold.is_empty() { // Save changes to disk let next_due = message.next_event_after(now()); message.save_changes(&server, None, None).await; @@ -1358,7 +1355,7 @@ impl DeliveryAttempt { .save_changes(&server, self.event.due.into(), due.into()) .await; - QueueEvent::Reload + QueueEvent::Refresh(queue_id.into()) } else { trc::event!( Delivery(DeliveryEvent::Completed), @@ -1369,15 +1366,7 @@ impl DeliveryAttempt { // Delete message from queue message.remove(&server, self.event.due).await; - QueueEvent::Reload - }; - if server.inner.ipc.queue_tx.send(result).await.is_err() { - trc::event!( - Server(ServerEvent::ThreadError), - Reason = "Channel closed.", - CausedBy = trc::location!(), - SpanId = span_id - ); + QueueEvent::WorkerDone(queue_id) } } } diff --git a/crates/smtp/src/outbound/mod.rs b/crates/smtp/src/outbound/mod.rs index 5461116c..1268e55e 100644 --- a/crates/smtp/src/outbound/mod.rs +++ b/crates/smtp/src/outbound/mod.rs @@ -11,7 +11,7 @@ use common::{ server::ServerProtocol, smtp::queue::{RelayHost, RequireOptional}, }, - ipc::QueueEventLock, + ipc::QueuedMessage, }; use mail_send::Credentials; use smtp_proto::{Response, Severity}; @@ -204,7 +204,7 @@ impl From for Status<(), Error> { } impl DeliveryAttempt { - pub fn new(event: QueueEventLock) -> Self { + pub fn new(event: QueuedMessage) -> Self { DeliveryAttempt { in_flight: Vec::new(), event, diff --git a/crates/smtp/src/queue/manager.rs b/crates/smtp/src/queue/manager.rs index 6f16d7d9..ff7e570d 100644 --- a/crates/smtp/src/queue/manager.rs +++ b/crates/smtp/src/queue/manager.rs @@ -6,86 +6,152 @@ use std::{ sync::{atomic::Ordering, Arc}, - time::Duration, + time::{Duration, Instant}, }; +use ahash::AHashSet; use common::{ core::BuildServer, - ipc::{OnHold, QueueEvent, QueueEventLock}, + ipc::{OnHold, QueueEvent, QueuedMessage}, Inner, }; use store::write::now; use tokio::sync::mpsc; -use super::{spool::SmtpSpool, DeliveryAttempt, Message, Status}; - -pub(crate) const SHORT_WAIT: Duration = Duration::from_millis(1); -pub(crate) const LONG_WAIT: Duration = Duration::from_secs(86400 * 365); +use super::{ + spool::{SmtpSpool, QUEUE_REFRESH}, + DeliveryAttempt, Message, QueueId, Status, +}; pub struct Queue { pub core: Arc, - pub on_hold: Vec>, - pub next_wake_up: Duration, + pub on_hold: Vec>, + pub in_flight: AHashSet, + pub next_wake_up: Instant, + pub rx: mpsc::Receiver, } impl SpawnQueue for mpsc::Receiver { - fn spawn(mut self, core: Arc) { + fn spawn(self, core: Arc) { tokio::spawn(async move { - let mut queue = Queue::new(core); - - loop { - let on_hold = match tokio::time::timeout(queue.next_wake_up, self.recv()).await { - Ok(Some(QueueEvent::OnHold(on_hold))) => on_hold.into(), - Ok(Some(QueueEvent::Stop)) | Ok(None) => { - break; - } - _ => None, - }; - - queue.process_events().await; - - // Add message on hold - if let Some(on_hold) = on_hold { - queue.on_hold(on_hold); - } - } + Queue::new(core, self).start().await; }); } } impl Queue { - pub fn new(core: Arc) -> Self { + pub fn new(core: Arc, rx: mpsc::Receiver) -> Self { Queue { core, on_hold: Vec::with_capacity(128), - next_wake_up: SHORT_WAIT, + in_flight: AHashSet::with_capacity(128), + next_wake_up: Instant::now(), + rx, } } - pub async fn process_events(&mut self) { - // Deliver any concurrency limited messages - let server = self.core.build_server(); - while let Some(queue_event) = self.next_on_hold() { - DeliveryAttempt::new(queue_event) - .try_deliver(server.clone()) - .await; - } + pub async fn start(&mut self) { + let mut is_paused = false; - // Deliver scheduled messages - let now = now(); - self.next_wake_up = LONG_WAIT; - for queue_event in server.next_event().await { - if queue_event.due <= now { - DeliveryAttempt::new(queue_event) - .try_deliver(server.clone()) - .await; + loop { + let (on_hold, refresh_queue) = match tokio::time::timeout( + self.next_wake_up.duration_since(Instant::now()), + self.rx.recv(), + ) + .await + { + Ok(Some(QueueEvent::Refresh(queue_id))) => { + if let Some(queue_id) = queue_id { + self.in_flight.remove(&queue_id); + } + + (None, true) + } + Ok(Some(QueueEvent::WorkerDone(queue_id))) => { + self.in_flight.remove(&queue_id); + + (None, false) + } + Ok(Some(QueueEvent::OnHold(on_hold))) => { + self.in_flight.remove(&on_hold.message.queue_id); + + (on_hold.into(), false) + } + Ok(Some(QueueEvent::Paused(paused))) => { + is_paused = paused; + (None, false) + } + Err(_) => (None, true), + Ok(Some(QueueEvent::Stop)) | Ok(None) => { + break; + } + }; + + if !is_paused { + // Deliver any concurrency limited messages + let server = self.core.build_server(); + while let Some(queue_event) = self.next_on_hold() { + if let Some(message) = + DeliveryAttempt::new(queue_event).try_deliver(server.clone()) + { + self.on_hold(message); + } else { + self.in_flight.insert(queue_event.queue_id); + } + } + + // Deliver scheduled messages + if refresh_queue || self.next_wake_up <= Instant::now() { + let now = now(); + let mut next_wake_up = QUEUE_REFRESH; + for queue_event in server.next_event().await { + match self.is_on_hold(queue_event.queue_id) { + None => { + if queue_event.due <= now { + if !self.in_flight.contains(&queue_event.queue_id) { + if let Some(message) = DeliveryAttempt::new(queue_event) + .try_deliver(server.clone()) + { + self.on_hold(message); + } else { + self.in_flight.insert(queue_event.queue_id); + } + } + } else { + let due_in = queue_event.due - now; + if due_in < next_wake_up { + next_wake_up = due_in; + } + } + } + Some(on_hold) + if on_hold.limiters.is_empty() + && on_hold.next_due.map_or(false, |due| due < next_wake_up) => + { + next_wake_up = on_hold.next_due.unwrap(); + } + _ => (), + } + } + self.next_wake_up = Instant::now() + Duration::from_secs(next_wake_up); + } } else { - self.next_wake_up = Duration::from_secs(queue_event.due - now); + // Queue is paused + self.next_wake_up = Instant::now() + Duration::from_secs(86400); + } + + // Add message on hold + if let Some(on_hold) = on_hold { + self.on_hold(on_hold); } } } - pub fn on_hold(&mut self, message: OnHold) { + pub fn is_on_hold(&self, queue_id: QueueId) -> Option<&OnHold> { + self.on_hold.iter().find(|o| o.message.queue_id == queue_id) + } + + pub fn on_hold(&mut self, message: OnHold) { self.on_hold.push(OnHold { next_due: message.next_due, limiters: message.limiters, @@ -93,7 +159,7 @@ impl Queue { }); } - pub fn next_on_hold(&mut self) -> Option { + pub fn next_on_hold(&mut self) -> Option { let now = now(); self.on_hold .iter() diff --git a/crates/smtp/src/queue/mod.rs b/crates/smtp/src/queue/mod.rs index fd8d9d8d..c3aa1a5f 100644 --- a/crates/smtp/src/queue/mod.rs +++ b/crates/smtp/src/queue/mod.rs @@ -12,7 +12,7 @@ use std::{ use common::{ expr::{self, functions::ResolveVariable, *}, - ipc::QueueEventLock, + ipc::QueuedMessage, listener::limiter::InFlight, }; use serde::{Deserialize, Serialize}; @@ -133,7 +133,7 @@ pub struct ErrorDetails { pub struct DeliveryAttempt { pub in_flight: Vec, - pub event: QueueEventLock, + pub event: QueuedMessage, } impl Ord for Schedule { diff --git a/crates/smtp/src/queue/spool.rs b/crates/smtp/src/queue/spool.rs index 5c33cc12..d4b6378b 100644 --- a/crates/smtp/src/queue/spool.rs +++ b/crates/smtp/src/queue/spool.rs @@ -5,14 +5,14 @@ */ use crate::queue::DomainPart; -use common::ipc::{QueueEvent, QueueEventLock}; -use common::Server; +use common::ipc::{QueueEvent, QueuedMessage}; +use common::{Server, KV_LOCK_QUEUE_MESSAGE}; use std::borrow::Cow; use std::future::Future; use std::time::{Duration, SystemTime}; use store::write::key::DeserializeBigEndian; use store::write::{now, BatchBuilder, Bincode, BlobOp, QueueClass, ValueClass}; -use store::{Deserialize, IterateParams, Serialize, ValueKey, U64_LEN}; +use store::{IterateParams, Serialize, ValueKey, U64_LEN}; use trc::ServerEvent; use utils::BlobHash; @@ -21,6 +21,7 @@ use super::{ }; pub const LOCK_EXPIRY: u64 = 300; +pub const QUEUE_REFRESH: u64 = 300; pub trait SmtpSpool: Sync + Send { fn new_message( @@ -31,12 +32,11 @@ pub trait SmtpSpool: Sync + Send { span_id: u64, ) -> Message; - fn next_event(&self) -> impl Future> + Send; + fn next_event(&self) -> impl Future> + Send; - fn try_lock_event( - &self, - event: QueueEventLock, - ) -> impl Future> + Send; + fn try_lock_event(&self, queue_id: QueueId) -> impl Future + Send; + + fn unlock_event(&self, queue_id: QueueId) -> impl Future + Send; fn read_message(&self, id: QueueId) -> impl Future> + Send; } @@ -70,7 +70,8 @@ impl SmtpSpool for Server { } } - async fn next_event(&self) -> Vec { + async fn next_event(&self) -> Vec { + let now = now(); let from_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent( store::write::QueueEvent { due: 0, @@ -79,35 +80,24 @@ impl SmtpSpool for Server { ))); let to_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent( store::write::QueueEvent { - due: u64::MAX, + due: now + QUEUE_REFRESH, queue_id: u64::MAX, }, ))); let mut events = Vec::new(); - let now = now(); + let result = self .store() .iterate( - IterateParams::new(from_key, to_key).ascending(), - |key, value| { - let event = QueueEventLock { - due: key.deserialize_be_u64(0)?, - queue_id: key.deserialize_be_u64(U64_LEN)?, - lock_expiry: u64::deserialize(value)?, - }; - let do_continue = event.due <= now; - if event.lock_expiry < now { - events.push(event); - } else { - trc::event!( - Queue(trc::QueueEvent::Locked), - SpanId = event.queue_id, - Due = trc::Value::Timestamp(event.due), - Expires = trc::Value::Timestamp(event.lock_expiry), - ); - } - Ok(do_continue) + IterateParams::new(from_key, to_key).ascending().no_values(), + |key, _| { + let due = key.deserialize_be_u64(0)?; + let queue_id = key.deserialize_be_u64(U64_LEN)?; + + events.push(QueuedMessage { due, queue_id }); + + Ok(due <= now) }, ) .await; @@ -121,44 +111,39 @@ impl SmtpSpool for Server { events } - async fn try_lock_event(&self, mut event: QueueEventLock) -> Option { - let mut batch = BatchBuilder::new(); - batch.assert_value( - ValueClass::Queue(QueueClass::MessageEvent(store::write::QueueEvent { - due: event.due, - queue_id: event.queue_id, - })), - event.lock_expiry, - ); - event.lock_expiry = now() + LOCK_EXPIRY; - batch.set( - ValueClass::Queue(QueueClass::MessageEvent(store::write::QueueEvent { - due: event.due, - queue_id: event.queue_id, - })), - event.lock_expiry.serialize(), - ); - match self.store().write(batch.build()).await { - Ok(_) => Some(event), - Err(err) if err.is_assertion_failure() => { - trc::event!( - Queue(trc::QueueEvent::LockBusy), - SpanId = event.queue_id, - Due = trc::Value::Timestamp(event.due), - CausedBy = err, - ); - - None + async fn try_lock_event(&self, queue_id: QueueId) -> bool { + match self + .in_memory_store() + .try_lock(KV_LOCK_QUEUE_MESSAGE, &queue_id.to_be_bytes(), LOCK_EXPIRY) + .await + { + Ok(result) => { + if !result { + trc::event!(Queue(trc::QueueEvent::Locked), QueueId = queue_id,); + } + result } Err(err) => { trc::error!(err .details("Failed to lock event.") .caused_by(trc::location!())); - None + false } } } + async fn unlock_event(&self, queue_id: QueueId) { + if let Err(err) = self + .in_memory_store() + .remove_lock(KV_LOCK_QUEUE_MESSAGE, &queue_id.to_be_bytes()) + .await + { + trc::error!(err + .details("Failed to unlock event.") + .caused_by(trc::location!())); + } + } + async fn read_message(&self, id: QueueId) -> Option { match self .store() @@ -323,7 +308,7 @@ impl Message { .inner .ipc .queue_tx - .send(QueueEvent::Reload) + .send(QueueEvent::Refresh(None)) .await .is_err() { diff --git a/crates/smtp/src/queue/throttle.rs b/crates/smtp/src/queue/throttle.rs index 7aef6253..2f28210e 100644 --- a/crates/smtp/src/queue/throttle.rs +++ b/crates/smtp/src/queue/throttle.rs @@ -10,7 +10,7 @@ use common::{ config::smtp::Throttle, expr::functions::ResolveVariable, listener::limiter::{ConcurrencyLimiter, InFlight}, - Server, KV_RATE_LIMIT_HASH, + Server, ThrottleKey, KV_RATE_LIMIT_HASH, }; use dashmap::mapref::entry::Entry; use store::write::now; @@ -33,6 +33,8 @@ pub trait IsAllowed: Sync + Send { in_flight: &mut Vec, session_id: u64, ) -> impl Future> + Send; + + fn is_outbound_allowed(&self, in_flight: &mut Vec) -> Result<(), ConcurrencyLimiter>; } impl IsAllowed for Server { @@ -113,6 +115,36 @@ impl IsAllowed for Server { Ok(()) } + + fn is_outbound_allowed(&self, in_flight: &mut Vec) -> Result<(), ConcurrencyLimiter> { + match self.inner.data.smtp_queue_throttle.entry(ThrottleKey { + hash: [u8::MAX; 32], + }) { + Entry::Occupied(mut e) => { + let limiter = e.get_mut(); + if let Some(inflight) = limiter.is_allowed() { + in_flight.push(inflight); + } else { + trc::event!( + Queue(trc::QueueEvent::ConcurrencyLimitExceeded), + Details = "Outbound concurrency limit exceeded", + Limit = self.core.smtp.queue.throttle.outbound_concurrency, + ); + + return Err(limiter.clone()); + } + } + Entry::Vacant(e) => { + let limiter = + ConcurrencyLimiter::new(self.core.smtp.queue.throttle.outbound_concurrency); + if let Some(inflight) = limiter.is_allowed() { + in_flight.push(inflight); + } + e.insert(limiter); + } + } + Ok(()) + } } impl Domain { diff --git a/crates/smtp/src/reporting/dmarc.rs b/crates/smtp/src/reporting/dmarc.rs index 08f74a40..8a6c17bd 100644 --- a/crates/smtp/src/reporting/dmarc.rs +++ b/crates/smtp/src/reporting/dmarc.rs @@ -33,7 +33,7 @@ use crate::{ reporting::SmtpReporting, }; -use super::{AggregateTimestamp, ReportLock, SerializedSize}; +use super::{AggregateTimestamp, SerializedSize}; #[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct DmarcFormat { @@ -646,12 +646,6 @@ impl DmarcReporting for Server { ValueClass::Queue(QueueClass::DmarcReportHeader(report_event.clone())), Bincode::new(entry).serialize(), ); - - // Add lock - builder.set( - ValueClass::Queue(QueueClass::dmarc_lock(&report_event)), - 0u64.serialize(), - ); } // Write entry diff --git a/crates/smtp/src/reporting/mod.rs b/crates/smtp/src/reporting/mod.rs index 92c81777..8ca7f15f 100644 --- a/crates/smtp/src/reporting/mod.rs +++ b/crates/smtp/src/reporting/mod.rs @@ -18,7 +18,7 @@ use mail_auth::{ }; use mail_parser::DateTime; -use store::write::{QueueClass, ReportEvent}; +use store::write::{key::KeySerializer, ReportEvent}; use tokio::io::{AsyncRead, AsyncWrite}; use crate::{ @@ -323,26 +323,25 @@ impl io::Write for SerializedSize { } pub trait ReportLock { - fn tls_lock(event: &ReportEvent) -> Self; - fn dmarc_lock(event: &ReportEvent) -> Self; + fn tls_lock(&self) -> Vec; + fn dmarc_lock(&self) -> Vec; } -impl ReportLock for QueueClass { - fn tls_lock(event: &ReportEvent) -> Self { - QueueClass::TlsReportHeader(ReportEvent { - due: event.due, - policy_hash: 0, - seq_id: 0, - domain: event.domain.clone(), - }) +impl ReportLock for ReportEvent { + fn tls_lock(&self) -> Vec { + KeySerializer::new(self.domain.len() + std::mem::size_of::() + 1) + .write(0u8) + .write(self.due) + .write(self.domain.as_bytes()) + .finalize() } - fn dmarc_lock(event: &ReportEvent) -> Self { - QueueClass::DmarcReportHeader(ReportEvent { - due: event.due, - policy_hash: event.policy_hash, - seq_id: 0, - domain: event.domain.clone(), - }) + fn dmarc_lock(&self) -> Vec { + KeySerializer::new(self.domain.len() + (std::mem::size_of::() * 2) + 1) + .write(1u8) + .write(self.due) + .write(self.policy_hash) + .write(self.domain.as_bytes()) + .finalize() } } diff --git a/crates/smtp/src/reporting/scheduler.rs b/crates/smtp/src/reporting/scheduler.rs index ba1c9fc3..ce0494d9 100644 --- a/crates/smtp/src/reporting/scheduler.rs +++ b/crates/smtp/src/reporting/scheduler.rs @@ -5,7 +5,7 @@ */ use ahash::AHashMap; -use common::{core::BuildServer, ipc::ReportingEvent, Inner, Server}; +use common::{core::BuildServer, ipc::ReportingEvent, Inner, Server, KV_LOCK_QUEUE_REPORT}; use std::{ future::Future, @@ -13,15 +13,17 @@ use std::{ time::{Duration, SystemTime}, }; use store::{ - write::{now, BatchBuilder, QueueClass, ReportEvent, ValueClass}, - Deserialize, IterateParams, Key, Serialize, Store, ValueKey, + write::{now, QueueClass, ReportEvent, ValueClass}, + Deserialize, IterateParams, Store, ValueKey, }; use tokio::sync::mpsc; -use crate::queue::{manager::LONG_WAIT, spool::LOCK_EXPIRY}; +use crate::queue::spool::LOCK_EXPIRY; use super::{dmarc::DmarcReporting, tls::TlsReporting, ReportLock}; +pub const REPORT_REFRESH: Duration = Duration::from_secs(86400); + impl SpawnReport for mpsc::Receiver { fn spawn(mut self, inner: Arc) { tokio::spawn(async move { @@ -41,7 +43,7 @@ impl SpawnReport for mpsc::Receiver { } _ => None, }) - .unwrap_or(LONG_WAIT); + .unwrap_or(REPORT_REFRESH); let server = inner.build_server(); let server_ = server.clone(); @@ -50,8 +52,10 @@ impl SpawnReport for mpsc::Receiver { for report_event in events { match report_event { QueueClass::DmarcReportHeader(event) if event.due <= now => { - if server.try_lock_report(QueueClass::dmarc_lock(&event)).await { + let lock_name = event.dmarc_lock(); + if server.try_lock_report(&lock_name).await { server.send_dmarc_aggregate_report(event).await; + server.unlock_report(&lock_name).await; } } QueueClass::TlsReportHeader(event) if event.due <= now => { @@ -65,11 +69,10 @@ impl SpawnReport for mpsc::Receiver { } for (_, tls_report) in tls_reports { - if server - .try_lock_report(QueueClass::tls_lock(tls_report.first().unwrap())) - .await - { + let lock_name = tls_report.first().unwrap().tls_lock(); + if server.try_lock_report(&lock_name).await { server.send_tls_aggregate_report(tls_report).await; + server.unlock_report(&lock_name).await; } } }); @@ -93,6 +96,7 @@ impl SpawnReport for mpsc::Receiver { } async fn next_report_event(store: Store) -> Vec { + let now = now(); let from_key = ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportHeader( ReportEvent { due: 0, @@ -103,7 +107,7 @@ async fn next_report_event(store: Store) -> Vec { ))); let to_key = ValueKey::from(ValueClass::Queue(QueueClass::TlsReportHeader( ReportEvent { - due: u64::MAX, + due: now + REPORT_REFRESH.as_secs(), policy_hash: 0, seq_id: 0, domain: String::new(), @@ -111,16 +115,11 @@ async fn next_report_event(store: Store) -> Vec { ))); let mut events = Vec::new(); - let now = now(); let result = store .iterate( IterateParams::new(from_key, to_key).ascending().no_values(), |key, _| { let event = ReportEvent::deserialize(key)?; - if event.seq_id == 0 { - // Skip lock - return Ok(true); - } let do_continue = event.due <= now; events.push(if *key.last().unwrap() == 0 { QueueClass::DmarcReportHeader(event) @@ -142,71 +141,48 @@ async fn next_report_event(store: Store) -> Vec { } pub trait LockReport: Sync + Send { - fn try_lock_report(&self, lock: QueueClass) -> impl Future + Send; + fn try_lock_report(&self, lock: &[u8]) -> impl Future + Send; + + fn unlock_report(&self, lock: &[u8]) -> impl Future + Send; } impl LockReport for Server { - async fn try_lock_report(&self, lock: QueueClass) -> bool { - let now = now(); + async fn try_lock_report(&self, key: &[u8]) -> bool { match self - .store() - .get_value::(ValueKey::from(ValueClass::Queue(lock.clone()))) + .in_memory_store() + .try_lock(KV_LOCK_QUEUE_REPORT, key, LOCK_EXPIRY) .await { - Ok(Some(expiry)) => { - if expiry < now { - let mut batch = BatchBuilder::new(); - batch.assert_value(ValueClass::Queue(lock.clone()), expiry); - batch.set( - ValueClass::Queue(lock.clone()), - (now + LOCK_EXPIRY).serialize(), - ); - match self.core.storage.data.write(batch.build()).await { - Ok(_) => true, - Err(err) if err.is_assertion_failure() => { - trc::event!( - OutgoingReport(trc::OutgoingReportEvent::LockBusy), - Expires = trc::Value::Timestamp(expiry), - CausedBy = err, - Key = ValueKey::from(ValueClass::Queue(lock)).serialize(0) - ); - false - } - Err(err) => { - trc::error!(err - .caused_by(trc::location!()) - .details("Failed to lock report")); - - false - } - } - } else { + Ok(result) => { + if !result { trc::event!( OutgoingReport(trc::OutgoingReportEvent::Locked), - Expires = trc::Value::Timestamp(expiry), - Key = ValueKey::from(ValueClass::Queue(lock)).serialize(0) + Expires = trc::Value::Timestamp(LOCK_EXPIRY), + Key = key ); - - false } - } - Ok(None) => { - trc::event!( - OutgoingReport(trc::OutgoingReportEvent::LockDeleted), - Key = ValueKey::from(ValueClass::Queue(lock)).serialize(0) - ); - - false + result } Err(err) => { trc::error!(err - .caused_by(trc::location!()) - .details("Failed to lock report")); - + .details("Failed to lock report.") + .caused_by(trc::location!())); false } } } + + async fn unlock_report(&self, key: &[u8]) { + if let Err(err) = self + .in_memory_store() + .remove_lock(KV_LOCK_QUEUE_REPORT, key) + .await + { + trc::error!(err + .details("Failed to unlock event.") + .caused_by(trc::location!())); + } + } } pub trait ToTimestamp { diff --git a/crates/smtp/src/reporting/tls.rs b/crates/smtp/src/reporting/tls.rs index 70333102..a389c5e3 100644 --- a/crates/smtp/src/reporting/tls.rs +++ b/crates/smtp/src/reporting/tls.rs @@ -34,7 +34,7 @@ use trc::{AddContext, OutgoingReportEvent}; use crate::{queue::RecipientDomain, reporting::SmtpReporting}; -use super::{AggregateTimestamp, ReportLock, SerializedSize}; +use super::{AggregateTimestamp, SerializedSize}; #[derive(Debug, Clone)] pub struct TlsRptOptions { @@ -492,12 +492,6 @@ impl TlsReporting for Server { ValueClass::Queue(QueueClass::TlsReportHeader(report_event.clone())), Bincode::new(entry).serialize(), ); - - // Add lock - builder.set( - ValueClass::Queue(QueueClass::tls_lock(&report_event)), - 0u64.serialize(), - ); } // Write entry @@ -517,7 +511,7 @@ impl TlsReporting for Server { async fn delete_tls_report(&self, events: Vec) { let mut batch = BatchBuilder::new(); - for (pos, event) in events.into_iter().enumerate() { + for event in events { let from_key = ReportEvent { due: event.due, policy_hash: event.policy_hash, @@ -549,11 +543,6 @@ impl TlsReporting for Server { return; } - if pos == 0 { - // Remove lock - batch.clear(ValueClass::Queue(QueueClass::tls_lock(&event))); - } - // Remove report header batch.clear(ValueClass::Queue(QueueClass::TlsReportHeader(event))); } diff --git a/crates/store/src/dispatch/lookup.rs b/crates/store/src/dispatch/lookup.rs index 5c32896a..1a0e46e5 100644 --- a/crates/store/src/dispatch/lookup.rs +++ b/crates/store/src/dispatch/lookup.rs @@ -207,6 +207,11 @@ impl InMemoryStore { .map(|count| count == 1) } + pub async fn remove_lock(&self, prefix: u8, key: &[u8]) -> trc::Result<()> { + self.counter_delete(KeyValue::<()>::build_key(prefix, key)) + .await + } + pub async fn purge_in_memory_store(&self) -> trc::Result<()> { match self { InMemoryStore::Store(store) => { diff --git a/crates/trc/src/event/description.rs b/crates/trc/src/event/description.rs index dbbedf55..0d06806a 100644 --- a/crates/trc/src/event/description.rs +++ b/crates/trc/src/event/description.rs @@ -196,8 +196,7 @@ impl FtsIndexEvent { pub fn description(&self) -> &'static str { match self { FtsIndexEvent::Index => "Full-text search index done", - FtsIndexEvent::Locked => "Full-text search index is locked", - FtsIndexEvent::LockBusy => "Full-text search index lock is busy", + FtsIndexEvent::Locked => "Full-text search index is locked by another process", FtsIndexEvent::BlobNotFound => "Blob not found for full-text indexing", FtsIndexEvent::MetadataNotFound => "Metadata not found for full-text indexing", } @@ -206,8 +205,7 @@ impl FtsIndexEvent { pub fn explain(&self) -> &'static str { match self { FtsIndexEvent::Index => "The full-text search index has been updated", - FtsIndexEvent::Locked => "The full-text search index is locked", - FtsIndexEvent::LockBusy => "The full-text search index lock is busy", + FtsIndexEvent::Locked => "The full-text search index is locked by another process", FtsIndexEvent::BlobNotFound => "The blob was not found for full-text indexing", FtsIndexEvent::MetadataNotFound => "The metadata was not found for full-text indexing", } @@ -714,8 +712,7 @@ impl QueueEvent { pub fn description(&self) -> &'static str { match self { QueueEvent::Rescheduled => "Message rescheduled for delivery", - QueueEvent::LockBusy => "Queue lock is busy", - QueueEvent::Locked => "Queue is locked", + QueueEvent::Locked => "Queue event is locked by another process", QueueEvent::BlobNotFound => "Message blob not found", QueueEvent::RateLimitExceeded => "Rate limit exceeded", QueueEvent::ConcurrencyLimitExceeded => "Concurrency limit exceeded", @@ -731,8 +728,7 @@ impl QueueEvent { pub fn explain(&self) -> &'static str { match self { QueueEvent::Rescheduled => "The message was rescheduled for delivery", - QueueEvent::LockBusy => "The queue lock is busy", - QueueEvent::Locked => "The queue is locked", + QueueEvent::Locked => "The queue event is locked by another process", QueueEvent::BlobNotFound => "The message blob was not found", QueueEvent::RateLimitExceeded => "The queue rate limit was exceeded", QueueEvent::ConcurrencyLimitExceeded => "The queue concurrency limit was exceeded", @@ -817,9 +813,7 @@ impl OutgoingReportEvent { OutgoingReportEvent::NotFound => "Report not found", OutgoingReportEvent::SubmissionError => "Error submitting report", OutgoingReportEvent::NoRecipientsFound => "No recipients found for report", - OutgoingReportEvent::LockBusy => "Report lock is busy", - OutgoingReportEvent::LockDeleted => "Report lock was deleted", - OutgoingReportEvent::Locked => "Report is locked", + OutgoingReportEvent::Locked => "Report is locked by another process", } } @@ -843,9 +837,7 @@ impl OutgoingReportEvent { OutgoingReportEvent::NotFound => "The report was not found", OutgoingReportEvent::SubmissionError => "Error submitting the report", OutgoingReportEvent::NoRecipientsFound => "No recipients found for the report", - OutgoingReportEvent::LockBusy => "The report lock is busy", - OutgoingReportEvent::LockDeleted => "The report lock was deleted", - OutgoingReportEvent::Locked => "The report is locked", + OutgoingReportEvent::Locked => "The report is locked by another process", } } } @@ -1020,7 +1012,7 @@ impl SpamEvent { SpamEvent::RemoteList => "Remote list updated", SpamEvent::RemoteListError => "Error updating remote list", SpamEvent::Train => "Training spam filter", - SpamEvent::TrainBalance => "Balancing spam filter training data", + SpamEvent::TrainBalance => "Spam filter model balance verify", SpamEvent::TrainError => "Error training spam filter", SpamEvent::Classify => "Classifying message for spam", SpamEvent::ClassifyError => "Not enough training data for spam filter", @@ -1033,7 +1025,7 @@ impl SpamEvent { match self { SpamEvent::PyzorError => "An error occurred with Pyzor", SpamEvent::Train => "The spam filter is being trained with the message", - SpamEvent::TrainBalance => "The spam filter training data is being balanced", + SpamEvent::TrainBalance => "The spam filter training data is verified for balance", SpamEvent::TrainError => "An error occurred while training the spam filter", SpamEvent::Classify => "The message is being classified for spam", SpamEvent::ClassifyError => "There is not enough training data for the spam filter", diff --git a/crates/trc/src/event/level.rs b/crates/trc/src/event/level.rs index 692d2cf1..ba48bf73 100644 --- a/crates/trc/src/event/level.rs +++ b/crates/trc/src/event/level.rs @@ -381,7 +381,6 @@ impl EventType { }, EventType::FtsIndex(event) => match event { FtsIndexEvent::Index => Level::Info, - FtsIndexEvent::LockBusy => Level::Warn, FtsIndexEvent::BlobNotFound | FtsIndexEvent::Locked | FtsIndexEvent::MetadataNotFound => Level::Debug, @@ -475,9 +474,7 @@ impl EventType { | QueueEvent::ConcurrencyLimitExceeded | QueueEvent::Rescheduled | QueueEvent::QuotaExceeded => Level::Info, - QueueEvent::LockBusy | QueueEvent::Locked | QueueEvent::BlobNotFound => { - Level::Debug - } + QueueEvent::Locked | QueueEvent::BlobNotFound => Level::Debug, }, EventType::TlsRpt(event) => match event { TlsRptEvent::RecordFetch @@ -510,10 +507,7 @@ impl EventType { | IncomingReportEvent::DecompressError => Level::Info, }, EventType::OutgoingReport(event) => match event { - OutgoingReportEvent::LockBusy - | OutgoingReportEvent::LockDeleted - | OutgoingReportEvent::Locked - | OutgoingReportEvent::NotFound => Level::Info, + OutgoingReportEvent::Locked | OutgoingReportEvent::NotFound => Level::Info, OutgoingReportEvent::SpfReport | OutgoingReportEvent::SpfRateLimited | OutgoingReportEvent::DkimReport diff --git a/crates/trc/src/lib.rs b/crates/trc/src/lib.rs index cd26ace0..763bbab5 100644 --- a/crates/trc/src/lib.rs +++ b/crates/trc/src/lib.rs @@ -238,7 +238,6 @@ pub enum HousekeeperEvent { pub enum FtsIndexEvent { Index, Locked, - LockBusy, BlobNotFound, MetadataNotFound, } @@ -482,7 +481,6 @@ pub enum QueueEvent { QueueDsn, QueueAutogenerated, Rescheduled, - LockBusy, Locked, BlobNotFound, RateLimitExceeded, @@ -525,8 +523,6 @@ pub enum OutgoingReportEvent { NotFound, SubmissionError, NoRecipientsFound, - LockBusy, - LockDeleted, Locked, } diff --git a/crates/trc/src/serializers/binary.rs b/crates/trc/src/serializers/binary.rs index 170c1ed0..af13ec98 100644 --- a/crates/trc/src/serializers/binary.rs +++ b/crates/trc/src/serializers/binary.rs @@ -448,7 +448,6 @@ impl EventType { EventType::Eval(EvalEvent::StoreNotFound) => 140, EventType::FtsIndex(FtsIndexEvent::BlobNotFound) => 141, EventType::FtsIndex(FtsIndexEvent::Index) => 142, - EventType::FtsIndex(FtsIndexEvent::LockBusy) => 143, EventType::FtsIndex(FtsIndexEvent::Locked) => 144, EventType::FtsIndex(FtsIndexEvent::MetadataNotFound) => 145, EventType::Housekeeper(HousekeeperEvent::PurgeAccounts) => 146, @@ -640,8 +639,6 @@ impl EventType { EventType::OutgoingReport(OutgoingReportEvent::DmarcRateLimited) => 332, EventType::OutgoingReport(OutgoingReportEvent::DmarcReport) => 333, EventType::OutgoingReport(OutgoingReportEvent::HttpSubmission) => 334, - EventType::OutgoingReport(OutgoingReportEvent::LockBusy) => 335, - EventType::OutgoingReport(OutgoingReportEvent::LockDeleted) => 336, EventType::OutgoingReport(OutgoingReportEvent::Locked) => 337, EventType::OutgoingReport(OutgoingReportEvent::NoRecipientsFound) => 338, EventType::OutgoingReport(OutgoingReportEvent::NotFound) => 339, @@ -681,7 +678,6 @@ impl EventType { EventType::PushSubscription(PushSubscriptionEvent::Success) => 373, EventType::Queue(QueueEvent::BlobNotFound) => 374, EventType::Queue(QueueEvent::ConcurrencyLimitExceeded) => 375, - EventType::Queue(QueueEvent::LockBusy) => 376, EventType::Queue(QueueEvent::Locked) => 377, EventType::Queue(QueueEvent::QueueAutogenerated) => 378, EventType::Queue(QueueEvent::QueueDsn) => 379, @@ -1016,7 +1012,6 @@ impl EventType { 140 => Some(EventType::Eval(EvalEvent::StoreNotFound)), 141 => Some(EventType::FtsIndex(FtsIndexEvent::BlobNotFound)), 142 => Some(EventType::FtsIndex(FtsIndexEvent::Index)), - 143 => Some(EventType::FtsIndex(FtsIndexEvent::LockBusy)), 144 => Some(EventType::FtsIndex(FtsIndexEvent::Locked)), 145 => Some(EventType::FtsIndex(FtsIndexEvent::MetadataNotFound)), 146 => Some(EventType::Housekeeper(HousekeeperEvent::PurgeAccounts)), @@ -1234,8 +1229,6 @@ impl EventType { 334 => Some(EventType::OutgoingReport( OutgoingReportEvent::HttpSubmission, )), - 335 => Some(EventType::OutgoingReport(OutgoingReportEvent::LockBusy)), - 336 => Some(EventType::OutgoingReport(OutgoingReportEvent::LockDeleted)), 337 => Some(EventType::OutgoingReport(OutgoingReportEvent::Locked)), 338 => Some(EventType::OutgoingReport( OutgoingReportEvent::NoRecipientsFound, @@ -1285,7 +1278,6 @@ impl EventType { 373 => Some(EventType::PushSubscription(PushSubscriptionEvent::Success)), 374 => Some(EventType::Queue(QueueEvent::BlobNotFound)), 375 => Some(EventType::Queue(QueueEvent::ConcurrencyLimitExceeded)), - 376 => Some(EventType::Queue(QueueEvent::LockBusy)), 377 => Some(EventType::Queue(QueueEvent::Locked)), 378 => Some(EventType::Queue(QueueEvent::QueueAutogenerated)), 379 => Some(EventType::Queue(QueueEvent::QueueDsn)), diff --git a/tests/Cargo.toml b/tests/Cargo.toml index e877b636..7da0e584 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -6,7 +6,7 @@ resolver = "2" [features] default = ["sqlite", "postgres", "mysql", "rocks", "elastic", "s3", "redis", "azure", "foundationdb"] -#default = ["rocks"] +#default = ["rocks", "sqlite"] sqlite = ["store/sqlite"] foundationdb = ["store/foundation", "common/foundation"] postgres = ["store/postgres"] diff --git a/tests/src/imap/mod.rs b/tests/src/imap/mod.rs index 584228f8..b23462d6 100644 --- a/tests/src/imap/mod.rs +++ b/tests/src/imap/mod.rs @@ -107,6 +107,9 @@ directory = "'{STORE}'" total = 5 wait = "1ms" +[spam-filter] +enable = false + [queue] path = "{TMP}" hash = 64 diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index bdad1e1a..fd384a42 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -338,6 +338,9 @@ signature-algorithm = "ES256" expn = true vrfy = true +[spam-filter] +enable = false + [tracer.console] type = "console" level = "{LEVEL}" diff --git a/tests/src/smtp/inbound/antispam.rs b/tests/src/smtp/inbound/antispam.rs index d35de182..52d783e4 100644 --- a/tests/src/smtp/inbound/antispam.rs +++ b/tests/src/smtp/inbound/antispam.rs @@ -47,7 +47,7 @@ use utils::config::Config; use crate::{ http_server::{spawn_mock_http_server, HttpMessage}, - //jmap::enterprise::EnterpriseCore, + jmap::enterprise::EnterpriseCore, smtp::{session::TestSession, TempDir, TestSMTP}, }; @@ -121,33 +121,6 @@ allow-invalid-certs = true "hta" = "BAD|NZ" } "#; -pub trait EnterpriseCore { - fn enable_enterprise(self) -> Self; -} - -impl EnterpriseCore for Core { - fn enable_enterprise(mut self) -> Self { - let todo = "remove"; - self.enterprise = common::enterprise::Enterprise { - license: common::enterprise::license::LicenseKey { - valid_to: store::write::now() + 3600, - valid_from: store::write::now() - 3600, - domain: String::new(), - accounts: 100, - }, - undelete: None, - trace_store: None, - metrics_store: None, - metrics_alerts: vec![], - logo_url: None, - ai_apis: Default::default(), - spam_filter_llm: None, - } - .into(); - self - } -} - #[tokio::test(flavor = "multi_thread")] async fn antispam() { // Enable logging diff --git a/tests/src/smtp/inbound/mod.rs b/tests/src/smtp/inbound/mod.rs index e8be409d..ded08466 100644 --- a/tests/src/smtp/inbound/mod.rs +++ b/tests/src/smtp/inbound/mod.rs @@ -7,7 +7,7 @@ use std::time::Duration; use common::{ - ipc::{DmarcEvent, OnHold, QueueEvent, QueueEventLock, ReportingEvent, TlsEvent}, + ipc::{DmarcEvent, OnHold, QueueEvent, QueuedMessage, ReportingEvent, TlsEvent}, Server, }; use store::{ @@ -117,12 +117,12 @@ impl QueueReceiver { } pub async fn expect_message(&mut self) -> Message { - self.read_event().await.assert_reload(); + self.read_event().await.assert_refresh(); self.last_queued_message().await } pub async fn consume_message(&mut self, server: &Server) -> Message { - self.read_event().await.assert_reload(); + self.read_event().await.assert_refresh(); let message = self.last_queued_message().await; message .clone() @@ -138,10 +138,9 @@ impl QueueReceiver { } pub async fn delivery_attempt(&mut self, queue_id: u64) -> DeliveryAttempt { - DeliveryAttempt::new(QueueEventLock { + DeliveryAttempt::new(QueuedMessage { due: self.message_due(queue_id).await, queue_id, - lock_expiry: 0, }) } @@ -300,19 +299,35 @@ impl ReportReceiver { } pub trait TestQueueEvent { - fn assert_reload(self); - fn unwrap_on_hold(self) -> OnHold; + fn assert_refresh(self); + fn assert_done(self); + fn assert_refresh_or_done(self); + fn unwrap_on_hold(self) -> OnHold; } impl TestQueueEvent for QueueEvent { - fn assert_reload(self) { + fn assert_refresh(self) { match self { - QueueEvent::Reload => (), + QueueEvent::Refresh(_) => (), e => panic!("Unexpected event: {e:?}"), } } - fn unwrap_on_hold(self) -> OnHold { + fn assert_done(self) { + match self { + QueueEvent::WorkerDone(_) => (), + e => panic!("Unexpected event: {e:?}"), + } + } + + fn assert_refresh_or_done(self) { + match self { + QueueEvent::Refresh(_) | QueueEvent::WorkerDone(_) => (), + e => panic!("Unexpected event: {e:?}"), + } + } + + fn unwrap_on_hold(self) -> OnHold { match self { QueueEvent::OnHold(value) => value, e => panic!("Unexpected event: {e:?}"), diff --git a/tests/src/smtp/inbound/scripts.rs b/tests/src/smtp/inbound/scripts.rs index 4240939b..72edc4cd 100644 --- a/tests/src/smtp/inbound/scripts.rs +++ b/tests/src/smtp/inbound/scripts.rs @@ -242,8 +242,8 @@ async fn sieve_scripts() { "250", ) .await; - qr.read_event().await.assert_reload(); - qr.read_event().await.assert_reload(); + qr.read_event().await.assert_refresh(); + qr.read_event().await.assert_refresh(); let messages = qr.read_queued_messages().await; assert_eq!(messages.len(), 2); let mut messages = messages.into_iter(); @@ -289,8 +289,8 @@ async fn sieve_scripts() { "250", ) .await; - qr.read_event().await.assert_reload(); - qr.read_event().await.assert_reload(); + qr.read_event().await.assert_refresh(); + qr.read_event().await.assert_refresh(); let messages = qr.read_queued_messages().await; assert_eq!(messages.len(), 2); let mut messages = messages.into_iter(); diff --git a/tests/src/smtp/mod.rs b/tests/src/smtp/mod.rs index bf70df35..08074ef0 100644 --- a/tests/src/smtp/mod.rs +++ b/tests/src/smtp/mod.rs @@ -4,12 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::path::PathBuf; +use std::{path::PathBuf, sync::Arc}; use common::{ config::server::{Listeners, ServerProtocol}, ipc::{QueueEvent, ReportingEvent}, - manager::boot::build_ipc, + manager::boot::{build_ipc, IpcReceivers}, Core, Data, Inner, Server, }; @@ -25,7 +25,7 @@ use crate::AssertConfig; pub mod config; pub mod inbound; pub mod lookup; -//pub mod management; +pub mod management; pub mod outbound; pub mod queue; pub mod reporting; @@ -144,6 +144,20 @@ impl TestSMTP { Self::from_core_and_tempdir(core, Default::default(), None) } + pub fn inner_with_rxs(&self) -> (Arc, IpcReceivers) { + let (ipc, ipc_rxs) = build_ipc(); + + ( + Inner { + shared_core: self.server.core.as_ref().clone().into_shared(), + data: Default::default(), + ipc, + } + .into(), + ipc_rxs, + ) + } + fn from_core_and_tempdir(core: Core, data: Data, temp_dir: Option) -> Self { let store = core.storage.data.clone(); let blob_store = core.storage.blob.clone(); diff --git a/tests/src/smtp/outbound/dane.rs b/tests/src/smtp/outbound/dane.rs index 4bc3d853..bca3a3bd 100644 --- a/tests/src/smtp/outbound/dane.rs +++ b/tests/src/smtp/outbound/dane.rs @@ -120,8 +120,7 @@ async fn dane_verify() { .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()) - .await; + .try_deliver(core.clone()); local .queue_receiver .expect_message() @@ -130,7 +129,7 @@ async fn dane_verify() { .await .assert_contains(" (DANE failed to authenticate") .assert_contains("No TLSA records found"); - local.queue_receiver.read_event().await.assert_reload(); + local.queue_receiver.read_event().await.assert_done(); local.queue_receiver.assert_no_events(); // Expect TLS failure report @@ -173,8 +172,7 @@ async fn dane_verify() { .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()) - .await; + .try_deliver(core.clone()); local .queue_receiver .expect_message() @@ -183,7 +181,7 @@ async fn dane_verify() { .await .assert_contains(" (DANE failed to authenticate") .assert_contains("No matching certificates found"); - local.queue_receiver.read_event().await.assert_reload(); + local.queue_receiver.read_event().await.assert_done(); local.queue_receiver.assert_no_events(); // Expect TLS failure report @@ -221,9 +219,8 @@ async fn dane_verify() { .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()) - .await; - local.queue_receiver.read_event().await.assert_reload(); + .try_deliver(core.clone()); + local.queue_receiver.read_event().await.assert_done(); local.queue_receiver.assert_no_events(); remote .queue_receiver diff --git a/tests/src/smtp/outbound/extensions.rs b/tests/src/smtp/outbound/extensions.rs index e434769c..87eb7881 100644 --- a/tests/src/smtp/outbound/extensions.rs +++ b/tests/src/smtp/outbound/extensions.rs @@ -92,8 +92,7 @@ async fn extensions() { .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()) - .await; + .try_deliver(core.clone()); local .queue_receiver @@ -104,7 +103,7 @@ async fn extensions() { .assert_contains(" (delivered to") .assert_contains("Final-Recipient: rfc822;bill@foobar.org") .assert_contains("Action: delivered"); - local.queue_receiver.read_event().await.assert_reload(); + local.queue_receiver.read_event().await.assert_done(); remote .queue_receiver .expect_message() @@ -121,8 +120,7 @@ async fn extensions() { .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()) - .await; + .try_deliver(core.clone()); local .queue_receiver .expect_message() @@ -133,7 +131,7 @@ async fn extensions() { .assert_contains("Action: failed") .assert_contains("Diagnostic-Code: smtp;552") .assert_contains("Status: 5.3.4"); - local.queue_receiver.read_event().await.assert_reload(); + local.queue_receiver.read_event().await.assert_done(); remote.queue_receiver.assert_no_events(); // Test DSN, SMTPUTF8 and REQUIRETLS extensions @@ -149,9 +147,8 @@ async fn extensions() { .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()) - .await; - local.queue_receiver.read_event().await.assert_reload(); + .try_deliver(core.clone()); + local.queue_receiver.read_event().await.assert_done(); let message = remote.queue_receiver.expect_message().await; assert_eq!(message.env_id, Some("abc123".to_string())); assert!((message.flags & MAIL_RET_HDRS) != 0); diff --git a/tests/src/smtp/outbound/fallback_relay.rs b/tests/src/smtp/outbound/fallback_relay.rs index 7c7a8831..7947ec22 100644 --- a/tests/src/smtp/outbound/fallback_relay.rs +++ b/tests/src/smtp/outbound/fallback_relay.rs @@ -91,8 +91,7 @@ async fn fallback_relay() { .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()) - .await; + .try_deliver(core.clone()); let mut retry = local.queue_receiver.expect_message().await; let prev_due = retry.domains[0].retry.due; let next_due = now(); @@ -105,8 +104,7 @@ async fn fallback_relay() { .queue_receiver .delivery_attempt(queue_id) .await - .try_deliver(core.clone()) - .await; + .try_deliver(core.clone()); tokio::time::sleep(Duration::from_millis(100)).await; remote.queue_receiver.expect_message().await; } diff --git a/tests/src/smtp/outbound/ip_lookup.rs b/tests/src/smtp/outbound/ip_lookup.rs index d19fc2d4..f8f9f33e 100644 --- a/tests/src/smtp/outbound/ip_lookup.rs +++ b/tests/src/smtp/outbound/ip_lookup.rs @@ -75,8 +75,7 @@ async fn ip_lookup_strategy() { .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()) - .await; + .try_deliver(core.clone()); tokio::time::sleep(Duration::from_millis(100)).await; if matches!(strategy, IpLookupStrategy::Ipv6thenIpv4) { remote.queue_receiver.expect_message().await; diff --git a/tests/src/smtp/outbound/lmtp.rs b/tests/src/smtp/outbound/lmtp.rs index f47e40d9..5e7a26bb 100644 --- a/tests/src/smtp/outbound/lmtp.rs +++ b/tests/src/smtp/outbound/lmtp.rs @@ -103,13 +103,12 @@ async fn lmtp_delivery() { .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()) - .await; + .try_deliver(core.clone()); let mut dsn = Vec::new(); loop { match local.queue_receiver.try_read_event().await { - Some(QueueEvent::Reload) => {} - Some(QueueEvent::OnHold(_)) => unreachable!(), + Some(QueueEvent::Refresh(_) | QueueEvent::WorkerDone(_)) => {} + Some(QueueEvent::OnHold(_)) | Some(QueueEvent::Paused(_)) => unreachable!(), None | Some(QueueEvent::Stop) => break, } @@ -128,7 +127,7 @@ async fn lmtp_delivery() { message.clone().remove(&core, event.due).await; dsn.push(message); } else { - DeliveryAttempt::new(event).try_deliver(core.clone()).await; + DeliveryAttempt::new(event).try_deliver(core.clone()); tokio::time::sleep(Duration::from_millis(100)).await; } } diff --git a/tests/src/smtp/outbound/mta_sts.rs b/tests/src/smtp/outbound/mta_sts.rs index 29e8a605..c05df24d 100644 --- a/tests/src/smtp/outbound/mta_sts.rs +++ b/tests/src/smtp/outbound/mta_sts.rs @@ -101,8 +101,7 @@ async fn mta_sts_verify() { .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()) - .await; + .try_deliver(core.clone()); local .queue_receiver .expect_message() @@ -111,7 +110,7 @@ async fn mta_sts_verify() { .await .assert_contains(" (MTA-STS failed to authenticate") .assert_contains("Record not found"); - local.queue_receiver.read_event().await.assert_reload(); + local.queue_receiver.read_event().await.assert_done(); // Expect TLS failure report let report = local.report_receiver.read_report().await.unwrap_tls(); @@ -139,8 +138,7 @@ async fn mta_sts_verify() { .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()) - .await; + .try_deliver(core.clone()); local .queue_receiver .expect_message() @@ -149,7 +147,7 @@ async fn mta_sts_verify() { .await .assert_contains(" (MTA-STS failed to authenticate") .assert_contains("No 'mx' entries found"); - local.queue_receiver.read_event().await.assert_reload(); + local.queue_receiver.read_event().await.assert_done(); // Expect TLS failure report let report = local.report_receiver.read_report().await.unwrap_tls(); @@ -174,8 +172,7 @@ async fn mta_sts_verify() { .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()) - .await; + .try_deliver(core.clone()); local .queue_receiver .expect_message() @@ -184,7 +181,7 @@ async fn mta_sts_verify() { .await .assert_contains(" (MTA-STS failed to authenticate") .assert_contains("not authorized by policy"); - local.queue_receiver.read_event().await.assert_reload(); + local.queue_receiver.read_event().await.assert_done(); // Expect TLS failure report let report = local.report_receiver.read_report().await.unwrap_tls(); @@ -225,9 +222,8 @@ async fn mta_sts_verify() { .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()) - .await; - local.queue_receiver.read_event().await.assert_reload(); + .try_deliver(core.clone()); + local.queue_receiver.read_event().await.assert_done(); remote .queue_receiver .expect_message() diff --git a/tests/src/smtp/outbound/smtp.rs b/tests/src/smtp/outbound/smtp.rs index 13b2f7ff..0abed03e 100644 --- a/tests/src/smtp/outbound/smtp.rs +++ b/tests/src/smtp/outbound/smtp.rs @@ -131,14 +131,13 @@ async fn smtp_delivery() { .queue_receiver .delivery_attempt(message.queue_id) .await - .try_deliver(core.clone()) - .await; + .try_deliver(core.clone()); let mut dsn = Vec::new(); let mut domain_retries = vec![0; num_domains]; loop { match local.queue_receiver.try_read_event().await { - Some(QueueEvent::Reload) => {} - Some(QueueEvent::OnHold(_)) => unreachable!(), + Some(QueueEvent::Refresh(_) | QueueEvent::WorkerDone(_)) => {} + Some(QueueEvent::OnHold(_)) | Some(QueueEvent::Paused(_)) => unreachable!(), None | Some(QueueEvent::Stop) => break, } @@ -160,7 +159,7 @@ async fn smtp_delivery() { for (idx, domain) in message.domains.iter().enumerate() { domain_retries[idx] = domain.retry.inner; } - DeliveryAttempt::new(event).try_deliver(core.clone()).await; + DeliveryAttempt::new(event).try_deliver(core.clone()); tokio::time::sleep(Duration::from_millis(100)).await; } } @@ -259,9 +258,12 @@ async fn smtp_delivery() { .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()) - .await; - local.queue_receiver.read_event().await.assert_reload(); + .try_deliver(core.clone()); + local + .queue_receiver + .read_event() + .await + .assert_refresh_or_done(); let message = remote .queue_receiver diff --git a/tests/src/smtp/outbound/throttle.rs b/tests/src/smtp/outbound/throttle.rs index 9c625a51..53c87b32 100644 --- a/tests/src/smtp/outbound/throttle.rs +++ b/tests/src/smtp/outbound/throttle.rs @@ -107,8 +107,7 @@ async fn throttle_outbound() { .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()) - .await; + .try_deliver(core.clone()); tokio::time::sleep(Duration::from_millis(100)).await; local.queue_receiver.read_event().await.unwrap_on_hold(); in_flight.clear(); @@ -133,10 +132,9 @@ async fn throttle_outbound() { .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()) - .await; + .try_deliver(core.clone()); tokio::time::sleep(Duration::from_millis(100)).await; - local.queue_receiver.read_event().await.assert_reload(); + local.queue_receiver.read_event().await.assert_refresh(); let due = local.queue_receiver.last_queued_due().await - now(); assert!(due > 0, "Due: {}", due); @@ -172,8 +170,7 @@ async fn throttle_outbound() { .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()) - .await; + .try_deliver(core.clone()); tokio::time::sleep(Duration::from_millis(100)).await; local.queue_receiver.read_event().await.unwrap_on_hold(); in_flight.clear(); @@ -209,10 +206,9 @@ async fn throttle_outbound() { .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()) - .await; + .try_deliver(core.clone()); tokio::time::sleep(Duration::from_millis(100)).await; - local.queue_receiver.read_event().await.assert_reload(); + local.queue_receiver.read_event().await.assert_refresh(); let due = local.queue_receiver.last_queued_due().await - now(); assert!(due > 0, "Due: {}", due); @@ -255,8 +251,7 @@ async fn throttle_outbound() { .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()) - .await; + .try_deliver(core.clone()); local.queue_receiver.read_event().await.unwrap_on_hold(); in_flight.clear(); @@ -292,11 +287,10 @@ async fn throttle_outbound() { .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()) - .await; + .try_deliver(core.clone()); tokio::time::sleep(Duration::from_millis(100)).await; - local.queue_receiver.read_event().await.assert_reload(); + local.queue_receiver.read_event().await.assert_refresh(); let due = local.queue_receiver.last_queued_due().await - now(); assert!(due > 0, "Due: {}", due); } diff --git a/tests/src/smtp/outbound/tls.rs b/tests/src/smtp/outbound/tls.rs index 105c7f3e..d78b5d51 100644 --- a/tests/src/smtp/outbound/tls.rs +++ b/tests/src/smtp/outbound/tls.rs @@ -80,8 +80,7 @@ async fn starttls_optional() { .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()) - .await; + .try_deliver(core.clone()); let mut retry = local.queue_receiver.expect_message().await; let prev_due = retry.domains[0].retry.due; let next_due = now(); @@ -94,8 +93,7 @@ async fn starttls_optional() { .queue_receiver .delivery_attempt(queue_id) .await - .try_deliver(core.clone()) - .await; + .try_deliver(core.clone()); tokio::time::sleep(Duration::from_millis(100)).await; remote .queue_receiver diff --git a/tests/src/smtp/queue/concurrent.rs b/tests/src/smtp/queue/concurrent.rs index ef66190f..2cc84340 100644 --- a/tests/src/smtp/queue/concurrent.rs +++ b/tests/src/smtp/queue/concurrent.rs @@ -6,18 +6,24 @@ use std::time::{Duration, Instant}; -use common::config::server::ServerProtocol; +use common::{config::server::ServerProtocol, ipc::QueueEvent}; use mail_auth::MX; use crate::smtp::{session::TestSession, TestSMTP}; use smtp::queue::manager::Queue; const LOCAL: &str = r#" +[spam-filter] +enable = false + [session.rcpt] relay = true [session.data.limits] -messages = 200 +messages = 2000 + +[queue.outbound] +concurrency = 1 "#; @@ -28,6 +34,9 @@ reject-non-fqdn = false [session.rcpt] relay = true +[spam-filter] +enable = false + "#; #[tokio::test] @@ -63,27 +72,35 @@ async fn concurrent_queue() { session.eval_session_params().await; session.ehlo("mx.test.org").await; - // Send 100 test messages + // Spawn 20 concurrent queues + let mut inners = vec![]; + for _ in 0..20 { + let (inner, rxs) = local.inner_with_rxs(); + inners.push(inner.clone()); + tokio::spawn(async move { + Queue::new(inner, rxs.queue_rx.unwrap()).start().await; + }); + } + + tokio::time::sleep(Duration::from_millis(200)).await; + + // Send 1000 test messages for _ in 0..100 { session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; } - // Spawn 20 concurrent queues at different times - for _ in 0..10 { - let local = local.server.clone(); - tokio::spawn(async move { - Queue::new(local.inner).process_events().await; - }); - } - tokio::time::sleep(Duration::from_millis(500)).await; - for _ in 0..10 { - let local = local.server.clone(); - tokio::spawn(async move { - Queue::new(local.inner).process_events().await; - }); + // Wake up all queues + for inner in &inners { + inner + .ipc + .queue_tx + .send(QueueEvent::Refresh(None)) + .await + .unwrap(); } + tokio::time::sleep(Duration::from_millis(1500)).await; local.queue_receiver.assert_queue_is_empty().await; diff --git a/tests/src/smtp/queue/retry.rs b/tests/src/smtp/queue/retry.rs index 090ca34e..8b97d3fa 100644 --- a/tests/src/smtp/queue/retry.rs +++ b/tests/src/smtp/queue/retry.rs @@ -11,6 +11,7 @@ use crate::smtp::{ session::{TestSession, VerifyResponse}, TestSMTP, }; +use ahash::AHashSet; use common::ipc::QueueEvent; use smtp::queue::{spool::SmtpSpool, DeliveryAttempt}; use store::write::now; @@ -56,7 +57,7 @@ async fn queue_retry() { let attempt = qr.expect_message_then_deliver().await; // Expect a failed DSN - attempt.try_deliver(core.clone()).await; + attempt.try_deliver(core.clone()); let message = qr.expect_message().await; assert_eq!(message.return_path, ""); assert_eq!(message.domains.first().unwrap().domain, "test.org"); @@ -67,7 +68,7 @@ async fn queue_retry() { .assert_contains("Content-Type: multipart/report") .assert_contains("Final-Recipient: rfc822;bill@foobar.org") .assert_contains("Action: failed"); - qr.read_event().await.assert_reload(); + qr.read_event().await.assert_done(); qr.clear_queue(&core).await; // Expect a failed DSN for foobar.org, followed by two delayed DSN and @@ -80,23 +81,35 @@ async fn queue_retry() { "250", ) .await; + let mut in_fight = AHashSet::new(); let attempt = qr.expect_message_then_deliver().await; let mut dsn = Vec::new(); let mut retries = Vec::new(); - attempt.try_deliver(core.clone()).await; + in_fight.insert(attempt.event.queue_id); + attempt.try_deliver(core.clone()); + loop { match qr.try_read_event().await { - Some(QueueEvent::Reload) => {} - Some(QueueEvent::OnHold(_)) => unreachable!(), - None | Some(QueueEvent::Stop) => break, + Some(QueueEvent::Refresh(Some(queue_id)) | QueueEvent::WorkerDone(queue_id)) => { + in_fight.remove(&queue_id); + } + Some(QueueEvent::OnHold(event)) => { + panic!("unexpected on hold event: {:?}", event); + } + Some(QueueEvent::Refresh(None)) => (), + None | Some(QueueEvent::Stop) | Some(QueueEvent::Paused(_)) => break, } let now = now(); let events = core.next_event().await; - if events.is_empty() { + + if events.is_empty() && in_fight.is_empty() { break; } for event in events { + if in_fight.contains(&event.queue_id) { + continue; + } if event.due > now { tokio::time::sleep(Duration::from_secs(event.due - now)).await; } @@ -106,8 +119,11 @@ async fn queue_retry() { message.clone().remove(&core, event.due).await; dsn.push(message); } else { - retries.push(event.due - now); - DeliveryAttempt::new(event).try_deliver(core.clone()).await; + retries.push(event.due.saturating_sub(now)); + in_fight.insert(event.queue_id); + assert!(DeliveryAttempt::new(event) + .try_deliver(core.clone()) + .is_none()); tokio::time::sleep(Duration::from_millis(100)).await; } } diff --git a/tests/src/smtp/reporting/analyze.rs b/tests/src/smtp/reporting/analyze.rs index 529f7e58..bd971bca 100644 --- a/tests/src/smtp/reporting/analyze.rs +++ b/tests/src/smtp/reporting/analyze.rs @@ -116,6 +116,6 @@ async fn report_analyze() { session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; - qr.read_event().await.assert_reload(); + qr.read_event().await.assert_refresh(); qr.last_queued_message().await; }