diff --git a/Cargo.lock b/Cargo.lock index fcd08e7b..6f9036d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,9 +70,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a824f2aa7e75a0c98c5a504fceb80649e9c35265d44525b5f94de4771a395cd" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ "getrandom", "once_cell", @@ -81,9 +81,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" +checksum = "42cd52102d3df161c77a887b608d7a4897d7cc112886a9537b738a887a03aaff" dependencies = [ "cfg-if", "getrandom", @@ -139,9 +139,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2faccea4cc4ab4a667ce676a30e8ec13922a692c99bb8f5b11f1502c72e04220" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" [[package]] name = "anstyle-parse" @@ -497,16 +497,16 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.69.2" +version = "0.69.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4c69fae65a523209d34240b60abe0c42d33d1045d445c0839d8a4894a736e2d" +checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" dependencies = [ "bitflags 2.4.2", "cexpr", "clang-sys", + "itertools 0.12.1", "lazy_static", "lazycell", - "peeking_take_while", "proc-macro2", "quote", "regex", @@ -685,9 +685,9 @@ checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "bytecheck" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6372023ac861f6e6dc89c8344a8f398fb42aaba2b5dbc649ca0c0e9dbcb627" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" dependencies = [ "bytecheck_derive", "ptr_meta", @@ -696,9 +696,9 @@ dependencies = [ [[package]] name = "bytecheck_derive" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7ec4c6f261935ad534c0c22dbef2201b45918860eb1c574b972bd213a76af61" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" dependencies = [ "proc-macro2", "quote", @@ -707,9 +707,9 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.14.1" +version = "1.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2490600f404f2b94c167e31d3ed1d5f3c225a0f3b80230053b3e0b7b962bd9" +checksum = "a2ef034f05691a48569bd920a96c81b9d91bbad1ab5ac7c4616c1f6ef36cb79f" [[package]] name = "byteorder" @@ -834,9 +834,9 @@ checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" [[package]] name = "chrono" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f13690e35a5e4ace198e7beea2895d29f3a9cc55015fcebe6336bd2010af9eb" +checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" dependencies = [ "android-tzdata", "iana-time-zone", @@ -880,9 +880,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.18" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e578d6ec4194633722ccf9544794b71b1385c3c027efe0c55db226fc880865c" +checksum = "80c21025abd42669a92efc996ef13cfb2c5c627858421ea58d5c3b331a6c134f" dependencies = [ "clap_builder", "clap_derive", @@ -890,21 +890,21 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.18" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7" +checksum = "458bf1f341769dfcf849846f65dffdf9146daa56bcd2a47cb4e1de9915567c99" dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim", + "strsim 0.11.0", ] [[package]] name = "clap_derive" -version = "4.4.7" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +checksum = "307bc0538d5f0f83b8248db3087aa92fe504e4691294d0c96c0eabc33f47ba47" dependencies = [ "heck", "proc-macro2", @@ -914,9 +914,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" [[package]] name = "cmac" @@ -1028,9 +1028,9 @@ checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff" [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" dependencies = [ "cfg-if", ] @@ -1162,9 +1162,9 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.1.1" +version = "4.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89b8c6a2e4b1f45971ad09761aafb85514a84744b67a95e32c3cc1352d1f65c" +checksum = "0a677b8922c94e01bdbb12126b0bc852f00447528dee1782229af9c720c3f348" dependencies = [ "cfg-if", "cpufeatures", @@ -1218,7 +1218,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", + "strsim 0.10.0", "syn 1.0.109", ] @@ -1232,7 +1232,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", + "strsim 0.10.0", "syn 2.0.48", ] @@ -1413,7 +1413,7 @@ dependencies = [ name = "directory" version = "0.1.0" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "argon2", "async-trait", "deadpool", @@ -1596,9 +1596,9 @@ dependencies = [ [[package]] name = "ed25519-dalek" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f628eaec48bfd21b865dc2950cfa014450c01d2fa2b69a86c2fd5844ec523c0" +checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871" dependencies = [ "curve25519-dalek", "ed25519", @@ -1611,9 +1611,9 @@ dependencies = [ [[package]] name = "either" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" [[package]] name = "elasticsearch" @@ -1773,9 +1773,9 @@ dependencies = [ [[package]] name = "fiat-crypto" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27573eac26f4dd11e2b1916c3fe1baa56407c83c71a773a8ba17ec0bca03b6b7" +checksum = "1676f435fc1dadde4d03e43f5d62b259e1ce5f40bd4ffb21db2b42ebe59c1382" [[package]] name = "filetime" @@ -2153,7 +2153,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.11", - "indexmap 2.2.1", + "indexmap 2.2.3", "slab", "tokio", "tokio-util", @@ -2172,7 +2172,7 @@ dependencies = [ "futures-sink", "futures-util", "http 1.0.0", - "indexmap 2.2.1", + "indexmap 2.2.3", "slab", "tokio", "tokio-util", @@ -2185,7 +2185,7 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" dependencies = [ - "ahash 0.7.7", + "ahash 0.7.8", ] [[package]] @@ -2194,7 +2194,7 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "allocator-api2", ] @@ -2215,9 +2215,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" +checksum = "d0c62115964e08cb8039170eb33c1d0e2388a256930279edca206fff675f82c3" [[package]] name = "hex" @@ -2485,9 +2485,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.59" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -2549,9 +2549,9 @@ checksum = "029d73f573d8e8d63e6d5020011d3255b28c3ba85d6cf870a07184ed23de9284" [[package]] name = "imap" -version = "0.5.3" +version = "0.6.0" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "dashmap", "directory", "imap_proto", @@ -2576,7 +2576,7 @@ dependencies = [ name = "imap_proto" version = "0.1.0" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "chrono", "jmap_proto", "mail-parser", @@ -2596,9 +2596,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.1" +version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433de089bd45971eecf4668ee0ee8f4cec17db4f8bd8f7bc3197a6ce37aa7d9b" +checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -2606,9 +2606,9 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.17.7" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" +checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3" dependencies = [ "console", "instant", @@ -2665,12 +2665,12 @@ checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "is-terminal" -version = "0.4.10" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" dependencies = [ "hermit-abi", - "rustix", + "libc", "windows-sys 0.52.0", ] @@ -2683,6 +2683,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.10" @@ -2726,7 +2735,7 @@ dependencies = [ [[package]] name = "jmap" -version = "0.5.3" +version = "0.6.0" dependencies = [ "aes", "aes-gcm", @@ -2783,7 +2792,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12c697483ad894a8184d0fd61848e057f86b16642049993b3e6a80c959dbc90a" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "async-stream", "base64 0.13.1", "chrono", @@ -2803,7 +2812,7 @@ dependencies = [ name = "jmap_proto" version = "0.1.0" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "fast-float", "mail-parser", "serde", @@ -2816,18 +2825,18 @@ dependencies = [ [[package]] name = "jobserver" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" +checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.67" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" +checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee" dependencies = [ "wasm-bindgen", ] @@ -2838,7 +2847,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d" dependencies = [ - "indexmap 2.2.1", + "indexmap 2.2.3", ] [[package]] @@ -2872,7 +2881,7 @@ dependencies = [ "diff", "ena", "is-terminal", - "itertools", + "itertools 0.10.5", "lalrpop-util", "petgraph", "regex", @@ -3083,7 +3092,7 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "224aa436a40caeef3bd3fa1b5b619b28b26d83fcc088c008536886f74ad27951" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "flate2", "hickory-resolver", "lru-cache", @@ -3131,12 +3140,12 @@ dependencies = [ "smtp-proto", "tokio", "tokio-rustls 0.25.0", - "webpki-roots 0.26.0", + "webpki-roots 0.26.1", ] [[package]] name = "mail-server" -version = "0.5.3" +version = "0.6.0" dependencies = [ "directory", "imap", @@ -3153,9 +3162,9 @@ dependencies = [ [[package]] name = "managesieve" -version = "0.5.3" +version = "0.6.0" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "bincode", "directory", "imap", @@ -3286,9 +3295,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", ] @@ -3357,7 +3366,7 @@ dependencies = [ "twox-hash", "url", "webpki", - "webpki-roots 0.25.3", + "webpki-roots 0.25.4", ] [[package]] @@ -3368,7 +3377,7 @@ checksum = "06f19e4cfa0ab5a76b627cec2d81331c49b034988eaf302c3bafeada684eadef" dependencies = [ "base64 0.21.7", "bigdecimal", - "bindgen 0.69.2", + "bindgen 0.69.4", "bitflags 2.4.2", "bitvec", "btoi", @@ -3420,9 +3429,9 @@ dependencies = [ [[package]] name = "nlp" -version = "0.5.3" +version = "0.6.0" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "bincode", "farmhash", "jieba-rs", @@ -3496,20 +3505,25 @@ dependencies = [ ] [[package]] -name = "num-integer" -version = "0.1.45" +name = "num-conv" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] [[package]] name = "num-iter" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9" dependencies = [ "autocfg", "num-integer", @@ -3518,9 +3532,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg", "libm", @@ -3606,9 +3620,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "300.2.2+3.2.1" +version = "300.2.3+3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bbfad0063610ac26ee79f7484739e2b07555a75c42453b89263830b5c8103bc" +checksum = "5cff92b6f71555b61bb9315f7c64da3ca43d87531622120fea0195fc761b4843" dependencies = [ "cc", ] @@ -3634,7 +3648,7 @@ checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" dependencies = [ "futures-core", "futures-sink", - "indexmap 2.2.1", + "indexmap 2.2.3", "js-sys", "once_cell", "pin-project-lite", @@ -3863,7 +3877,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.2.1", + "indexmap 2.2.3", ] [[package]] @@ -4169,7 +4183,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" dependencies = [ "anyhow", - "itertools", + "itertools 0.10.5", "proc-macro2", "quote", "syn 1.0.109", @@ -4340,7 +4354,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48d1fd02e16232e942b5e7ce305b447c550d09a9146255a3e8a2cf62a0e2ac2d" dependencies = [ "either", - "itertools", + "itertools 0.10.5", "proc-macro2", "quote", "rayon", @@ -4492,9 +4506,9 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "rend" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2571463863a6bd50c32f94402933f03457a3fbaf697a707c5be741e459f08fd" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" dependencies = [ "bytecheck", ] @@ -4540,7 +4554,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 0.25.3", + "webpki-roots 0.25.4", "winreg", ] @@ -4610,9 +4624,9 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.7.43" +version = "0.7.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "527a97cdfef66f65998b5f3b637c26f5a5ec09cc52a3f9932313ac645f4190f5" +checksum = "5cba464629b3394fc4dbc6f940ff8f5b4ff5c7aef40f29166fd4ad12acbc99c0" dependencies = [ "bitvec", "bytecheck", @@ -4628,9 +4642,9 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.7.43" +version = "0.7.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5c462a1328c8e67e4d6dbad1eb0355dd43e8ab432c6e227a43657f16ade5033" +checksum = "a7dddfff8de25e6f62b9d64e6e432bf1c6736c57d20323e15ee10435fbda7c65" dependencies = [ "proc-macro2", "quote", @@ -4767,9 +4781,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.34.0" +version = "1.34.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7de2711cae7bdec993f4d2319352599ceb0d003e9f7900ea7c6ef4c5fc16831" +checksum = "b39449a79f45e8da28c57c341891b69a183044b29518bb8f86dbac9df60bb7df" dependencies = [ "arrayvec", "borsh", @@ -4822,9 +4836,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.30" +version = "0.38.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" dependencies = [ "bitflags 2.4.2", "errno", @@ -4866,7 +4880,7 @@ dependencies = [ "log", "ring 0.17.7", "rustls-pki-types", - "rustls-webpki 0.102.1", + "rustls-webpki 0.102.2", "subtle", "zeroize", ] @@ -4904,9 +4918,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e9d979b3ce68192e42760c7810125eb6cf2ea10efae545a156063e61f314e2a" +checksum = "0a716eb65e3158e90e17cd93d855216e27bde02745ab842f2cab4a39dba1bacf" [[package]] name = "rustls-webpki" @@ -4930,9 +4944,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.1" +version = "0.102.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef4ca26037c909dedb327b48c3327d0ba91d3dd3c4e05dad328f210ffb68e95b" +checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" dependencies = [ "ring 0.17.7", "rustls-pki-types", @@ -5321,7 +5335,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25494c13da6c336430906aa783e4bb2ae251c84158d6e5a4fdf0449a779c2521" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "bincode", "fancy-regex", "mail-builder", @@ -5384,9 +5398,9 @@ checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" [[package]] name = "smtp" -version = "0.5.3" +version = "0.6.0" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "bincode", "blake3", "dashmap", @@ -5428,7 +5442,7 @@ dependencies = [ "tracing", "unicode-security", "utils", - "webpki-roots 0.26.0", + "webpki-roots 0.26.1", "whatlang", "x509-parser", ] @@ -5509,7 +5523,7 @@ dependencies = [ [[package]] name = "stalwart-cli" -version = "0.5.3" +version = "0.6.0" dependencies = [ "clap", "console", @@ -5533,7 +5547,7 @@ dependencies = [ [[package]] name = "stalwart-install" -version = "0.5.3" +version = "0.6.0" dependencies = [ "base64 0.21.7", "clap", @@ -5561,7 +5575,8 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" name = "store" version = "0.1.0" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", + "arc-swap", "async-trait", "bincode", "blake3", @@ -5633,6 +5648,12 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strsim" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" + [[package]] name = "subprocess" version = "0.2.9" @@ -5741,13 +5762,12 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.9.0" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" +checksum = "a365e8cd18e44762ef95d87f284f4b5cd04107fec2ff3052bd6a3e6069669e67" dependencies = [ "cfg-if", "fastrand", - "redox_syscall", "rustix", "windows-sys 0.52.0", ] @@ -5776,7 +5796,7 @@ dependencies = [ name = "tests" version = "0.1.0" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "async-trait", "base64 0.21.7", "bytes", @@ -5823,18 +5843,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" +checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" +checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" dependencies = [ "proc-macro2", "quote", @@ -5853,12 +5873,13 @@ dependencies = [ [[package]] name = "time" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" +checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" dependencies = [ "deranged", "itoa", + "num-conv", "powerfmt", "serde", "time-core", @@ -5873,10 +5894,11 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" +checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" dependencies = [ + "num-conv", "time-core", ] @@ -5916,9 +5938,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.35.1" +version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" dependencies = [ "backtrace", "bytes", @@ -6025,7 +6047,7 @@ dependencies = [ "tokio", "tokio-rustls 0.25.0", "tungstenite", - "webpki-roots 0.26.0", + "webpki-roots 0.26.1", ] [[package]] @@ -6054,7 +6076,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.2.1", + "indexmap 2.2.3", "toml_datetime", "winnow", ] @@ -6065,7 +6087,7 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ - "indexmap 2.2.1", + "indexmap 2.2.3", "toml_datetime", "winnow", ] @@ -6344,9 +6366,9 @@ checksum = "7d817255e1bed6dfd4ca47258685d14d2bdcfbc64fdc9e3819bd5848057b8ecc" [[package]] name = "unicode-security" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ef5756b3097992b934b06608c69f48448a0fbe804bb1e72b982f6d7983e9e63" +checksum = "ee9e13753df674873f3c4693b240ae5c03245ddc157dfccf7c26db9329af3a11" dependencies = [ "unicode-normalization", "unicode-script", @@ -6417,9 +6439,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "utils" -version = "0.5.3" +version = "0.6.0" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "arc-swap", "base64 0.21.7", "blake3", @@ -6454,7 +6476,7 @@ dependencies = [ "tracing-journald", "tracing-opentelemetry", "tracing-subscriber", - "webpki-roots 0.26.0", + "webpki-roots 0.26.1", "x509-parser", ] @@ -6508,9 +6530,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" +checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -6518,9 +6540,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" +checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b" dependencies = [ "bumpalo", "log", @@ -6533,9 +6555,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.40" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" +checksum = "877b9c3f61ceea0e56331985743b13f3d25c406a7098d45180fb5f09bc19ed97" dependencies = [ "cfg-if", "js-sys", @@ -6545,9 +6567,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" +checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -6555,9 +6577,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" +checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" dependencies = [ "proc-macro2", "quote", @@ -6568,9 +6590,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" +checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" [[package]] name = "wasm-streams" @@ -6587,9 +6609,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.67" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" +checksum = "96565907687f7aceb35bc5fc03770a8a0471d82e479f25832f54a0e3f4b28446" dependencies = [ "js-sys", "wasm-bindgen", @@ -6635,15 +6657,15 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.25.3" +version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "webpki-roots" -version = "0.26.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de2cfda980f21be5a7ed2eadb3e6fe074d56022bea2cdeb1a62eb220fc04188" +checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009" dependencies = [ "rustls-pki-types", ] @@ -6860,9 +6882,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.36" +version = "0.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818ce546a11a9986bc24f93d0cdf38a8a1a400f1473ea8c82e59f6e0ffab9249" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" dependencies = [ "memchr", ] @@ -6888,9 +6910,9 @@ dependencies = [ [[package]] name = "x25519-dalek" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb66477291e7e8d2b0ff1bcb900bf29489a9692816d79874bea351e7a8b6de96" +checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" dependencies = [ "curve25519-dalek", "rand_core", diff --git a/README.md b/README.md index 207957f5..595d2349 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Key features: - **JMAP** server: - JMAP Core ([RFC 8620](https://datatracker.ietf.org/doc/html/rfc8620)) - JMAP Mail ([RFC 8621](https://datatracker.ietf.org/doc/html/rfc8621)) - - JMAP for Sieve Scripts ([DRAFT-SIEVE-15](https://www.ietf.org/archive/id/draft-ietf-jmap-sieve-15.html)) + - JMAP for Sieve Scripts ([DRAFT-SIEVE-19](https://www.ietf.org/archive/id/draft-ietf-jmap-sieve-19.html)) - JMAP over WebSocket ([RFC 8887](https://datatracker.ietf.org/doc/html/rfc8887)), JMAP Blob Management ([RFC9404](https://www.rfc-editor.org/rfc/rfc9404.html)) and JMAP for Quotas ([RFC9425](https://www.rfc-editor.org/rfc/rfc9425.html)) extensions. - **IMAP4** server: - IMAP4rev2 ([RFC 9051](https://datatracker.ietf.org/doc/html/rfc9051)) full compliance. @@ -55,7 +55,7 @@ Key features: - Built-in [DMARC](https://datatracker.ietf.org/doc/html/rfc7489), [DKIM](https://datatracker.ietf.org/doc/html/rfc6376), [SPF](https://datatracker.ietf.org/doc/html/rfc7208) and [ARC](https://datatracker.ietf.org/doc/html/rfc8617) support for message authentication. - Strong transport security through [DANE](https://datatracker.ietf.org/doc/html/rfc6698), [MTA-STS](https://datatracker.ietf.org/doc/html/rfc8461) and [SMTP TLS](https://datatracker.ietf.org/doc/html/rfc8460) reporting. - Inbound throttling and filtering with granular configuration rules, sieve scripting and milter integration. - - Virtual queues with delayed delivery, priority delivery, quotas, routing rules and throttling support. + - Distributed virtual queues with delayed delivery, priority delivery, quotas, routing rules and throttling support. - Envelope rewriting and message modification. - **Spam and Phishing** filter: - Comprehensive set of filtering **rules** on par with popular solutions. @@ -105,10 +105,10 @@ Additionally you may become a sponsor to obtain priority support from Stalwart L ## Roadmap - [x] Performance enhancements +- [x] Distributed SMTP queues - [ ] Web-based admin panel - [ ] JMAP Calendar, Contacts and Tasks support -- [ ] Brand Indicators for Message Identification (BIMI) support -- [ ] Distributed SMTP queues +- [ ] CalDAV and CardDAV support See the [open issues](https://github.com/stalwartlabs/mail-server/issues) for a full list of proposed features (and known issues). @@ -116,7 +116,7 @@ See the [open issues](https://github.com/stalwartlabs/mail-server/issues) for a Part of the development of this project was funded through the [NGI0 Entrust Fund](https://nlnet.nl/entrust), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 101069594. -If you find the project useful you can help by [becoming a sponsor](https://github.com/sponsors/stalwartlabs). Thank you! +If you find the project useful you can help by [becoming a sponsor](https://liberapay.com/stalwartlabs). Thank you! ## License diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index eb56d6c3..7b46d35f 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Stalwart Labs Ltd. "] license = "AGPL-3.0-only" repository = "https://github.com/stalwartlabs/cli" homepage = "https://github.com/stalwartlabs/cli" -version = "0.5.3" +version = "0.6.0" edition = "2021" readme = "README.md" resolver = "2" diff --git a/crates/directory/src/core/config.rs b/crates/directory/src/core/config.rs index 928f9719..1b8744b3 100644 --- a/crates/directory/src/core/config.rs +++ b/crates/directory/src/core/config.rs @@ -26,11 +26,11 @@ use deadpool::{ Runtime, }; use std::{sync::Arc, time::Duration}; -use store::{Store, Stores}; +use store::{dispatch::blocked::BlockedIps, Store, Stores}; use utils::{ config::{ utils::{AsKey, ParseValue}, - Config, Servers, + Config, }, expr::Token, }; @@ -52,7 +52,6 @@ pub trait ConfigDirectory { async fn parse_directory( &self, stores: &Stores, - servers: &Servers, data_store: Store, ) -> utils::config::Result; } @@ -61,12 +60,14 @@ impl ConfigDirectory for Config { async fn parse_directory( &self, stores: &Stores, - servers: &Servers, data_store: Store, ) -> utils::config::Result { let mut config = Directories { directories: AHashMap::new(), }; + let blocked_ips = Arc::new(BlockedIps::new( + stores.get_lookup_store(self, "storage.lookup")?, + )); for id in self.sub_keys("directory", ".type") { if id.ends_with(".columns") || id.ends_with(".attributes") || id.contains(".principals") @@ -152,7 +153,7 @@ impl ConfigDirectory for Config { ("directory", id, "options.subaddressing"), )?, cache: CachedDirectory::try_from_config(self, ("directory", id))?, - blocked_ips: servers.blocked_ips.clone(), + blocked_ips: blocked_ips.clone(), }); // Add directory diff --git a/crates/directory/src/core/dispatch.rs b/crates/directory/src/core/dispatch.rs index 69626aa5..0b0fac72 100644 --- a/crates/directory/src/core/dispatch.rs +++ b/crates/directory/src/core/dispatch.rs @@ -52,6 +52,7 @@ impl Directory { if let Some(banned) = self .blocked_ips .is_fail2banned(remote_ip, login.to_string()) + .await { tracing::info!( context = "directory", diff --git a/crates/directory/src/lib.rs b/crates/directory/src/lib.rs index eb0bcff0..69badef3 100644 --- a/crates/directory/src/lib.rs +++ b/crates/directory/src/lib.rs @@ -36,8 +36,8 @@ use backend::{ use deadpool::managed::PoolError; use ldap3::LdapError; use mail_send::Credentials; -use store::Store; -use utils::{config::if_block::IfBlock, expr::Variable, listener::blocked::BlockedIps}; +use store::{dispatch::blocked::BlockedIps, Store}; +use utils::{config::if_block::IfBlock, expr::Variable}; pub mod backend; pub mod core; diff --git a/crates/imap/Cargo.toml b/crates/imap/Cargo.toml index 09126e08..bd5380ca 100644 --- a/crates/imap/Cargo.toml +++ b/crates/imap/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "imap" -version = "0.5.3" +version = "0.6.0" edition = "2021" resolver = "2" diff --git a/crates/imap/src/core/client.rs b/crates/imap/src/core/client.rs index bb371dea..76ede758 100644 --- a/crates/imap/src/core/client.rs +++ b/crates/imap/src/core/client.rs @@ -27,11 +27,8 @@ use imap_proto::{ receiver::{self, Request}, Command, ResponseCode, StatusResponse, }; -use jmap::auth::rate_limit::AuthenticatedLimiter; -use utils::listener::{ - limiter::{ConcurrencyLimiter, RateLimiter}, - SessionStream, -}; +use jmap::auth::rate_limit::ConcurrencyLimiters; +use utils::listener::{limiter::ConcurrencyLimiter, SessionStream}; use super::{SelectedMailbox, Session, SessionData, State, IMAP}; @@ -52,7 +49,7 @@ impl Session { loop { match self.receiver.parse(&mut bytes) { - Ok(request) => match self.is_allowed(request) { + Ok(request) => match self.is_allowed(request).await { Ok(request) => { requests.push(request); } @@ -223,19 +220,34 @@ pub fn group_requests( } impl Session { - fn is_allowed(&self, request: Request) -> Result, StatusResponse> { + async fn is_allowed( + &self, + request: Request, + ) -> Result, StatusResponse> { let state = &self.state; // Rate limit request if let State::Authenticated { data } | State::Selected { data, .. } = state { - if !data - .imap - .get_authenticated_limiter(data.account_id) - .request_limiter - .is_allowed(&self.imap.rate_requests) + match data + .jmap + .lookup_store + .is_rate_allowed( + format!("ireq:{}", data.account_id).as_bytes(), + &self.imap.rate_requests, + true, + ) + .await { - return Err(StatusResponse::no("Too many requests") - .with_tag(request.tag) - .with_code(ResponseCode::Limit)); + Ok(None) => {} + Ok(Some(_)) => { + return Err(StatusResponse::no("Too many requests") + .with_tag(request.tag) + .with_code(ResponseCode::Limit)); + } + Err(_) => { + return Err(StatusResponse::no("Internal server error") + .with_tag(request.tag) + .with_code(ResponseCode::ContactAdmin)); + } } } @@ -390,13 +402,12 @@ impl State { } impl IMAP { - pub fn get_authenticated_limiter(&self, account_id: u32) -> Arc { + pub fn get_concurrency_limiter(&self, account_id: u32) -> Arc { self.rate_limiter .get(&account_id) .map(|limiter| limiter.clone()) .unwrap_or_else(|| { - let limiter = Arc::new(AuthenticatedLimiter { - request_limiter: RateLimiter::new(&self.rate_requests), + let limiter = Arc::new(ConcurrencyLimiters { concurrent_requests: ConcurrencyLimiter::new(self.rate_concurrent), concurrent_uploads: ConcurrencyLimiter::new(self.rate_concurrent), }); diff --git a/crates/imap/src/core/mod.rs b/crates/imap/src/core/mod.rs index b3dc64c3..8a8c751f 100644 --- a/crates/imap/src/core/mod.rs +++ b/crates/imap/src/core/mod.rs @@ -36,7 +36,7 @@ use imap_proto::{ Command, ResponseCode, StatusResponse, }; use jmap::{ - auth::{rate_limit::AuthenticatedLimiter, AccessToken}, + auth::{rate_limit::ConcurrencyLimiters, AccessToken}, JMAP, }; use store::roaring::RoaringBitmap; @@ -80,7 +80,7 @@ pub struct IMAP { pub greeting_plain: Vec, pub greeting_tls: Vec, - pub rate_limiter: DashMap>, + pub rate_limiter: DashMap>, pub rate_requests: Rate, pub rate_concurrent: u64, } diff --git a/crates/imap/src/core/session.rs b/crates/imap/src/core/session.rs index 447d5e3b..5ca2cc2c 100644 --- a/crates/imap/src/core/session.rs +++ b/crates/imap/src/core/session.rs @@ -51,6 +51,10 @@ impl SessionManager for ImapSessionManager { fn shutdown(&self) -> impl std::future::Future + Send { async {} } + + fn is_ip_blocked(&self, addr: &std::net::IpAddr) -> bool { + self.jmap.directory.blocked_ips.is_blocked(addr) + } } impl Session { diff --git a/crates/imap/src/op/authenticate.rs b/crates/imap/src/op/authenticate.rs index 875bf5c2..21c60dec 100644 --- a/crates/imap/src/op/authenticate.rs +++ b/crates/imap/src/op/authenticate.rs @@ -101,7 +101,12 @@ impl Session { tag: String, ) -> crate::Result<()> { // Throttle authentication requests - if self.jmap.is_auth_allowed_soft(&self.remote_addr).is_err() { + if self + .jmap + .is_auth_allowed_soft(&self.remote_addr) + .await + .is_err() + { self.write_bytes( StatusResponse::bye("Too many authentication requests from this IP address.") .into_bytes(), @@ -151,7 +156,7 @@ impl Session { // Enforce concurrency limits let in_flight = self .imap - .get_authenticated_limiter(access_token.primary_id()) + .get_concurrency_limiter(access_token.primary_id()) .concurrent_requests .is_allowed(); if let Some(in_flight) = in_flight { diff --git a/crates/install/Cargo.toml b/crates/install/Cargo.toml index 6462149f..1e9267ff 100644 --- a/crates/install/Cargo.toml +++ b/crates/install/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Stalwart Labs Ltd. "] license = "AGPL-3.0-only" repository = "https://github.com/stalwartlabs/mail-server" homepage = "https://github.com/stalwartlabs/mail-server" -version = "0.5.3" +version = "0.6.0" edition = "2021" readme = "README.md" resolver = "2" diff --git a/crates/install/src/main.rs b/crates/install/src/main.rs index 9f6448dd..a71f2adf 100644 --- a/crates/install/src/main.rs +++ b/crates/install/src/main.rs @@ -112,14 +112,12 @@ enum SmtpDirectory { Imap, } -const DIRECTORIES: [[&str; 2]; 7] = [ +const DIRECTORIES: [[&str; 2]; 5] = [ ["bin", ""], ["etc", "dkim"], ["etc", "acme"], ["data", "blobs"], ["logs", ""], - ["queue", ""], - ["reports", ""], ]; #[derive(Debug, Parser)] @@ -560,8 +558,6 @@ fn main() -> std::io::Result<()> { .arg("770") .arg(&format!("{}/etc", base_path.display())) .arg(&format!("{}/data", base_path.display())) - .arg(&format!("{}/queue", base_path.display())) - .arg(&format!("{}/reports", base_path.display())) .arg(&format!("{}/logs", base_path.display())); if let Err(err) = cmd.status() { eprintln!("Warning: Failed to set permissions: {}", err); @@ -659,7 +655,7 @@ fn main() -> std::io::Result<()> { ); } - eprintln!("\nāœ… {dkim_instructions}\nšŸŽ‰ Installation completed!\n"); + eprintln!("\nāœ… {dkim_instructions}\nšŸŽ‰ Installation completed! Please consider sponsoring Stalwart at https://liberapay.com/stalwartlabs\n"); Ok(()) } diff --git a/crates/jmap/Cargo.toml b/crates/jmap/Cargo.toml index 27f6766a..8cb53609 100644 --- a/crates/jmap/Cargo.toml +++ b/crates/jmap/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jmap" -version = "0.5.3" +version = "0.6.0" edition = "2021" resolver = "2" diff --git a/crates/jmap/src/api/http.rs b/crates/jmap/src/api/http.rs index 6cde1ce3..01550a63 100644 --- a/crates/jmap/src/api/http.rs +++ b/crates/jmap/src/api/http.rs @@ -190,7 +190,7 @@ pub async fn parse_jmap_request( ("oauth-authorization-server", &Method::GET) => { let remote_addr = jmap.build_remote_addr(&req, remote_ip); // Limit anonymous requests - return match jmap.is_anonymous_allowed(&remote_addr) { + return match jmap.is_anonymous_allowed(&remote_addr).await { Ok(_) => { JsonResponse::new(OAuthMetadata::new(&instance.data)).into_http_response() } @@ -207,13 +207,13 @@ pub async fn parse_jmap_request( match (path.next().unwrap_or(""), req.method()) { ("", &Method::GET) => { - return match jmap.is_anonymous_allowed(&remote_addr) { + return match jmap.is_anonymous_allowed(&remote_addr).await { Ok(_) => jmap.handle_user_device_auth(&mut req).await, Err(err) => err.into_http_response(), } } ("", &Method::POST) => { - return match jmap.is_auth_allowed_soft(&remote_addr) { + return match jmap.is_auth_allowed_soft(&remote_addr).await { Ok(_) => { jmap.handle_user_device_auth_post(&mut req, remote_addr) .await @@ -222,25 +222,25 @@ pub async fn parse_jmap_request( } } ("code", &Method::GET) => { - return match jmap.is_anonymous_allowed(&remote_addr) { + return match jmap.is_anonymous_allowed(&remote_addr).await { Ok(_) => jmap.handle_user_code_auth(&mut req).await, Err(err) => err.into_http_response(), } } ("code", &Method::POST) => { - return match jmap.is_auth_allowed_soft(&remote_addr) { + return match jmap.is_auth_allowed_soft(&remote_addr).await { Ok(_) => jmap.handle_user_code_auth_post(&mut req, remote_addr).await, Err(err) => err.into_http_response(), } } ("device", &Method::POST) => { - return match jmap.is_anonymous_allowed(&remote_addr) { + return match jmap.is_anonymous_allowed(&remote_addr).await { Ok(_) => jmap.handle_device_auth(&mut req, instance).await, Err(err) => err.into_http_response(), } } ("token", &Method::POST) => { - return match jmap.is_anonymous_allowed(&remote_addr) { + return match jmap.is_anonymous_allowed(&remote_addr).await { Ok(_) => jmap.handle_token_request(&mut req).await, Err(err) => err.into_http_response(), } @@ -259,7 +259,7 @@ pub async fn parse_jmap_request( return jmap.handle_crypto_update(&mut req, remote_addr).await; } Method::POST => { - return match jmap.is_auth_allowed_soft(&remote_addr) { + return match jmap.is_auth_allowed_soft(&remote_addr).await { Ok(_) => jmap.handle_crypto_update(&mut req, remote_addr).await, Err(err) => err.into_http_response(), } @@ -298,6 +298,10 @@ impl SessionManager for JmapSessionManager { let _ = self.inner.state_tx.send(state::Event::Stop).await; } } + + fn is_ip_blocked(&self, addr: &IpAddr) -> bool { + self.inner.directory.blocked_ips.is_blocked(addr) + } } async fn handle_request(jmap: Arc, session: SessionData) { diff --git a/crates/jmap/src/auth/authenticate.rs b/crates/jmap/src/auth/authenticate.rs index db192d48..672091b7 100644 --- a/crates/jmap/src/auth/authenticate.rs +++ b/crates/jmap/src/auth/authenticate.rs @@ -52,7 +52,7 @@ impl JMAP { let addr = self.build_remote_addr(req, remote_ip); if mechanism.eq_ignore_ascii_case("basic") { // Enforce rate limit for authentication requests - self.is_auth_allowed_soft(&addr)?; + self.is_auth_allowed_soft(&addr).await?; // Decode the base64 encoded credentials if let Some((account, secret)) = base64_decode(token.as_bytes()) @@ -80,7 +80,7 @@ impl JMAP { } } else if mechanism.eq_ignore_ascii_case("bearer") { // Enforce anonymous rate limit for bearer auth requests - self.is_anonymous_allowed(&addr)?; + self.is_anonymous_allowed(&addr).await?; match self.validate_access_token("access_token", &token).await { Ok((account_id, _, _)) => self.get_access_token(account_id).await, @@ -95,7 +95,7 @@ impl JMAP { } } else { // Enforce anonymous rate limit - self.is_anonymous_allowed(&addr)?; + self.is_anonymous_allowed(&addr).await?; None } .map(|access_token| { @@ -108,13 +108,14 @@ impl JMAP { if let Some(session) = session { // Enforce authenticated rate limit - Ok(Some((self.is_account_allowed(&session)?, session))) + Ok(Some((self.is_account_allowed(&session).await?, session))) } else { Ok(None) } } else { // Enforce anonymous rate limit - self.is_anonymous_allowed(&self.build_remote_addr(req, remote_ip))?; + self.is_anonymous_allowed(&self.build_remote_addr(req, remote_ip)) + .await?; Ok(None) } @@ -190,7 +191,7 @@ impl JMAP { { Ok(AuthResult::Success(principal)) => AuthResult::Success(AccessToken::new(principal)), Ok(AuthResult::Failure) => { - let _ = self.is_auth_allowed_hard(&remote_ip); + let _ = self.is_auth_allowed_hard(&remote_ip).await; AuthResult::Failure } Ok(AuthResult::Banned) => AuthResult::Banned, diff --git a/crates/jmap/src/auth/rate_limit.rs b/crates/jmap/src/auth/rate_limit.rs index 3d8b6905..359c978a 100644 --- a/crates/jmap/src/auth/rate_limit.rs +++ b/crates/jmap/src/auth/rate_limit.rs @@ -24,62 +24,50 @@ use std::{net::IpAddr, sync::Arc}; use jmap_proto::error::request::{RequestError, RequestLimitError}; -use utils::listener::limiter::{ConcurrencyLimiter, InFlight, RateLimiter}; +use utils::listener::limiter::{ConcurrencyLimiter, InFlight}; use crate::JMAP; use super::AccessToken; -pub struct AuthenticatedLimiter { - pub request_limiter: RateLimiter, +pub struct ConcurrencyLimiters { pub concurrent_requests: ConcurrencyLimiter, pub concurrent_uploads: ConcurrencyLimiter, } -#[derive(Debug)] -pub struct AnonymousLimiter { - request_limiter: RateLimiter, - auth_limiter: RateLimiter, -} - impl JMAP { - pub fn get_authenticated_limiter(&self, account_id: u32) -> Arc { - self.rate_limit_auth + pub fn get_concurrency_limiter(&self, account_id: u32) -> Arc { + self.concurrency_limiter .get(&account_id) .map(|limiter| limiter.clone()) .unwrap_or_else(|| { - let limiter = Arc::new(AuthenticatedLimiter { - request_limiter: RateLimiter::new(&self.config.rate_authenticated), + let limiter = Arc::new(ConcurrencyLimiters { concurrent_requests: ConcurrencyLimiter::new( self.config.request_max_concurrent, ), concurrent_uploads: ConcurrencyLimiter::new(self.config.upload_max_concurrent), }); - self.rate_limit_auth.insert(account_id, limiter.clone()); + self.concurrency_limiter.insert(account_id, limiter.clone()); limiter }) } - pub fn get_anonymous_limiter(&self, addr: &IpAddr) -> Arc { - self.rate_limit_unauth - .get(addr) - .map(|limiter| limiter.clone()) - .unwrap_or_else(|| { - let limiter = Arc::new(AnonymousLimiter { - request_limiter: RateLimiter::new(&self.config.rate_anonymous), - auth_limiter: RateLimiter::new(&self.config.rate_authenticate_req), - }); - self.rate_limit_unauth.insert(*addr, limiter.clone()); - limiter - }) - } + pub async fn is_account_allowed( + &self, + access_token: &AccessToken, + ) -> Result { + let limiter = self.get_concurrency_limiter(access_token.primary_id()); - pub fn is_account_allowed(&self, access_token: &AccessToken) -> Result { - let limiter = self.get_authenticated_limiter(access_token.primary_id()); - - if limiter - .request_limiter - .is_allowed(&self.config.rate_authenticated) + if self + .lookup_store + .is_rate_allowed( + format!("j:{}", access_token.primary_id).as_bytes(), + &self.config.rate_authenticated, + false, + ) + .await + .map_err(|_| RequestError::internal_server_error())? + .is_none() { if let Some(in_flight_request) = limiter.concurrent_requests.is_allowed() { Ok(in_flight_request) @@ -95,11 +83,17 @@ impl JMAP { } } - pub fn is_anonymous_allowed(&self, addr: &IpAddr) -> Result<(), RequestError> { + pub async fn is_anonymous_allowed(&self, addr: &IpAddr) -> Result<(), RequestError> { if self - .get_anonymous_limiter(addr) - .request_limiter - .is_allowed(&self.config.rate_anonymous) + .lookup_store + .is_rate_allowed( + format!("jreq:{}", addr).as_bytes(), + &self.config.rate_anonymous, + false, + ) + .await + .map_err(|_| RequestError::internal_server_error())? + .is_none() { Ok(()) } else { @@ -109,7 +103,7 @@ impl JMAP { pub fn is_upload_allowed(&self, access_token: &AccessToken) -> Result { if let Some(in_flight_request) = self - .get_authenticated_limiter(access_token.primary_id()) + .get_concurrency_limiter(access_token.primary_id()) .concurrent_uploads .is_allowed() { @@ -121,24 +115,35 @@ impl JMAP { } } - pub fn is_auth_allowed_soft(&self, addr: &IpAddr) -> Result<(), RequestError> { - match self.rate_limit_unauth.get(addr) { - Some(limiter) - if !limiter - .auth_limiter - .is_allowed_soft(&self.config.rate_authenticate_req) => - { - Err(RequestError::too_many_auth_attempts()) - } - _ => Ok(()), + pub async fn is_auth_allowed_soft(&self, addr: &IpAddr) -> Result<(), RequestError> { + if self + .lookup_store + .is_rate_allowed( + format!("jauth:{}", addr).as_bytes(), + &self.config.rate_authenticate_req, + true, + ) + .await + .map_err(|_| RequestError::internal_server_error())? + .is_none() + { + Ok(()) + } else { + Err(RequestError::too_many_auth_attempts()) } } - pub fn is_auth_allowed_hard(&self, addr: &IpAddr) -> Result<(), RequestError> { + pub async fn is_auth_allowed_hard(&self, addr: &IpAddr) -> Result<(), RequestError> { if self - .get_anonymous_limiter(addr) - .auth_limiter - .is_allowed(&self.config.rate_authenticate_req) + .lookup_store + .is_rate_allowed( + format!("jauth:{}", addr).as_bytes(), + &self.config.rate_authenticate_req, + false, + ) + .await + .map_err(|_| RequestError::internal_server_error())? + .is_none() { Ok(()) } else { @@ -147,16 +152,8 @@ impl JMAP { } } -impl AuthenticatedLimiter { +impl ConcurrencyLimiters { pub fn is_active(&self) -> bool { - self.request_limiter.is_active() - || self.concurrent_requests.is_active() - || self.concurrent_uploads.is_active() - } -} - -impl AnonymousLimiter { - pub fn is_active(&self) -> bool { - self.request_limiter.is_active() || self.auth_limiter.is_active() + self.concurrent_requests.is_active() || self.concurrent_uploads.is_active() } } diff --git a/crates/jmap/src/lib.rs b/crates/jmap/src/lib.rs index 8ee26c50..fc235497 100644 --- a/crates/jmap/src/lib.rs +++ b/crates/jmap/src/lib.rs @@ -21,17 +21,11 @@ * for more details. */ -use std::{ - collections::hash_map::RandomState, fmt::Display, net::IpAddr, sync::Arc, time::Duration, -}; +use std::{collections::hash_map::RandomState, fmt::Display, sync::Arc, time::Duration}; use ::sieve::{Compiler, Runtime}; use api::session::BaseCapabilities; -use auth::{ - oauth::OAuthCode, - rate_limit::{AnonymousLimiter, AuthenticatedLimiter}, - AccessToken, -}; +use auth::{oauth::OAuthCode, rate_limit::ConcurrencyLimiters, AccessToken}; use dashmap::DashMap; use directory::{Directories, Directory, QueryBy}; use jmap_proto::{ @@ -55,7 +49,7 @@ use store::{ query::{sort::Pagination, Comparator, Filter, ResultSet, SortedResultSet}, roaring::RoaringBitmap, write::{BatchBuilder, BitmapClass, DirectoryClass, TagValue, ValueClass}, - BitmapKey, BlobStore, Deserialize, FtsStore, Store, Stores, ValueKey, + BitmapKey, BlobStore, Deserialize, FtsStore, LookupStore, Store, Stores, ValueKey, }; use tokio::sync::mpsc; use utils::{ @@ -89,6 +83,7 @@ pub struct JMAP { pub store: Store, pub blob_store: BlobStore, pub fts_store: FtsStore, + pub lookup_store: LookupStore, pub config: Config, pub directory: Arc, @@ -96,9 +91,7 @@ pub struct JMAP { pub access_tokens: TtlDashMap>, pub snowflake_id: SnowflakeIdGenerator, - pub rate_limit_auth: DashMap>, - pub rate_limit_unauth: DashMap>, - + pub concurrency_limiter: DashMap>, pub oauth_codes: TtlDashMap>, pub state_tx: mpsc::Sender, @@ -211,6 +204,7 @@ impl JMAP { store: stores.get_store(config, "storage.data")?, fts_store: stores.get_fts_store(config, "storage.fts")?, blob_store: stores.get_blob_store(config, "storage.blob")?, + lookup_store: stores.get_lookup_store(config, "storage.lookup")?, config: Config::new(config).failed("Invalid configuration file"), sessions: TtlDashMap::with_capacity( config.property("jmap.session.cache.size")?.unwrap_or(100), @@ -220,14 +214,7 @@ impl JMAP { config.property("jmap.session.cache.size")?.unwrap_or(100), shard_amount, ), - rate_limit_auth: DashMap::with_capacity_and_hasher_and_shard_amount( - config - .property("jmap.rate-limit.cache.size")? - .unwrap_or(1024), - RandomState::default(), - shard_amount, - ), - rate_limit_unauth: DashMap::with_capacity_and_hasher_and_shard_amount( + concurrency_limiter: DashMap::with_capacity_and_hasher_and_shard_amount( config .property("jmap.rate-limit.cache.size")? .unwrap_or(1024), diff --git a/crates/jmap/src/services/housekeeper.rs b/crates/jmap/src/services/housekeeper.rs index 6a968aaf..5b00e5ff 100644 --- a/crates/jmap/src/services/housekeeper.rs +++ b/crates/jmap/src/services/housekeeper.rs @@ -23,10 +23,10 @@ use std::sync::Arc; +use store::dispatch::blocked::BLOCKED_IP_KEY; use tokio::sync::mpsc; use utils::{ config::{cron::SimpleCron, Config, Servers}, - listener::blocked::BLOCKED_IP_KEY, map::ttl_dashmap::TtlMap, UnwrapFailure, }; @@ -57,7 +57,6 @@ pub fn spawn_housekeeper( .failed("Initialize housekeeper"); let certificates = std::mem::take(&mut servers.certificates); - let blocked_ips = servers.blocked_ips.clone(); tokio::spawn(async move { tracing::debug!("Housekeeper task started."); @@ -110,11 +109,12 @@ pub fn spawn_housekeeper( // Future releases will support reloading the configuration // for now, we just reload the blocked IP addresses let core = core.clone(); - let blocked_ips = blocked_ips.clone(); tokio::spawn(async move { match core.store.config_list(BLOCKED_IP_KEY).await { Ok(config) => { - if let Err(err) = blocked_ips.reload_blocked_ips(&config) { + if let Err(err) = + core.directory.blocked_ips.reload_blocked_ips(&config) + { tracing::error!( context = "store", event = "error", @@ -176,16 +176,12 @@ pub fn spawn_housekeeper( if do_purge { let core = core.clone(); - let blocked_ips = blocked_ips.clone(); tokio::spawn(async move { tracing::info!("Purging session cache."); - blocked_ips.cleanup(); core.sessions.cleanup(); core.access_tokens.cleanup(); core.oauth_codes.cleanup(); - core.rate_limit_auth - .retain(|_, limiter| limiter.is_active()); - core.rate_limit_unauth + core.concurrency_limiter .retain(|_, limiter| limiter.is_active()); }); } diff --git a/crates/jmap/src/submission/get.rs b/crates/jmap/src/submission/get.rs index b8b9e384..39e58727 100644 --- a/crates/jmap/src/submission/get.rs +++ b/crates/jmap/src/submission/get.rs @@ -126,16 +126,21 @@ impl JMAP { .with_property( Property::SmtpReply, match &rcpt.status { - queue::Status::Completed(reply) => { - reply.response.message() - } + queue::Status::Completed(reply) => reply + .response + .to_string() + .replace('\n', " "), queue::Status::TemporaryFailure(reply) | queue::Status::PermanentFailure(reply) => { - reply.response.message() + reply + .response + .to_string() + .replace('\n', " ") } - queue::Status::Scheduled => "250 2.1.5 Queued", - } - .replace('\n', " "), + queue::Status::Scheduled => { + "250 2.1.5 Queued".to_string() + } + }, ) .with_property(Property::Displayed, "unknown"), ); diff --git a/crates/main/Cargo.toml b/crates/main/Cargo.toml index 3bcaafb0..0c7b451a 100644 --- a/crates/main/Cargo.toml +++ b/crates/main/Cargo.toml @@ -7,7 +7,7 @@ homepage = "https://stalw.art" keywords = ["imap", "jmap", "smtp", "email", "mail", "server"] categories = ["email"] license = "AGPL-3.0-only" -version = "0.5.3" +version = "0.6.0" edition = "2021" resolver = "2" @@ -32,7 +32,7 @@ jemallocator = "0.5.0" [features] #default = ["sqlite", "foundationdb", "postgres", "mysql", "rocks", "elastic", "s3", "redis"] -default = ["sqlite", "postgres", "mysql", "redis"] +default = ["sqlite", "postgres", "mysql", "rocks", "elastic", "s3", "redis"] sqlite = ["store/sqlite"] foundationdb = ["store/foundation"] postgres = ["store/postgres"] diff --git a/crates/main/src/main.rs b/crates/main/src/main.rs index b69834b6..8c11b1bb 100644 --- a/crates/main/src/main.rs +++ b/crates/main/src/main.rs @@ -68,14 +68,10 @@ async fn main() -> std::io::Result<()> { // Update configuration config.update(data_store.config_list("").await.failed("Storage error")); - servers - .blocked_ips - .reload(&config) - .failed("Invalid configuration"); // Parse directories let directory = config - .parse_directory(&stores, &servers, data_store) + .parse_directory(&stores, data_store) .await .failed("Invalid configuration"); let schedulers = config @@ -105,6 +101,10 @@ async fn main() -> std::io::Result<()> { let imap = IMAP::init(&config) .await .failed("Invalid configuration file"); + jmap.directory + .blocked_ips + .reload(&config) + .failed("Invalid configuration"); // Spawn servers let (shutdown_tx, shutdown_rx) = servers.spawn(|server, shutdown_rx| { diff --git a/crates/managesieve/Cargo.toml b/crates/managesieve/Cargo.toml index eae37824..79f8ec95 100644 --- a/crates/managesieve/Cargo.toml +++ b/crates/managesieve/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "managesieve" -version = "0.5.3" +version = "0.6.0" edition = "2021" resolver = "2" diff --git a/crates/managesieve/src/core/client.rs b/crates/managesieve/src/core/client.rs index 9a05c4ab..5aaf8a5d 100644 --- a/crates/managesieve/src/core/client.rs +++ b/crates/managesieve/src/core/client.rs @@ -21,7 +21,6 @@ * for more details. */ -use imap::core::IMAP; use imap_proto::receiver::{self, Request}; use jmap_proto::types::{collection::Collection, property::Property}; use store::query::Filter; @@ -43,16 +42,14 @@ impl Session { loop { match self.receiver.parse(&mut bytes) { - Ok(request) => { - match request.validate_request(&self.imap, &self.state, self.stream.is_tls()) { - Ok(request) => { - requests.push(request); - } - Err(response) => { - self.write(&response.into_bytes()).await?; - } + Ok(request) => match self.validate_request(request).await { + Ok(request) => { + requests.push(request); } - } + Err(response) => { + self.write(&response.into_bytes()).await?; + } + }, Err(receiver::Error::NeedsMoreData) => { break; } @@ -108,6 +105,64 @@ impl Session { Ok(true) } + + async fn validate_request( + &self, + command: Request, + ) -> Result, StatusResponse> { + match &command.command { + Command::Capability | Command::Logout | Command::Noop => Ok(command), + Command::Authenticate => { + if let State::NotAuthenticated { .. } = &self.state { + if self.stream.is_tls() { + Ok(command) + } else { + Err(StatusResponse::no("Cannot authenticate over plain-text.") + .with_code(ResponseCode::EncryptNeeded)) + } + } else { + Err(StatusResponse::no("Already authenticated.")) + } + } + Command::StartTls => { + if !self.stream.is_tls() { + Ok(command) + } else { + Err(StatusResponse::no("Already in TLS mode.")) + } + } + Command::HaveSpace + | Command::PutScript + | Command::ListScripts + | Command::SetActive + | Command::GetScript + | Command::DeleteScript + | Command::RenameScript + | Command::CheckScript + | Command::Unauthenticate => { + if let State::Authenticated { access_token, .. } = &self.state { + match self + .jmap + .lookup_store + .is_rate_allowed( + format!("ireq:{}", access_token.primary_id()).as_bytes(), + &self.imap.rate_requests, + true, + ) + .await + { + Ok(None) => Ok(command), + Ok(Some(_)) => Err(StatusResponse::no("Too many requests") + .with_code(ResponseCode::TryLater)), + Err(_) => Err(StatusResponse::no("Internal server error") + .with_code(ResponseCode::TryLater)), + } + } else { + Err(StatusResponse::no("Not authenticated.")) + } + } + } + } } impl Session { @@ -177,68 +232,3 @@ impl Session { }) } } - -trait ValidateRequest: Sized { - fn validate_request( - self, - imap: &IMAP, - state: &State, - is_tls: bool, - ) -> Result; -} - -impl ValidateRequest for Request { - fn validate_request( - self, - imap: &IMAP, - state: &State, - is_tls: bool, - ) -> Result { - match &self.command { - Command::Capability | Command::Logout | Command::Noop => Ok(self), - Command::Authenticate => { - if let State::NotAuthenticated { .. } = state { - if is_tls { - Ok(self) - } else { - Err(StatusResponse::no("Cannot authenticate over plain-text.") - .with_code(ResponseCode::EncryptNeeded)) - } - } else { - Err(StatusResponse::no("Already authenticated.")) - } - } - Command::StartTls => { - if !is_tls { - Ok(self) - } else { - Err(StatusResponse::no("Already in TLS mode.")) - } - } - Command::HaveSpace - | Command::PutScript - | Command::ListScripts - | Command::SetActive - | Command::GetScript - | Command::DeleteScript - | Command::RenameScript - | Command::CheckScript - | Command::Unauthenticate => { - if let State::Authenticated { access_token, .. } = state { - if imap - .get_authenticated_limiter(access_token.primary_id()) - .request_limiter - .is_allowed(&imap.rate_requests) - { - Ok(self) - } else { - Err(StatusResponse::no("Too many requests") - .with_code(ResponseCode::TryLater)) - } - } else { - Err(StatusResponse::no("Not authenticated.")) - } - } - } - } -} diff --git a/crates/managesieve/src/core/session.rs b/crates/managesieve/src/core/session.rs index 735e39d1..c7c27f41 100644 --- a/crates/managesieve/src/core/session.rs +++ b/crates/managesieve/src/core/session.rs @@ -68,6 +68,10 @@ impl SessionManager for ManageSieveSessionManager { fn shutdown(&self) -> impl std::future::Future + Send { async {} } + + fn is_ip_blocked(&self, addr: &std::net::IpAddr) -> bool { + self.jmap.directory.blocked_ips.is_blocked(addr) + } } impl Session { diff --git a/crates/managesieve/src/op/authenticate.rs b/crates/managesieve/src/op/authenticate.rs index 8608abf4..f5e8c2f4 100644 --- a/crates/managesieve/src/op/authenticate.rs +++ b/crates/managesieve/src/op/authenticate.rs @@ -77,7 +77,12 @@ impl Session { }; // Throttle authentication requests - if self.jmap.is_auth_allowed_soft(&self.remote_addr).is_err() { + if self + .jmap + .is_auth_allowed_soft(&self.remote_addr) + .await + .is_err() + { tracing::debug!(parent: &self.span, event = "disconnect", "Too many authentication attempts, disconnecting.", @@ -128,7 +133,7 @@ impl Session { // Enforce concurrency limits let in_flight = self .imap - .get_authenticated_limiter(access_token.primary_id()) + .get_concurrency_limiter(access_token.primary_id()) .concurrent_requests .is_allowed(); if let Some(in_flight) = in_flight { diff --git a/crates/nlp/Cargo.toml b/crates/nlp/Cargo.toml index 530a919f..a986cb47 100644 --- a/crates/nlp/Cargo.toml +++ b/crates/nlp/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nlp" -version = "0.5.3" +version = "0.6.0" edition = "2021" resolver = "2" diff --git a/crates/smtp/Cargo.toml b/crates/smtp/Cargo.toml index 53580d1f..f214910d 100644 --- a/crates/smtp/Cargo.toml +++ b/crates/smtp/Cargo.toml @@ -7,7 +7,7 @@ homepage = "https://stalw.art/smtp" keywords = ["smtp", "email", "mail", "server"] categories = ["email"] license = "AGPL-3.0-only" -version = "0.5.3" +version = "0.6.0" edition = "2021" resolver = "2" diff --git a/crates/smtp/src/core/management.rs b/crates/smtp/src/core/management.rs index 125e6102..86182439 100644 --- a/crates/smtp/src/core/management.rs +++ b/crates/smtp/src/core/management.rs @@ -124,6 +124,14 @@ impl SessionManager for SmtpAdminSessionManager { fn shutdown(&self) -> impl std::future::Future + Send { async {} } + + fn is_ip_blocked(&self, addr: &IpAddr) -> bool { + self.inner + .shared + .default_directory + .blocked_ips + .is_blocked(addr) + } } async fn handle_request( @@ -391,7 +399,9 @@ impl SMTP { let mut result = Vec::with_capacity(queue_ids.len()); for queue_id in queue_ids { if let Some(message) = self.read_message(queue_id).await { - result.push(Message::from(&message)); + result.push(Message::from(&message).into()); + } else { + result.push(None); } } @@ -661,32 +671,34 @@ impl SMTP { domain: String::new(), }), )); - let _ = - self.shared - .default_data_store - .iterate( - IterateParams::new(from_key, to_key).ascending().no_values(), - |key, _| { - if type_.map_or(true, |t| t == *key.last().unwrap()) { - let event = ReportEvent::deserialize(key)?; - if domain.as_ref().map_or(true, |d| { + let _ = self + .shared + .default_data_store + .iterate( + IterateParams::new(from_key, to_key).ascending().no_values(), + |key, _| { + if type_.map_or(true, |t| t == *key.last().unwrap()) { + let event = ReportEvent::deserialize(key)?; + if event.seq_id != 0 + && domain.as_ref().map_or(true, |d| { d.eq_ignore_ascii_case(&event.domain) - }) { - result.push( - if *key.last().unwrap() == 0 { - QueueClass::DmarcReportHeader(event) - } else { - QueueClass::TlsReportHeader(event) - } - .queue_id(), - ); - } + }) + { + result.push( + if *key.last().unwrap() == 0 { + QueueClass::DmarcReportHeader(event) + } else { + QueueClass::TlsReportHeader(event) + } + .queue_id(), + ); } + } - Ok(true) - }, - ) - .await; + Ok(true) + }, + ) + .await; ( StatusCode::OK, @@ -720,13 +732,24 @@ impl SMTP { } } + let mut result = Vec::with_capacity(report_ids.len()); + for report_id in report_ids { + if let Ok(Some(_)) = self + .shared + .default_data_store + .get_value::<()>(ValueKey::from(ValueClass::Queue(report_id.clone()))) + .await + { + result.push(Report::from(report_id).into()); + } else { + result.push(None); + } + } + match error { None => ( StatusCode::OK, - serde_json::to_string(&Response { - data: report_ids.into_iter().map(Report::from).collect::>(), - }) - .unwrap_or_default(), + serde_json::to_string(&Response { data: result }).unwrap_or_default(), ), Some(error) => error.into_bad_request(), } @@ -873,14 +896,14 @@ impl From for Report { QueueClass::DmarcReportHeader(event) => Report { domain: event.domain, type_: "dmarc".to_string(), - range_from: DateTime::from_timestamp(event.due as i64), + range_from: DateTime::from_timestamp(event.seq_id as i64), range_to: DateTime::from_timestamp(event.due as i64), size: 0, }, QueueClass::TlsReportHeader(event) => Report { domain: event.domain, type_: "tls".to_string(), - range_from: DateTime::from_timestamp(event.due as i64), + range_from: DateTime::from_timestamp(event.seq_id as i64), range_to: DateTime::from_timestamp(event.due as i64), size: 0, }, diff --git a/crates/smtp/src/core/mod.rs b/crates/smtp/src/core/mod.rs index 7808333c..9722f4ce 100644 --- a/crates/smtp/src/core/mod.rs +++ b/crates/smtp/src/core/mod.rs @@ -388,7 +388,6 @@ static ref SIEVE: Arc = Arc::new(utils::listener::ServerInstance limiter: utils::listener::limiter::ConcurrencyLimiter::new(0), shutdown_rx: tokio::sync::watch::channel(false).1, proxy_networks: vec![], - blocked_ips: Arc::new(Default::default()), }); } diff --git a/crates/smtp/src/core/throttle.rs b/crates/smtp/src/core/throttle.rs index f2a336dd..0041d65a 100644 --- a/crates/smtp/src/core/throttle.rs +++ b/crates/smtp/src/core/throttle.rs @@ -238,32 +238,9 @@ impl Session { // Build throttle key let key = t.new_key(self); - // Check rate - if let Some(rate) = &t.rate { - if self - .core - .shared - .default_lookup_store - .is_rate_allowed(key.hash.as_slice(), rate, false) - .await - .unwrap_or_default() - .is_some() - { - tracing::debug!( - parent: &self.span, - context = "throttle", - event = "rate-limit-exceeded", - max_requests = rate.requests, - max_interval = rate.period.as_secs(), - "Rate limit exceeded." - ); - return false; - } - } - // Check concurrency if let Some(concurrency) = &t.concurrency { - match self.core.session.throttle.entry(key) { + match self.core.session.throttle.entry(key.clone()) { Entry::Occupied(mut e) => { let limiter = e.get_mut(); if let Some(inflight) = limiter.is_allowed() { @@ -288,6 +265,29 @@ impl Session { } } } + + // Check rate + if let Some(rate) = &t.rate { + if self + .core + .shared + .default_lookup_store + .is_rate_allowed(key.hash.as_slice(), rate, false) + .await + .unwrap_or_default() + .is_some() + { + tracing::debug!( + parent: &self.span, + context = "throttle", + event = "rate-limit-exceeded", + max_requests = rate.requests, + max_interval = rate.period.as_secs(), + "Rate limit exceeded." + ); + return false; + } + } } } diff --git a/crates/smtp/src/inbound/spawn.rs b/crates/smtp/src/inbound/spawn.rs index e968dd6d..c5cc24f4 100644 --- a/crates/smtp/src/inbound/spawn.rs +++ b/crates/smtp/src/inbound/spawn.rs @@ -21,7 +21,7 @@ * for more details. */ -use std::time::Instant; +use std::{net::IpAddr, time::Instant}; use tokio_rustls::server::TlsStream; use utils::listener::{SessionManager, SessionStream}; @@ -76,6 +76,14 @@ impl SessionManager for SmtpSessionManager { .await; } } + + fn is_ip_blocked(&self, addr: &IpAddr) -> bool { + self.inner + .shared + .default_directory + .blocked_ips + .is_blocked(addr) + } } impl Session { diff --git a/crates/smtp/src/outbound/delivery.rs b/crates/smtp/src/outbound/delivery.rs index c7496a0f..4f38bd5b 100644 --- a/crates/smtp/src/outbound/delivery.rs +++ b/crates/smtp/src/outbound/delivery.rs @@ -33,13 +33,13 @@ use mail_auth::{ }; use mail_send::SmtpClient; use smtp_proto::MAIL_REQUIRETLS; -use store::write::now; +use store::write::{now, BatchBuilder, QueueClass, QueueEvent, ValueClass}; use utils::config::ServerProtocol; use crate::{ config::{AggregateFrequency, RequireOptional, TlsStrategy}, core::SMTP, - queue::ErrorDetails, + queue::{ErrorDetails, Message}, reporting::{tls::TlsRptOptions, PolicyType, TlsEvent}, }; @@ -56,18 +56,51 @@ use crate::queue::{ impl DeliveryAttempt { pub async fn try_deliver(mut self, core: Arc) { tokio::spawn(async move { + // Lock message + self.event = if let Some(event) = core.try_lock_event(self.event).await { + event + } else { + return; + }; + + // Fetch message + let mut message = if let Some(message) = core.read_message(self.event.queue_id).await { + message + } else { + // Message no longer exists, delete queue event. + let mut batch = BatchBuilder::new(); + batch.clear(ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { + due: self.event.due, + queue_id: self.event.queue_id, + }))); + let _ = core.shared.default_data_store.write(batch.build()).await; + return; + }; + + let span = tracing::info_span!( + "delivery", + "id" = message.id, + "return_path" = if !message.return_path.is_empty() { + message.return_path.as_ref() + } else { + "<>" + }, + "nrcpt" = message.recipients.len(), + "size" = message.size + ); + // Check that the message still has recipients to be delivered - let has_pending_delivery = self.has_pending_delivery(); + let has_pending_delivery = message.has_pending_delivery(&span); // Send any due Delivery Status Notifications - core.send_dsn(&mut self).await; + core.send_dsn(&mut message, &span).await; if has_pending_delivery { // Re-queue the message if its not yet due for delivery - let due = self.message.next_delivery_event(); + let due = message.next_delivery_event(); if due > now() { // Save changes - self.message + message .save_changes(&core, self.event.due.into(), due.into()) .await; if core.queue.tx.send(Event::Reload).await.is_err() { @@ -77,7 +110,7 @@ impl DeliveryAttempt { } } else { // All message recipients expired, do not re-queue. (DSN has been already sent) - self.message.remove(&core, self.event.due).await; + message.remove(&core, self.event.due).await; if core.queue.tx.send(Event::Reload).await.is_err() { tracing::warn!("Channel closed while trying to notify queue manager."); } @@ -88,14 +121,14 @@ impl DeliveryAttempt { // Throttle sender for throttle in &core.queue.config.throttle.sender { if let Err(err) = core - .is_allowed(throttle, &self.message, &mut self.in_flight, &self.span) + .is_allowed(throttle, &message, &mut self.in_flight, &span) .await { let event = match err { throttle::Error::Concurrency { limiter } => { // Save changes to disk - let next_due = self.message.next_event_after(now()); - self.message.save_changes(&core, None, None).await; + let next_due = message.next_event_after(now()); + message.save_changes(&core, None, None).await; Event::OnHold(OnHold { next_due, @@ -107,9 +140,9 @@ impl DeliveryAttempt { // Save changes to disk let next_event = std::cmp::min( retry_at, - self.message.next_event_after(now()).unwrap_or(u64::MAX), + message.next_event_after(now()).unwrap_or(u64::MAX), ); - self.message + message .save_changes(&core, self.event.due.into(), next_event.into()) .await; @@ -128,8 +161,8 @@ impl DeliveryAttempt { let mut on_hold = Vec::new(); let no_ip = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)); - let mut domains = std::mem::take(&mut self.message.domains); - let mut recipients = std::mem::take(&mut self.message.recipients); + let mut domains = std::mem::take(&mut message.domains); + let mut recipients = std::mem::take(&mut message.recipients); 'next_domain: for (domain_idx, domain) in domains.iter_mut().enumerate() { // Only process domains due for delivery if !matches!(&domain.status, Status::Scheduled | Status::TemporaryFailure(_) @@ -140,7 +173,7 @@ impl DeliveryAttempt { // Create new span for domain let span = tracing::info_span!( - parent: &self.span, + parent: &span, "attempt", domain = domain.domain, attempt_number = domain.retry.inner, @@ -148,7 +181,7 @@ impl DeliveryAttempt { // Build envelope let mut envelope = QueueEnvelope { - message: &self.message, + message: &message, domain: &domain.domain, mx: "", remote_ip: no_ip, @@ -176,8 +209,7 @@ impl DeliveryAttempt { #[cfg(feature = "local_delivery")] Some(next_hop) if next_hop.protocol == ServerProtocol::Jmap => { // Deliver message locally - let delivery_result = self - .message + let delivery_result = message .deliver_local( recipients.iter_mut().filter(|r| r.domain_idx == domain_idx), &core.delivery_tx, @@ -707,7 +739,7 @@ impl DeliveryAttempt { // Prepare TLS connector let is_strict_tls = tls_strategy.is_tls_required() - || (self.message.flags & MAIL_REQUIRETLS) != 0 + || (message.flags & MAIL_REQUIRETLS) != 0 || mta_sts_policy.is_some() || dane_policy.is_some(); let tls_connector = @@ -823,7 +855,7 @@ impl DeliveryAttempt { } // Deliver message over TLS - self.message + message .deliver( smtp_client, recipients @@ -876,7 +908,7 @@ impl DeliveryAttempt { continue 'next_host; } else { // TLS is not required, proceed in plain-text - self.message + message .deliver( smtp_client, recipients @@ -936,7 +968,7 @@ impl DeliveryAttempt { reason = if domain.disable_tls {"TLS is disabled for this host"} else {"TLS is unavailable for this host, falling back to plain-text."}, ); - self.message + message .deliver( smtp_client, recipients @@ -989,7 +1021,7 @@ impl DeliveryAttempt { } // Deliver message - self.message + message .deliver( smtp_client, recipients.iter_mut().filter(|r| r.domain_idx == domain_idx), @@ -1020,18 +1052,18 @@ impl DeliveryAttempt { .unwrap_or_else(|| vec![Duration::from_secs(60)]), ); } - self.message.domains = domains; - self.message.recipients = recipients; + message.domains = domains; + message.recipients = recipients; // Send Delivery Status Notifications - core.send_dsn(&mut self).await; + core.send_dsn(&mut message, &span).await; // Notify queue manager - let span = self.span; + let span = span; let result = if !on_hold.is_empty() { // Save changes to disk - let next_due = self.message.next_event_after(now()); - self.message.save_changes(&core, None, None).await; + let next_due = message.next_event_after(now()); + message.save_changes(&core, None, None).await; tracing::info!( parent: &span, @@ -1046,9 +1078,9 @@ impl DeliveryAttempt { limiters: on_hold, message: self.event, }) - } else if let Some(due) = self.message.next_event() { + } else if let Some(due) = message.next_event() { // Save changes to disk - self.message + message .save_changes(&core, self.event.due.into(), due.into()) .await; @@ -1063,7 +1095,7 @@ impl DeliveryAttempt { Event::Reload } else { // Delete message from queue - self.message.remove(&core, self.event.due).await; + message.remove(&core, self.event.due).await; tracing::info!( parent: &span, @@ -1082,24 +1114,25 @@ impl DeliveryAttempt { } }); } +} +impl Message { /// Marks as failed all domains that reached their expiration time - pub fn has_pending_delivery(&mut self) -> bool { + pub fn has_pending_delivery(&mut self, span: &tracing::Span) -> bool { let now = now(); let mut has_pending_delivery = false; - let span = self.span.clone(); - for (idx, domain) in self.message.domains.iter_mut().enumerate() { + for (idx, domain) in self.domains.iter_mut().enumerate() { match &domain.status { Status::TemporaryFailure(err) if domain.expires <= now => { tracing::info!( - parent: &span, + parent: span, event = "delivery-expired", domain = domain.domain, reason = %err, ); - for rcpt in &mut self.message.recipients { + for rcpt in &mut self.recipients { if rcpt.domain_idx == idx { rcpt.status = std::mem::replace(&mut rcpt.status, Status::Scheduled) .into_permanent(); @@ -1111,13 +1144,13 @@ impl DeliveryAttempt { } Status::Scheduled if domain.expires <= now => { tracing::info!( - parent: &span, + parent: span, event = "delivery-expired", domain = domain.domain, reason = "Queue rate limit exceeded.", ); - for rcpt in &mut self.message.recipients { + for rcpt in &mut self.recipients { if rcpt.domain_idx == idx { rcpt.status = std::mem::replace(&mut rcpt.status, Status::Scheduled) .into_permanent(); diff --git a/crates/smtp/src/outbound/mod.rs b/crates/smtp/src/outbound/mod.rs index b5f3398c..d9c33443 100644 --- a/crates/smtp/src/outbound/mod.rs +++ b/crates/smtp/src/outbound/mod.rs @@ -25,12 +25,11 @@ use std::borrow::Cow; use mail_send::Credentials; use smtp_proto::{Response, Severity}; -use store::write::QueueEvent; use utils::config::ServerProtocol; use crate::{ config::RelayHost, - queue::{DeliveryAttempt, Error, ErrorDetails, HostResponse, Message, Status}, + queue::{spool::QueueEventLock, DeliveryAttempt, Error, ErrorDetails, HostResponse, Status}, }; pub mod dane; @@ -213,21 +212,9 @@ impl From for Status<(), Error> { } impl DeliveryAttempt { - pub fn new(message: Message, event: QueueEvent) -> Self { + pub fn new(event: QueueEventLock) -> Self { DeliveryAttempt { - span: tracing::info_span!( - "delivery", - "id" = message.id, - "return_path" = if !message.return_path.is_empty() { - message.return_path.as_ref() - } else { - "<>" - }, - "nrcpt" = message.recipients.len(), - "size" = message.size - ), in_flight: Vec::new(), - message, event, } } diff --git a/crates/smtp/src/queue/dsn.rs b/crates/smtp/src/queue/dsn.rs index aef875ac..79765313 100644 --- a/crates/smtp/src/queue/dsn.rs +++ b/crates/smtp/src/queue/dsn.rs @@ -36,45 +36,40 @@ use store::write::now; use crate::core::SMTP; use super::{ - DeliveryAttempt, Domain, Error, ErrorDetails, HostResponse, Message, Recipient, SimpleEnvelope, - Status, RCPT_DSN_SENT, RCPT_STATUS_CHANGED, + Domain, Error, ErrorDetails, HostResponse, Message, Recipient, SimpleEnvelope, Status, + RCPT_DSN_SENT, RCPT_STATUS_CHANGED, }; impl SMTP { - pub async fn send_dsn(&self, attempt: &mut DeliveryAttempt) { - if !attempt.message.return_path.is_empty() { - if let Some(dsn) = attempt.build_dsn(self).await { + pub async fn send_dsn(&self, message: &mut Message, span: &tracing::Span) { + if !message.return_path.is_empty() { + if let Some(dsn) = message.build_dsn(self, span).await { let mut dsn_message = self.queue.new_message("", "", ""); dsn_message .add_recipient_parts( - &attempt.message.return_path, - &attempt.message.return_path_lcase, - &attempt.message.return_path_domain, + &message.return_path, + &message.return_path_lcase, + &message.return_path_domain, self, ) .await; // Sign message let signature = self - .sign_message( - &mut attempt.message, - &self.queue.config.dsn.sign, - &dsn, - &attempt.span, - ) + .sign_message(message, &self.queue.config.dsn.sign, &dsn, span) .await; dsn_message - .queue(signature.as_deref(), &dsn, self, &attempt.span) + .queue(signature.as_deref(), &dsn, self, span) .await; } } else { - attempt.handle_double_bounce(); + message.handle_double_bounce(span); } } } -impl DeliveryAttempt { - pub async fn build_dsn(&mut self, core: &SMTP) -> Option> { +impl Message { + pub async fn build_dsn(&mut self, core: &SMTP, span: &tracing::Span) -> Option> { let config = &core.queue.config; let now = now(); @@ -83,11 +78,11 @@ impl DeliveryAttempt { let mut txt_failed = String::new(); let mut dsn = String::new(); - for rcpt in &mut self.message.recipients { + for rcpt in &mut self.recipients { if rcpt.has_flag(RCPT_DSN_SENT | RCPT_NOTIFY_NEVER) { continue; } - let domain = &self.message.domains[rcpt.domain_idx]; + let domain = &self.domains[rcpt.domain_idx]; match &rcpt.status { Status::Completed(response) => { rcpt.flags |= RCPT_DSN_SENT | RCPT_STATUS_CHANGED; @@ -227,14 +222,14 @@ impl DeliveryAttempt { // Update next delay notification time if has_delay { - let mut domains = std::mem::take(&mut self.message.domains); + let mut domains = std::mem::take(&mut self.domains); for domain in &mut domains { if matches!( &domain.status, Status::TemporaryFailure(_) | Status::Scheduled ) && domain.notify.due <= now { - let envelope = SimpleEnvelope::new(&self.message, &domain.domain); + let envelope = SimpleEnvelope::new(self, &domain.domain); if let Some(next_notify) = core .eval_if::, _>(&config.notify, &envelope) @@ -250,34 +245,33 @@ impl DeliveryAttempt { } } } - self.message.domains = domains; + self.domains = domains; } // Obtain hostname and sender addresses let from_name = core - .eval_if(&config.dsn.name, &self.message) + .eval_if(&config.dsn.name, self) .await .unwrap_or_else(|| String::from("Mail Delivery Subsystem")); let from_addr = core - .eval_if(&config.dsn.address, &self.message) + .eval_if(&config.dsn.address, self) .await .unwrap_or_else(|| String::from("MAILER-DAEMON@localhost")); let reporting_mta = core - .eval_if(&config.hostname, &self.message) + .eval_if(&config.hostname, self) .await .unwrap_or_else(|| String::from("localhost")); // Prepare DSN let mut dsn_header = String::with_capacity(dsn.len() + 128); - self.message - .write_dsn_headers(&mut dsn_header, &reporting_mta); + self.write_dsn_headers(&mut dsn_header, &reporting_mta); let dsn = dsn_header + &dsn; // Fetch up to 1024 bytes of message headers let headers = match core .shared .default_blob_store - .get_blob(self.message.blob_hash.as_slice(), 0..1024) + .get_blob(self.blob_hash.as_slice(), 0..1024) .await { Ok(Some(mut buf)) => { @@ -307,21 +301,21 @@ impl DeliveryAttempt { } Ok(None) => { tracing::error!( - parent: &self.span, + parent: span, context = "queue", event = "error", "Failed to open blob {:?}: not found", - self.message.blob_hash + self.blob_hash ); String::new() } Err(err) => { tracing::error!( - parent: &self.span, + parent: span, context = "queue", event = "error", "Failed to open blob {:?}: {}", - self.message.blob_hash, + self.blob_hash, err ); String::new() @@ -331,10 +325,7 @@ impl DeliveryAttempt { // Build message MessageBuilder::new() .from((from_name.as_str(), from_addr.as_str())) - .header( - "To", - HeaderType::Text(self.message.return_path.as_str().into()), - ) + .header("To", HeaderType::Text(self.return_path.as_str().into())) .header("Auto-Submitted", HeaderType::Text("auto-generated".into())) .message_id(format!("<{}@{}>", make_boundary("."), reporting_mta)) .subject(subject) @@ -357,11 +348,10 @@ impl DeliveryAttempt { .into() } - fn handle_double_bounce(&mut self) { + fn handle_double_bounce(&mut self, span: &tracing::Span) { let mut is_double_bounce = Vec::with_capacity(0); - let message = &mut self.message; - for rcpt in &mut message.recipients { + for rcpt in &mut self.recipients { if !rcpt.has_flag(RCPT_DSN_SENT | RCPT_NOTIFY_NEVER) { match &rcpt.status { Status::PermanentFailure(err) => { @@ -371,7 +361,7 @@ impl DeliveryAttempt { is_double_bounce.push(dsn); } Status::Scheduled => { - let domain = &message.domains[rcpt.domain_idx]; + let domain = &self.domains[rcpt.domain_idx]; if let Status::PermanentFailure(err) = &domain.status { rcpt.flags |= RCPT_DSN_SENT; let mut dsn = String::new(); @@ -385,7 +375,7 @@ impl DeliveryAttempt { } let now = now(); - for domain in &mut message.domains { + for domain in &mut self.domains { if domain.notify.due <= now { domain.notify.due = domain.expires + 10; } @@ -393,10 +383,10 @@ impl DeliveryAttempt { if !is_double_bounce.is_empty() { tracing::info!( - parent: &self.span, + parent: span, context = "queue", event = "double-bounce", - id = self.message.id, + id = self.id, failures = ?is_double_bounce, "Failed delivery of message with null return path.", ); diff --git a/crates/smtp/src/queue/manager.rs b/crates/smtp/src/queue/manager.rs index 4dfdc9a5..7379f850 100644 --- a/crates/smtp/src/queue/manager.rs +++ b/crates/smtp/src/queue/manager.rs @@ -26,29 +26,29 @@ use std::{ time::Duration, }; -use store::write::{now, BatchBuilder, QueueClass, QueueEvent, ValueClass}; +use store::write::now; use tokio::sync::mpsc; use crate::core::SMTP; -use super::{DeliveryAttempt, Event, Message, OnHold, Status}; +use super::{spool::QueueEventLock, DeliveryAttempt, Event, Message, OnHold, Status}; pub(crate) const SHORT_WAIT: Duration = Duration::from_millis(1); pub(crate) const LONG_WAIT: Duration = Duration::from_secs(86400 * 365); -#[derive(Debug)] pub struct Queue { - pub on_hold: Vec>, + pub core: Arc, + pub on_hold: Vec>, + pub next_wake_up: Duration, } impl SpawnQueue for mpsc::Receiver { fn spawn(mut self, core: Arc) { tokio::spawn(async move { - let mut queue = Queue::default(); - let mut next_wake_up = SHORT_WAIT; + let mut queue = Queue::new(core); loop { - let on_hold = match tokio::time::timeout(next_wake_up, self.recv()).await { + let on_hold = match tokio::time::timeout(queue.next_wake_up, self.recv()).await { Ok(Some(Event::OnHold(on_hold))) => on_hold.into(), Ok(Some(Event::Stop)) | Ok(None) => { break; @@ -56,46 +56,7 @@ impl SpawnQueue for mpsc::Receiver { _ => None, }; - // Deliver any concurrency limited messages - let mut delete_events = Vec::new(); - while let Some(queue_event) = queue.next_on_hold() { - if let Some(message) = core.read_message(queue_event.queue_id).await { - DeliveryAttempt::new(message, queue_event) - .try_deliver(core.clone()) - .await; - } else { - delete_events.push(queue_event); - } - } - - // Deliver scheduled messages - let now = now(); - next_wake_up = LONG_WAIT; - for queue_event in core.next_event().await { - if queue_event.due <= now { - if let Some(message) = core.read_message(queue_event.queue_id).await { - DeliveryAttempt::new(message, queue_event) - .try_deliver(core.clone()) - .await; - } else { - delete_events.push(queue_event); - } - } else { - next_wake_up = Duration::from_secs(queue_event.due - now); - } - } - - // Delete unlinked events - if !delete_events.is_empty() { - let core = core.clone(); - tokio::spawn(async move { - let mut batch = BatchBuilder::new(); - for queue_event in delete_events { - batch.clear(ValueClass::Queue(QueueClass::MessageEvent(queue_event))); - } - let _ = core.shared.default_data_store.write(batch.build()).await; - }); - } + queue.process_events().await; // Add message on hold if let Some(on_hold) = on_hold { @@ -107,7 +68,37 @@ impl SpawnQueue for mpsc::Receiver { } impl Queue { - pub fn on_hold(&mut self, message: OnHold) { + pub fn new(core: Arc) -> Self { + Queue { + core, + on_hold: Vec::with_capacity(128), + next_wake_up: SHORT_WAIT, + } + } + + pub async fn process_events(&mut self) { + // Deliver any concurrency limited messages + while let Some(queue_event) = self.next_on_hold() { + DeliveryAttempt::new(queue_event) + .try_deliver(self.core.clone()) + .await; + } + + // Deliver scheduled messages + let now = now(); + self.next_wake_up = LONG_WAIT; + for queue_event in self.core.next_event().await { + if queue_event.due <= now { + DeliveryAttempt::new(queue_event) + .try_deliver(self.core.clone()) + .await; + } else { + self.next_wake_up = Duration::from_secs(queue_event.due - now); + } + } + } + + pub fn on_hold(&mut self, message: OnHold) { self.on_hold.push(OnHold { next_due: message.next_due, limiters: message.limiters, @@ -115,7 +106,7 @@ impl Queue { }); } - pub fn next_on_hold(&mut self) -> Option { + pub fn next_on_hold(&mut self) -> Option { let now = now(); self.on_hold .iter() @@ -210,14 +201,6 @@ impl Message { } } -impl Default for Queue { - fn default() -> Self { - Queue { - on_hold: Vec::with_capacity(128), - } - } -} - pub trait SpawnQueue { fn spawn(self, core: Arc); } diff --git a/crates/smtp/src/queue/mod.rs b/crates/smtp/src/queue/mod.rs index 6b1b03a4..a961736b 100644 --- a/crates/smtp/src/queue/mod.rs +++ b/crates/smtp/src/queue/mod.rs @@ -29,7 +29,7 @@ use std::{ use serde::{Deserialize, Serialize}; use smtp_proto::Response; -use store::write::{now, QueueEvent}; +use store::write::now; use utils::{ listener::limiter::{ConcurrencyLimiter, InFlight}, BlobHash, @@ -37,6 +37,8 @@ use utils::{ use crate::core::{eval::*, ResolveVariable}; +use self::spool::QueueEventLock; + pub mod dsn; pub mod manager; pub mod quota; @@ -48,7 +50,7 @@ pub type QueueId = u64; #[derive(Debug)] pub enum Event { Reload, - OnHold(OnHold), + OnHold(OnHold), Stop, } @@ -59,13 +61,13 @@ pub struct OnHold { pub message: T, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Schedule { pub due: u64, pub inner: T, } -#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct Message { pub id: QueueId, pub created: u64, @@ -85,13 +87,13 @@ pub struct Message { pub quota_keys: Vec, } -#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub enum QuotaKey { Size { key: Vec, id: u64 }, Count { key: Vec, id: u64 }, } -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Domain { pub domain: String, pub retry: Schedule, @@ -101,7 +103,7 @@ pub struct Domain { pub disable_tls: bool, } -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Recipient { pub domain_idx: usize, pub address: String, @@ -114,7 +116,7 @@ pub struct Recipient { pub const RCPT_DSN_SENT: u64 = 1 << 32; pub const RCPT_STATUS_CHANGED: u64 = 2 << 32; -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum Status { #[serde(rename = "scheduled")] Scheduled, @@ -126,13 +128,13 @@ pub enum Status { PermanentFailure(E), } -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct HostResponse { pub hostname: T, pub response: Response, } -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum Error { DnsError(String), UnexpectedResponse(HostResponse), @@ -145,17 +147,15 @@ pub enum Error { Io(String), } -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ErrorDetails { pub entity: String, pub details: String, } pub struct DeliveryAttempt { - pub span: tracing::Span, pub in_flight: Vec, - pub message: Message, - pub event: QueueEvent, + pub event: QueueEventLock, } impl Ord for Schedule { diff --git a/crates/smtp/src/queue/quota.rs b/crates/smtp/src/queue/quota.rs index e3fa2268..c8502128 100644 --- a/crates/smtp/src/queue/quota.rs +++ b/crates/smtp/src/queue/quota.rs @@ -107,17 +107,15 @@ impl SMTP { { let key = quota.new_key(envelope); if let Some(max_size) = quota.size { - if self + let used_size = self .shared .default_data_store .get_counter(ValueKey::from(ValueClass::Queue(QueueClass::QuotaSize( key.as_ref().to_vec(), )))) .await - .unwrap_or(0) as usize - + size - > max_size - { + .unwrap_or(0) as usize; + if used_size + size > max_size { return false; } else { refs.push(QuotaKey::Size { @@ -128,17 +126,15 @@ impl SMTP { } if let Some(max_messages) = quota.messages { - if self + let total_messages = self .shared .default_data_store .get_counter(ValueKey::from(ValueClass::Queue(QueueClass::QuotaCount( key.as_ref().to_vec(), )))) .await - .unwrap_or(0) as usize - + 1 - > max_messages - { + .unwrap_or(0) as usize; + if total_messages + 1 > max_messages { return false; } else { refs.push(QuotaKey::Count { @@ -174,15 +170,19 @@ impl Message { quota_ids.push((pos + 1) as u64); } } + if !quota_ids.is_empty() { let mut quota_keys = Vec::new(); for quota_key in std::mem::take(&mut self.quota_keys) { match quota_key { QuotaKey::Count { id, key } if quota_ids.contains(&id) => { - batch.clear(ValueClass::Queue(QueueClass::QuotaCount(key))); + batch.add(ValueClass::Queue(QueueClass::QuotaCount(key)), -1); } QuotaKey::Size { id, key } if quota_ids.contains(&id) => { - batch.clear(ValueClass::Queue(QueueClass::QuotaSize(key))); + batch.add( + ValueClass::Queue(QueueClass::QuotaSize(key)), + -(self.size as i64), + ); } _ => { quota_keys.push(quota_key); diff --git a/crates/smtp/src/queue/spool.rs b/crates/smtp/src/queue/spool.rs index b21371c7..8b362faf 100644 --- a/crates/smtp/src/queue/spool.rs +++ b/crates/smtp/src/queue/spool.rs @@ -26,7 +26,7 @@ use std::borrow::Cow; use std::time::{Duration, SystemTime}; use store::write::key::DeserializeBigEndian; use store::write::{now, BatchBuilder, Bincode, BlobOp, QueueClass, QueueEvent, ValueClass}; -use store::{IterateParams, Serialize, ValueKey, U64_LEN}; +use store::{Deserialize, IterateParams, Serialize, ValueKey, U64_LEN}; use utils::BlobHash; use crate::core::{QueueCore, SMTP}; @@ -35,6 +35,17 @@ use super::{ Domain, Event, Message, QueueId, QuotaKey, Recipient, Schedule, SimpleEnvelope, Status, }; +pub const LOCK_EXPIRY: u64 = 300; +pub const BLOB_EXPIRY: u64 = 3600; +pub const SPOOL_ACCOUNT_ID: u32 = u32::MAX - 1; + +#[derive(Debug)] +pub struct QueueEventLock { + pub due: u64, + pub queue_id: u64, + pub lock_expiry: u64, +} + impl QueueCore { pub fn new_message( &self, @@ -64,7 +75,7 @@ impl QueueCore { } impl SMTP { - pub async fn next_event(&self) -> Vec { + pub async fn next_event(&self) -> Vec { let from_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { due: 0, queue_id: 0, @@ -80,14 +91,26 @@ impl SMTP { .shared .default_data_store .iterate( - IterateParams::new(from_key, to_key).ascending().no_values(), - |key, _| { - let event = QueueEvent { + IterateParams::new(from_key, to_key).ascending(), + |key, value| { + let event = QueueEventLock { due: key.deserialize_be_u64(1)?, queue_id: key.deserialize_be_u64(U64_LEN + 1)?, + lock_expiry: u64::deserialize(value)?, }; let do_continue = event.due <= now; - events.push(event); + if event.lock_expiry < now { + events.push(event); + } else { + tracing::debug!( + context = "queue", + event = "locked", + id = event.queue_id, + due = event.due, + expiry = event.lock_expiry - now, + "Queue event locked by another process." + ); + } Ok(do_continue) }, ) @@ -105,6 +128,47 @@ impl SMTP { events } + pub async fn try_lock_event(&self, mut event: QueueEventLock) -> Option { + let mut batch = BatchBuilder::new(); + batch.assert_value( + ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { + due: event.due, + queue_id: event.queue_id, + })), + event.lock_expiry, + ); + event.lock_expiry = now() + LOCK_EXPIRY; + batch.set( + ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { + due: event.due, + queue_id: event.queue_id, + })), + event.lock_expiry.serialize(), + ); + match self.shared.default_data_store.write(batch.build()).await { + Ok(_) => Some(event), + Err(store::Error::AssertValueFailed) => { + tracing::debug!( + context = "queue", + event = "locked", + id = event.queue_id, + due = event.due, + "Failed to lock event: Event already locked." + ); + None + } + Err(err) => { + tracing::error!( + context = "queue", + event = "error", + "Failed to lock event: {}", + err + ); + None + } + } + } + pub async fn read_message(&self, id: QueueId) -> Option { match self .shared @@ -155,10 +219,10 @@ impl Message { // Reserve and write blob let mut batch = BatchBuilder::new(); - batch.with_account_id(u32::MAX).set( + batch.with_account_id(SPOOL_ACCOUNT_ID).set( BlobOp::Reserve { hash: self.blob_hash.clone(), - until: self.next_delivery_event() + 3600, + until: self.next_delivery_event() + BLOB_EXPIRY, }, 0u32.serialize(), ); @@ -205,12 +269,33 @@ impl Message { // Write message to queue let mut batch = BatchBuilder::new(); + + // Reserve quotas + for quota_key in &self.quota_keys { + match quota_key { + QuotaKey::Count { key, .. } => { + batch.add(ValueClass::Queue(QueueClass::QuotaCount(key.clone())), 1); + } + QuotaKey::Size { key, .. } => { + batch.add( + ValueClass::Queue(QueueClass::QuotaSize(key.clone())), + self.size as i64, + ); + } + } + } batch .set( ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { due: self.next_event().unwrap_or_default(), queue_id: self.id, })), + 0u64.serialize(), + ) + .set( + BlobOp::Commit { + hash: self.blob_hash.clone(), + }, vec![], ) .set( @@ -305,34 +390,33 @@ impl Message { // Update message queue let mut batch = BatchBuilder::new(); - if let Some(prev_event) = prev_event { - batch.clear(ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { - due: prev_event, - queue_id: self.id, - }))); - } - if let Some(next_event) = next_event { - batch.set( - ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { - due: next_event, + if let (Some(prev_event), Some(next_event)) = (prev_event, next_event) { + batch + .clear(ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { + due: prev_event, queue_id: self.id, - })), - vec![], - ); + }))) + .set( + ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { + due: next_event, + queue_id: self.id, + })), + 0u64.serialize(), + ); } - batch - .with_account_id(u32::MAX) - .set( - BlobOp::Reserve { - hash: self.blob_hash.clone(), - until: self.next_delivery_event() + 3600, - }, - 0u32.serialize(), - ) - .set( - ValueClass::Queue(QueueClass::Message(self.id)), - Bincode::new(self).serialize(), - ); + + batch.with_account_id(SPOOL_ACCOUNT_ID).set( + BlobOp::Reserve { + hash: self.blob_hash.clone(), + until: self.next_delivery_event() + BLOB_EXPIRY, + }, + 0u32.serialize(), + ); + + batch.set( + ValueClass::Queue(QueueClass::Message(self.id)), + Bincode::new(self).serialize(), + ); if let Err(err) = core.shared.default_data_store.write(batch.build()).await { tracing::error!( @@ -354,15 +438,30 @@ impl Message { for quota_key in self.quota_keys { match quota_key { QuotaKey::Count { key, .. } => { - batch.clear(ValueClass::Queue(QueueClass::QuotaCount(key))); + batch.add(ValueClass::Queue(QueueClass::QuotaCount(key)), -1); } QuotaKey::Size { key, .. } => { - batch.clear(ValueClass::Queue(QueueClass::QuotaSize(key))); + batch.add( + ValueClass::Queue(QueueClass::QuotaSize(key)), + -(self.size as i64), + ); } } } batch + .with_account_id(SPOOL_ACCOUNT_ID) + .clear(BlobOp::Reserve { + hash: self.blob_hash.clone(), + until: prev_event + BLOB_EXPIRY, + }) + .set( + BlobOp::Reserve { + hash: self.blob_hash.clone(), + until: now() - 1, + }, + 0u32.serialize(), + ) .clear(ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { due: prev_event, queue_id: self.id, diff --git a/crates/smtp/src/reporting/dmarc.rs b/crates/smtp/src/reporting/dmarc.rs index c1b7d626..06adfda1 100644 --- a/crates/smtp/src/reporting/dmarc.rs +++ b/crates/smtp/src/reporting/dmarc.rs @@ -44,7 +44,7 @@ use crate::{ queue::{DomainPart, RecipientDomain}, }; -use super::{scheduler::ToHash, DmarcEvent, SerializedSize}; +use super::{scheduler::ToHash, DmarcEvent, ReportLock, SerializedSize}; #[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct DmarcFormat { @@ -584,6 +584,12 @@ impl SMTP { ValueClass::Queue(QueueClass::DmarcReportHeader(report_event.clone())), Bincode::new(entry).serialize(), ); + + // Add lock + builder.set( + ValueClass::Queue(QueueClass::dmarc_lock(&report_event)), + 0u64.serialize(), + ); } // Write entry diff --git a/crates/smtp/src/reporting/mod.rs b/crates/smtp/src/reporting/mod.rs index 4dd2c6f8..bb9c2c14 100644 --- a/crates/smtp/src/reporting/mod.rs +++ b/crates/smtp/src/reporting/mod.rs @@ -33,6 +33,7 @@ use mail_auth::{ }; use mail_parser::DateTime; +use store::write::{QueueClass, ReportEvent}; use tokio::io::{AsyncRead, AsyncWrite}; use utils::config::if_block::IfBlock; @@ -324,3 +325,28 @@ impl io::Write for SerializedSize { Ok(()) } } + +pub trait ReportLock { + fn tls_lock(event: &ReportEvent) -> Self; + fn dmarc_lock(event: &ReportEvent) -> Self; +} + +impl ReportLock for QueueClass { + fn tls_lock(event: &ReportEvent) -> Self { + QueueClass::TlsReportHeader(ReportEvent { + due: event.due, + policy_hash: 0, + seq_id: 0, + domain: event.domain.clone(), + }) + } + + fn dmarc_lock(event: &ReportEvent) -> Self { + QueueClass::DmarcReportHeader(ReportEvent { + due: event.due, + policy_hash: event.policy_hash, + seq_id: 0, + domain: event.domain.clone(), + }) + } +} diff --git a/crates/smtp/src/reporting/scheduler.rs b/crates/smtp/src/reporting/scheduler.rs index 5cc29268..f05592b7 100644 --- a/crates/smtp/src/reporting/scheduler.rs +++ b/crates/smtp/src/reporting/scheduler.rs @@ -29,17 +29,17 @@ use std::{ time::{Duration, Instant, SystemTime}, }; use store::{ - write::{now, QueueClass, ReportEvent, ValueClass}, - Deserialize, IterateParams, ValueKey, + write::{now, BatchBuilder, QueueClass, ReportEvent, ValueClass}, + Deserialize, IterateParams, Serialize, ValueKey, }; use tokio::sync::mpsc; use crate::{ core::{worker::SpawnCleanup, SMTP}, - queue::manager::LONG_WAIT, + queue::{manager::LONG_WAIT, spool::LOCK_EXPIRY}, }; -use super::Event; +use super::{Event, ReportLock}; impl SpawnReport for mpsc::Receiver { fn spawn(mut self, core: Arc) { @@ -69,7 +69,9 @@ impl SpawnReport for mpsc::Receiver { for report_event in events { match report_event { QueueClass::DmarcReportHeader(event) if event.due <= now => { - core_.generate_dmarc_report(event).await; + if core_.try_lock_report(QueueClass::dmarc_lock(&event)).await { + core_.generate_dmarc_report(event).await; + } } QueueClass::TlsReportHeader(event) if event.due <= now => { tls_reports @@ -82,7 +84,12 @@ impl SpawnReport for mpsc::Receiver { } for (domain_name, tls_report) in tls_reports { - core_.generate_tls_report(domain_name, tls_report).await; + if core_ + .try_lock_report(QueueClass::tls_lock(tls_report.first().unwrap())) + .await + { + core_.generate_tls_report(domain_name, tls_report).await; + } } }); @@ -138,6 +145,10 @@ impl SMTP { IterateParams::new(from_key, to_key).ascending().no_values(), |key, _| { let event = ReportEvent::deserialize(key)?; + if event.seq_id == 0 { + // Skip lock + return Ok(true); + } let do_continue = event.due <= now; events.push(if *key.last().unwrap() == 0 { QueueClass::DmarcReportHeader(event) @@ -160,6 +171,76 @@ impl SMTP { events } + + pub async fn try_lock_report(&self, lock: QueueClass) -> bool { + let now = now(); + match self + .shared + .default_data_store + .get_value::(ValueKey::from(ValueClass::Queue(lock.clone()))) + .await + { + Ok(Some(expiry)) => { + if expiry < now { + let mut batch = BatchBuilder::new(); + batch.assert_value(ValueClass::Queue(lock.clone()), expiry); + batch.set( + ValueClass::Queue(lock.clone()), + (now + LOCK_EXPIRY).serialize(), + ); + match self.shared.default_data_store.write(batch.build()).await { + Ok(_) => true, + Err(store::Error::AssertValueFailed) => { + tracing::debug!( + context = "queue", + event = "locked", + key = ?lock, + "Failed to lock report: Event already locked." + ); + false + } + Err(err) => { + tracing::error!( + context = "queue", + event = "error", + "Failed to lock report: {}", + err + ); + false + } + } + } else { + tracing::debug!( + context = "queue", + event = "locked", + key = ?lock, + expiry = expiry - now, + "Failed to lock report: Report already locked." + ); + false + } + } + Ok(None) => { + tracing::debug!( + context = "queue", + event = "locked", + key = ?lock, + "Failed to lock report: Report lock deleted." + ); + false + } + Err(err) => { + tracing::error!( + context = "queue", + event = "error", + key = ?lock, + "Failed to lock report: {}", + err + ); + false + } + } + } } pub trait ToHash { diff --git a/crates/smtp/src/reporting/tls.rs b/crates/smtp/src/reporting/tls.rs index a65e7e0e..dbe64c2f 100644 --- a/crates/smtp/src/reporting/tls.rs +++ b/crates/smtp/src/reporting/tls.rs @@ -48,7 +48,7 @@ use crate::{ USER_AGENT, }; -use super::{scheduler::ToHash, SerializedSize, TlsEvent}; +use super::{scheduler::ToHash, ReportLock, SerializedSize, TlsEvent}; #[derive(Debug, Clone)] pub struct TlsRptOptions { @@ -57,7 +57,7 @@ pub struct TlsRptOptions { } #[derive(Debug, serde::Serialize, serde::Deserialize)] -struct TlsFormat { +pub struct TlsFormat { rua: Vec, policy: PolicyDetails, records: Vec>, @@ -172,10 +172,9 @@ impl SMTP { if let Some(failure_details) = Bincode::>::deserialize(v)?.inner { - total_failure += 1; - match record_map.entry(failure_details) { Entry::Occupied(mut e) => { + total_failure += 1; *e.get_mut() += 1; Ok(true) } @@ -183,6 +182,7 @@ impl SMTP { if serde::Serialize::serialize(e.key(), &mut serialized_size) .is_ok() { + total_failure += 1; e.insert(1u32); Ok(true) } else { @@ -257,7 +257,7 @@ impl SMTP { for uri in &rua { match uri { ReportUri::Http(uri) => { - if let Ok(client) = reqwest::blocking::Client::builder() + if let Ok(client) = reqwest::Client::builder() .user_agent(USER_AGENT) .timeout(Duration::from_secs(2 * 60)) .build() @@ -274,6 +274,7 @@ impl SMTP { .header(CONTENT_TYPE, "application/tlsrpt+gzip") .body(json.to_vec()) .send() + .await { Ok(response) => { if response.status().is_success() { @@ -452,6 +453,12 @@ impl SMTP { ValueClass::Queue(QueueClass::TlsReportHeader(report_event.clone())), Bincode::new(entry).serialize(), ); + + // Add lock + builder.set( + ValueClass::Queue(QueueClass::tls_lock(&report_event)), + 0u64.serialize(), + ); } // Write entry @@ -474,7 +481,7 @@ impl SMTP { pub async fn delete_tls_report(&self, events: Vec) { let mut batch = BatchBuilder::new(); - for event in events { + for (pos, event) in events.into_iter().enumerate() { let from_key = ReportEvent { due: event.due, policy_hash: event.policy_hash, @@ -488,6 +495,7 @@ impl SMTP { domain: event.domain.clone(), }; + // Remove report events if let Err(err) = self .shared .default_data_store @@ -500,12 +508,18 @@ impl SMTP { tracing::warn!( context = "report", event = "error", - "Failed to remove repors: {}", + "Failed to remove reports: {}", err ); return; } + if pos == 0 { + // Remove lock + batch.clear(ValueClass::Queue(QueueClass::tls_lock(&event))); + } + + // Remove report header batch.clear(ValueClass::Queue(QueueClass::TlsReportHeader(event))); } @@ -513,7 +527,7 @@ impl SMTP { tracing::warn!( context = "report", event = "error", - "Failed to remove repors: {}", + "Failed to remove reports: {}", err ); } diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index f53ff273..804e1457 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -45,6 +45,7 @@ async-trait = "0.1.68" redis = { version = "0.24.0", features = [ "tokio-comp", "tokio-rustls-comp", "tls-rustls-insecure", "tls-rustls-webpki-roots", "cluster-async"], optional = true } deadpool = { version = "0.10.0", features = ["managed"], optional = true } bincode = "1.3.3" +arc-swap = "1.6.0" [dev-dependencies] tokio = { version = "1.23", features = ["full"] } diff --git a/crates/store/src/backend/foundationdb/blob.rs b/crates/store/src/backend/foundationdb/blob.rs index 390575a7..eb7e1c10 100644 --- a/crates/store/src/backend/foundationdb/blob.rs +++ b/crates/store/src/backend/foundationdb/blob.rs @@ -25,8 +25,9 @@ use std::ops::Range; use foundationdb::{options::StreamingMode, FdbError, KeySelector, RangeOption}; use futures::StreamExt; +use utils::BLOB_HASH_LEN; -use crate::{write::key::KeySerializer, Error, BLOB_HASH_LEN, SUBSPACE_BLOBS}; +use crate::{write::key::KeySerializer, Error, SUBSPACE_BLOBS}; use super::{FdbStore, MAX_VALUE_SIZE}; diff --git a/crates/store/src/backend/foundationdb/write.rs b/crates/store/src/backend/foundationdb/write.rs index 2ea9a100..f8e27558 100644 --- a/crates/store/src/backend/foundationdb/write.rs +++ b/crates/store/src/backend/foundationdb/write.rs @@ -40,7 +40,8 @@ use crate::{ key::KeySerializer, Batch, BitmapClass, Operation, ValueClass, ValueOp, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, }, - BitmapKey, IndexKey, Key, LogKey, ValueKey, SUBSPACE_BITMAPS, SUBSPACE_VALUES, WITH_SUBSPACE, + BitmapKey, IndexKey, Key, LogKey, ValueKey, SUBSPACE_BITMAPS, SUBSPACE_COUNTERS, + SUBSPACE_VALUES, WITH_SUBSPACE, }; use super::{ @@ -420,18 +421,58 @@ impl FdbStore { } } } + + // Obtain all zero counters + let trx = self.db.create_trx()?; + let mut iter = trx.get_ranges( + RangeOption { + begin: KeySelector::first_greater_or_equal(&[SUBSPACE_COUNTERS, 0u8][..]), + end: KeySelector::first_greater_or_equal( + &[ + SUBSPACE_COUNTERS, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + ][..], + ), + mode: options::StreamingMode::WantAll, + reverse: false, + ..Default::default() + }, + true, + ); + + while let Some(values) = iter.next().await { + for value in values? { + if value.value().iter().all(|byte| *byte == 0) { + delete_keys.push(value.key().to_vec()); + } + } + } + if delete_keys.is_empty() { return Ok(()); } // Delete keys let bitmap = DenseBitmap::empty(); + let integer = 0i64.to_le_bytes(); for chunk in delete_keys.chunks(1024) { let mut retry_count = 0; loop { let trx = self.db.create_trx()?; for key in chunk { - trx.atomic_op(key, &bitmap.bitmap, MutationType::CompareAndClear); + trx.atomic_op( + key, + if key[0] == SUBSPACE_BITMAPS { + &bitmap.bitmap + } else { + &integer + }, + MutationType::CompareAndClear, + ); } match trx.commit().await { Ok(_) => { diff --git a/crates/store/src/backend/mysql/write.rs b/crates/store/src/backend/mysql/write.rs index 141974b1..419ed37c 100644 --- a/crates/store/src/backend/mysql/write.rs +++ b/crates/store/src/backend/mysql/write.rs @@ -31,7 +31,7 @@ use crate::{ write::{ Batch, BitmapClass, Operation, ValueClass, ValueOp, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, }, - BitmapKey, IndexKey, Key, LogKey, ValueKey, + BitmapKey, IndexKey, Key, LogKey, ValueKey, SUBSPACE_COUNTERS, }; use super::MysqlStore; @@ -271,8 +271,15 @@ impl MysqlStore { } pub(crate) async fn purge_bitmaps(&self) -> crate::Result<()> { - // Not needed for PostgreSQL - Ok(()) + let mut conn = self.conn_pool.get_conn().await?; + + let s = conn + .prep(&format!( + "DELETE FROM {} WHERE v = 0", + char::from(SUBSPACE_COUNTERS), + )) + .await?; + conn.exec_drop(&s, ()).await.map_err(Into::into) } pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> crate::Result<()> { diff --git a/crates/store/src/backend/postgres/write.rs b/crates/store/src/backend/postgres/write.rs index 6373ecfb..9846fcd0 100644 --- a/crates/store/src/backend/postgres/write.rs +++ b/crates/store/src/backend/postgres/write.rs @@ -32,7 +32,7 @@ use crate::{ write::{ Batch, BitmapClass, Operation, ValueClass, ValueOp, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, }, - BitmapKey, IndexKey, Key, LogKey, ValueKey, + BitmapKey, IndexKey, Key, LogKey, ValueKey, SUBSPACE_COUNTERS, }; use super::PostgresStore; @@ -288,8 +288,15 @@ impl PostgresStore { } pub(crate) async fn purge_bitmaps(&self) -> crate::Result<()> { - // Not needed for PostgreSQL - Ok(()) + let conn = self.conn_pool.get().await?; + + let s = conn + .prepare_cached(&format!( + "DELETE FROM {} WHERE v = 0", + char::from(SUBSPACE_COUNTERS), + )) + .await?; + conn.execute(&s, &[]).await.map(|_| ()).map_err(Into::into) } pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> crate::Result<()> { diff --git a/crates/store/src/backend/redis/lookup.rs b/crates/store/src/backend/redis/lookup.rs index 431405fa..814fcd52 100644 --- a/crates/store/src/backend/redis/lookup.rs +++ b/crates/store/src/backend/redis/lookup.rs @@ -153,9 +153,10 @@ impl RedisStore { .incr(&key, value) .expire(&key, expires as i64) .ignore() - .query_async(conn) + .query_async::<_, Vec>(conn) .await .map_err(Into::into) + .map(|v| v.first().copied().unwrap_or(0)) } else { conn.incr(&key, value).await.map_err(Into::into) } diff --git a/crates/store/src/backend/rocksdb/write.rs b/crates/store/src/backend/rocksdb/write.rs index 5e93426b..bf4e393e 100644 --- a/crates/store/src/backend/rocksdb/write.rs +++ b/crates/store/src/backend/rocksdb/write.rs @@ -40,9 +40,10 @@ use super::{ }; use crate::{ write::{ - Batch, BitmapClass, Operation, ValueClass, ValueOp, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, + Batch, BitmapClass, LookupClass, Operation, ValueClass, ValueOp, MAX_COMMIT_ATTEMPTS, + MAX_COMMIT_TIME, }, - BitmapKey, Deserialize, IndexKey, Key, LogKey, ValueKey, WITHOUT_BLOCK_NUM, + BitmapKey, Deserialize, IndexKey, Key, LogKey, ValueKey, SUBSPACE_COUNTERS, WITHOUT_BLOCK_NUM, }; impl RocksDbStore { @@ -120,7 +121,40 @@ impl RocksDbStore { } pub(crate) async fn purge_bitmaps(&self) -> crate::Result<()> { - Ok(()) + let db = self.db.clone(); + self.spawn_worker(move || { + let cf = db + .cf_handle(std::str::from_utf8(&[SUBSPACE_COUNTERS]).unwrap()) + .unwrap(); + + let mut delete_keys = Vec::new(); + + for row in db.iterator_cf(&cf, IteratorMode::Start) { + let (key, value) = row?; + + if i64::deserialize(&value)? <= 0 { + delete_keys.push(key); + } + } + + let txn_opts = OptimisticTransactionOptions::default(); + for key in delete_keys { + let txn = db.transaction_opt(&WriteOptions::default(), &txn_opts); + if txn + .get_pinned_for_update_cf(&cf, &key, true)? + .map(|value| i64::deserialize(&value).map(|v| v == 0).unwrap_or(false)) + .unwrap_or(false) + { + txn.delete(key)?; + txn.commit()?; + } else { + txn.rollback()?; + } + } + + Ok(()) + }) + .await } } @@ -227,7 +261,14 @@ impl<'x> RocksDBTransaction<'x> { } } } else { - txn.delete_cf(&self.cf_values, &key)?; + txn.delete_cf( + if matches!(class, ValueClass::Lookup(LookupClass::Counter(_))) { + &self.cf_counters + } else { + &self.cf_values + }, + &key, + )?; } } Operation::Index { field, key, set } => { @@ -347,7 +388,14 @@ impl<'x> RocksDBTransaction<'x> { if let ValueOp::Set(value) = op { wb.put_cf(&self.cf_values, &key, value); } else { - wb.delete_cf(&self.cf_values, &key); + wb.delete_cf( + if matches!(class, ValueClass::Lookup(LookupClass::Counter(_))) { + &self.cf_counters + } else { + &self.cf_values + }, + &key, + ); } } Operation::Index { field, key, set } => { diff --git a/crates/store/src/backend/sqlite/write.rs b/crates/store/src/backend/sqlite/write.rs index 1db9d9c2..27ea4690 100644 --- a/crates/store/src/backend/sqlite/write.rs +++ b/crates/store/src/backend/sqlite/write.rs @@ -25,7 +25,7 @@ use rusqlite::{params, OptionalExtension, TransactionBehavior}; use crate::{ write::{Batch, BitmapClass, Operation, ValueClass, ValueOp}, - BitmapKey, IndexKey, Key, LogKey, ValueKey, + BitmapKey, IndexKey, Key, LogKey, ValueKey, SUBSPACE_COUNTERS, }; use super::SqliteStore; @@ -204,7 +204,17 @@ impl SqliteStore { } pub(crate) async fn purge_bitmaps(&self) -> crate::Result<()> { - Ok(()) + let conn = self.conn_pool.get()?; + self.spawn_worker(move || { + conn.prepare_cached(&format!( + "DELETE FROM {} WHERE v = 0", + char::from(SUBSPACE_COUNTERS), + ))? + .execute([])?; + + Ok(()) + }) + .await } pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> crate::Result<()> { diff --git a/crates/utils/src/listener/blocked.rs b/crates/store/src/dispatch/blocked.rs similarity index 74% rename from crates/utils/src/listener/blocked.rs rename to crates/store/src/dispatch/blocked.rs index 7e33fdcb..f34d1a7f 100644 --- a/crates/utils/src/listener/blocked.rs +++ b/crates/store/src/dispatch/blocked.rs @@ -30,42 +30,35 @@ use std::{ }, }; -use ahash::{AHashMap, AHashSet}; +use ahash::AHashSet; use arc_swap::{ArcSwap, ArcSwapOption}; -use parking_lot::{Mutex, RwLock}; +use parking_lot::RwLock; +use utils::config::{ipmask::IpAddrMask, utils::ParseKey, Config, ConfigKey, Rate}; -use crate::config::{ipmask::IpAddrMask, utils::ParseKey, Config, ConfigKey, Rate}; - -use super::limiter::RateLimiter; +use crate::LookupStore; pub struct BlockedIps { ip_addresses: RwLock>, ip_networks: ArcSwap>, has_networks: AtomicBool, - limiters: Mutex>, + store: LookupStore, limiter_rate: ArcSwapOption, } -#[derive(Debug, PartialEq, Eq, Hash)] -enum LimitBy { - IpAddr(IpAddr), - Login(String), -} - pub const BLOCKED_IP_KEY: &str = "server.security.blocked-networks"; impl BlockedIps { - pub fn new() -> Self { + pub fn new(store: LookupStore) -> Self { Self { ip_addresses: RwLock::new(AHashSet::new()), ip_networks: ArcSwap::new(Arc::new(Vec::new())), - limiters: Mutex::new(Default::default()), - limiter_rate: ArcSwapOption::empty(), has_networks: AtomicBool::new(false), + limiter_rate: ArcSwapOption::empty(), + store, } } - pub fn reload(&self, config: &Config) -> crate::config::Result<()> { + pub fn reload(&self, config: &Config) -> utils::config::Result<()> { self.limiter_rate.store( config .property::("server.security.fail2ban")? @@ -74,7 +67,7 @@ impl BlockedIps { self.reload_blocked_ips(config) } - pub fn reload_blocked_ips(&self, config: &Config) -> crate::config::Result<()> { + pub fn reload_blocked_ips(&self, config: &Config) -> utils::config::Result<()> { let mut ip_addresses = AHashSet::new(); let mut ip_networks = Vec::new(); @@ -94,21 +87,20 @@ impl BlockedIps { Ok(()) } - pub fn is_fail2banned(&self, ip: IpAddr, login: String) -> Option { + pub async fn is_fail2banned(&self, ip: IpAddr, login: String) -> Option { if let Some(rate) = self.limiter_rate.load().as_ref() { let is_allowed = self - .limiters - .lock() - .entry(LimitBy::IpAddr(ip)) - .or_insert_with(|| RateLimiter::new(rate)) - .is_allowed(rate) + .store + .is_rate_allowed(format!("b:{}", ip).as_bytes(), rate.as_ref(), false) + .await + .map(|v| v.is_none()) + .unwrap_or(false) && self - .limiters - .lock() - .entry(LimitBy::Login(login)) - .or_insert_with(|| RateLimiter::new(rate)) - .is_allowed(rate); - + .store + .is_rate_allowed(format!("b:{}", login).as_bytes(), rate.as_ref(), false) + .await + .map(|v| v.is_none()) + .unwrap_or(false); if !is_allowed { self.ip_addresses.write().insert(ip); return Some(ConfigKey { @@ -125,12 +117,6 @@ impl BlockedIps { self.limiter_rate.load().is_some() } - pub fn cleanup(&self) { - self.limiters - .lock() - .retain(|_, limiter| limiter.is_active()); - } - pub fn is_blocked(&self, ip: &IpAddr) -> bool { self.ip_addresses.read().contains(ip) || (self.has_networks.load(Ordering::Relaxed) @@ -147,14 +133,7 @@ impl Debug for BlockedIps { f.debug_struct("BlockedIps") .field("ip_addresses", &self.ip_addresses) .field("ip_networks", &self.ip_networks) - .field("limiters", &self.limiters) .field("limiter_rate", &self.limiter_rate) .finish() } } - -impl Default for BlockedIps { - fn default() -> Self { - Self::new() - } -} diff --git a/crates/store/src/dispatch/lookup.rs b/crates/store/src/dispatch/lookup.rs index 74ae894f..ab3eed28 100644 --- a/crates/store/src/dispatch/lookup.rs +++ b/crates/store/src/dispatch/lookup.rs @@ -271,19 +271,19 @@ impl LookupStore { let requests = if !soft_check { let requests = self.counter_incr(bucket, 1, expires_in.into()).await?; if requests > 0 { - requests - 1 + requests } else { // Increment and get not supported by store, fetch counter let mut bucket = Vec::with_capacity(key.len() + U64_LEN); bucket.extend_from_slice(key); bucket.extend_from_slice(range_start.to_be_bytes().as_slice()); - self.counter_get(bucket).await?.saturating_sub(1) + self.counter_get(bucket).await? } } else { - self.counter_get(bucket).await? + self.counter_get(bucket).await? + 1 }; - if requests < rate.requests as i64 { + if requests <= rate.requests as i64 { Ok(None) } else { Ok(Some(expires_in)) @@ -302,7 +302,7 @@ impl LookupStore { let mut expired_keys = Vec::new(); store .iterate(IterateParams::new(from_key, to_key), |key, value| { - if value.deserialize_be_u64(0)? < current_time { + if value.deserialize_be_u64(0)? <= current_time { expired_keys.push(key.get(1..).unwrap_or_default().to_vec()); } Ok(true) @@ -337,7 +337,7 @@ impl LookupStore { let mut expired_keys = Vec::new(); store .iterate(IterateParams::new(from_key, to_key), |key, value| { - if value.deserialize_be_u64(0)? < current_time { + if value.deserialize_be_u64(0)? <= current_time { expired_keys.push(key.get(1..).unwrap_or_default().to_vec()); } Ok(true) diff --git a/crates/store/src/dispatch/mod.rs b/crates/store/src/dispatch/mod.rs index c4641e59..1a361edb 100644 --- a/crates/store/src/dispatch/mod.rs +++ b/crates/store/src/dispatch/mod.rs @@ -22,6 +22,7 @@ */ pub mod blob; +pub mod blocked; pub mod config; pub mod fts; pub mod lookup; diff --git a/crates/store/src/dispatch/store.rs b/crates/store/src/dispatch/store.rs index c2928624..fe9ed1b5 100644 --- a/crates/store/src/dispatch/store.rs +++ b/crates/store/src/dispatch/store.rs @@ -550,14 +550,14 @@ impl Store { ); } SUBSPACE_VALUES - if key[0] == 3 - || key[0] >= 20 + if [3, 9, 10].contains(&key[0]) + || (key[0] >= 20 && key[0] < 30) || key.get(1..5).unwrap_or_default() == u32::MAX.to_be_bytes() => { // Ignore lastId counter and ID mappings return Ok(true); } - SUBSPACE_COUNTERS if key.len() <= 4 => { + SUBSPACE_COUNTERS if key[0] == 9 || key.len() <= 4 => { // Ignore named keys return Ok(true); } diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 33101bda..d2b48d4d 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -700,6 +700,23 @@ impl Stores { }) } + pub fn get_lookup_store( + &self, + config: &utils::config::Config, + key: &str, + ) -> utils::config::Result { + self.lookup_stores + .get(config.value_require(key)?) + .cloned() + .ok_or_else(|| { + format!( + "Unable to find Lookup store '{}' defined in key '{}'", + config.value_require(key).unwrap(), + key + ) + }) + } + pub fn get_fts_store( &self, config: &utils::config::Config, diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 34cd574b..f9837213 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "utils" -version = "0.5.3" +version = "0.6.0" edition = "2021" resolver = "2" diff --git a/crates/utils/src/config/listener.rs b/crates/utils/src/config/listener.rs index c506e0f2..d2494653 100644 --- a/crates/utils/src/config/listener.rs +++ b/crates/utils/src/config/listener.rs @@ -43,7 +43,6 @@ use tokio_rustls::TlsAcceptor; use crate::{ acme::{directory::ACME_TLS_ALPN_NAME, AcmeManager}, listener::{ - blocked::BlockedIps, tls::{Certificate, CertificateResolver}, TcpAcceptor, }, @@ -66,8 +65,7 @@ impl Config { // Parse servers for (internal_id, id) in self.sub_keys("server.listener", ".protocol").enumerate() { - let mut server = - self.parse_server(id, &certificates, &acmes, servers.blocked_ips.clone())?; + let mut server = self.parse_server(id, &certificates, &acmes)?; if !servers.inner.iter().any(|s| s.id == server.id) { server.internal_id = internal_id as u16; servers.inner.push(server); @@ -116,7 +114,6 @@ impl Config { id: &str, certificates: &AHashMap>, acmes: &AHashMap>, - blocked_ips: Arc, ) -> super::Result { // Build listeners let mut listeners = Vec::new(); @@ -378,7 +375,6 @@ impl Config { acceptor, tls_implicit, proxy_networks, - blocked_ips, }) } } diff --git a/crates/utils/src/config/mod.rs b/crates/utils/src/config/mod.rs index 6e9ea4fd..fc346af2 100644 --- a/crates/utils/src/config/mod.rs +++ b/crates/utils/src/config/mod.rs @@ -37,7 +37,7 @@ use tokio::net::TcpSocket; use crate::{ acme::AcmeManager, failed, - listener::{blocked::BlockedIps, tls::Certificate, TcpAcceptor}, + listener::{tls::Certificate, TcpAcceptor}, UnwrapFailure, }; @@ -63,7 +63,6 @@ pub struct Server { pub protocol: ServerProtocol, pub listeners: Vec, pub proxy_networks: Vec, - pub blocked_ips: Arc, pub acceptor: TcpAcceptor, pub tls_implicit: bool, pub max_connections: u64, @@ -74,7 +73,6 @@ pub struct Servers { pub inner: Vec, pub certificates: Vec>, pub acme_managers: Vec>, - pub blocked_ips: Arc, } #[derive(Debug)] diff --git a/crates/utils/src/listener/listen.rs b/crates/utils/src/listener/listen.rs index b140ae6a..bd3608d1 100644 --- a/crates/utils/src/listener/listen.rs +++ b/crates/utils/src/listener/listen.rs @@ -63,7 +63,6 @@ impl Server { hostname: self.hostname, acceptor: self.acceptor, proxy_networks: self.proxy_networks, - blocked_ips: self.blocked_ips, limiter: ConcurrencyLimiter::new(self.max_connections), shutdown_rx, }); @@ -116,7 +115,7 @@ impl Server { .proxied_address() .map(|addr| addr.source) .unwrap_or(remote_addr); - if let Some(session) = instance.build_session(stream, local_ip, remote_addr) { + if let Some(session) = instance.build_session(stream, local_ip, remote_addr, &manager) { // Spawn session manager.spawn(session, is_tls); } @@ -131,7 +130,7 @@ impl Server { } } }); - } else if let Some(session) = instance.build_session(stream, local_ip, remote_addr) { + } else if let Some(session) = instance.build_session(stream, local_ip, remote_addr, &manager) { // Set socket options opts.apply(&session.stream); @@ -165,20 +164,22 @@ impl Server { } trait BuildSession { - fn build_session( + fn build_session( &self, stream: T, local_ip: IpAddr, remote_addr: SocketAddr, + manager: &M, ) -> Option>; } impl BuildSession for Arc { - fn build_session( + fn build_session( &self, stream: T, local_ip: IpAddr, remote_addr: SocketAddr, + manager: &M, ) -> Option> { // Convert mapped IPv6 addresses to IPv4 let remote_ip = match remote_addr.ip() { @@ -191,7 +192,7 @@ impl BuildSession for Arc { let remote_port = remote_addr.port(); // Check if blocked - if self.blocked_ips.is_blocked(&remote_ip) { + if manager.is_ip_blocked(&remote_ip) { tracing::debug!( context = "listener", event = "blocked", diff --git a/crates/utils/src/listener/mod.rs b/crates/utils/src/listener/mod.rs index d8e52ac2..ad9f41cd 100644 --- a/crates/utils/src/listener/mod.rs +++ b/crates/utils/src/listener/mod.rs @@ -35,12 +35,8 @@ use tokio::{ }; use tokio_rustls::{Accept, TlsAcceptor}; -use self::{ - blocked::BlockedIps, - limiter::{ConcurrencyLimiter, InFlight}, -}; +use self::limiter::{ConcurrencyLimiter, InFlight}; -pub mod blocked; pub mod limiter; pub mod listen; pub mod stream; @@ -55,7 +51,6 @@ pub struct ServerInstance { pub acceptor: TcpAcceptor, pub limiter: ConcurrencyLimiter, pub proxy_networks: Vec, - pub blocked_ips: Arc, pub shutdown_rx: watch::Receiver, } @@ -144,6 +139,7 @@ pub trait SessionManager: Sync + Send + 'static + Clone { self, session: SessionData, ) -> impl std::future::Future + Send; + fn is_ip_blocked(&self, addr: &IpAddr) -> bool; fn shutdown(&self) -> impl std::future::Future + Send; } diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 3f554ca4..03d4e31b 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -6,7 +6,7 @@ resolver = "2" [features] #default = ["sqlite", "foundationdb", "postgres", "mysql", "rocks", "elastic", "s3", "redis"] -default = ["sqlite", "postgres", "mysql", "redis"] +default = ["sqlite", "postgres", "mysql", "rocks", "elastic", "s3", "redis"] sqlite = ["store/sqlite"] foundationdb = ["store/foundation"] postgres = ["store/postgres"] diff --git a/tests/src/directory/mod.rs b/tests/src/directory/mod.rs index a69dfeb0..57345712 100644 --- a/tests/src/directory/mod.rs +++ b/tests/src/directory/mod.rs @@ -38,7 +38,6 @@ use rustls_pki_types::PrivateKeyDer; use std::{borrow::Cow, io::BufReader, path::PathBuf, sync::Arc}; use store::{config::ConfigStore, LookupStore, Store, Stores}; use tokio_rustls::TlsAcceptor; -use utils::config::Servers; use crate::store::TempDir; @@ -95,6 +94,9 @@ verify = "SELECT address FROM emails WHERE address LIKE '%' || ? || '%' AND type expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = ? AND l.type = 'list' ORDER BY p.address LIMIT 50" domains = "SELECT 1 FROM emails WHERE address LIKE '%@' || ? LIMIT 1" +[storage] +lookup = "sqlite" + ############################################################################## [directory."postgresql"] @@ -315,7 +317,6 @@ impl DirectoryTest { directories: config .parse_directory( &stores, - &Servers::default(), id_store .map(|id| stores.stores.get(id).unwrap().clone()) .unwrap_or_default(), diff --git a/tests/src/imap/mod.rs b/tests/src/imap/mod.rs index c5a1a54c..9389798a 100644 --- a/tests/src/imap/mod.rs +++ b/tests/src/imap/mod.rs @@ -178,6 +178,7 @@ uidplus = true data = "{STORE}" fts = "{STORE}" blob = "{STORE}" +lookup = "{STORE}" directory = "auth" [jmap.protocol] @@ -277,11 +278,7 @@ async fn init_imap_tests(store_id: &str, delete_if_exists: bool) -> IMAPTest { let mut servers = config.parse_servers().unwrap(); let stores = config.parse_stores().await.failed("Invalid configuration"); let directory = config - .parse_directory( - &stores, - &servers, - stores.stores.get(store_id).unwrap().clone(), - ) + .parse_directory(&stores, stores.stores.get(store_id).unwrap().clone()) .await .unwrap(); diff --git a/tests/src/jmap/auth_limits.rs b/tests/src/jmap/auth_limits.rs index e61d6b05..90274b48 100644 --- a/tests/src/jmap/auth_limits.rs +++ b/tests/src/jmap/auth_limits.rs @@ -32,7 +32,7 @@ use jmap_client::{ mailbox::{self}, }; use jmap_proto::types::id::Id; -use utils::listener::blocked::BLOCKED_IP_KEY; +use store::{dispatch::blocked::BLOCKED_IP_KEY, write::now}; use crate::{ imap::{ImapConnection, Type}, @@ -64,8 +64,14 @@ pub async fn test(params: &mut JMAPTest) { .await; // Reset rate limiters - server.rate_limit_auth.clear(); - server.rate_limit_unauth.clear(); + server.concurrency_limiter.clear(); + + // Wait until the beginning of the 5 seconds bucket + const LIMIT: u64 = 5; + let now = now(); + let range_start = now / LIMIT; + let range_end = (range_start * LIMIT) + LIMIT; + tokio::time::sleep(Duration::from_secs(range_end - now)).await; // Incorrect passwords should be rejected with a 401 error assert!(matches!( diff --git a/tests/src/jmap/email_submission.rs b/tests/src/jmap/email_submission.rs index 380d0265..3ae38fc8 100644 --- a/tests/src/jmap/email_submission.rs +++ b/tests/src/jmap/email_submission.rs @@ -501,6 +501,9 @@ pub async fn test(params: &mut JMAPTest) { .unwrap() .take_ids() { + let _ = client + .email_submission_change_status(&id, UndoStatus::Canceled) + .await; client.email_submission_destroy(&id).await.unwrap(); } destroy_all_mailboxes(params).await; diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index ffe24fe4..8e89d5b8 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -185,6 +185,7 @@ private-key = "file://{PK}" data = "{STORE}" fts = "{STORE}" blob = "{STORE}" +lookup = "{STORE}" directory = "auth" [storage.spam] @@ -402,14 +403,9 @@ async fn init_jmap_tests(store_id: &str, delete_if_exists: bool) -> JMAPTest { let mut servers = config.parse_servers().unwrap(); let stores = config.parse_stores().await.failed("Invalid configuration"); let directory = config - .parse_directory( - &stores, - &servers, - stores.stores.get(store_id).unwrap().clone(), - ) + .parse_directory(&stores, stores.stores.get(store_id).unwrap().clone()) .await .unwrap(); - servers.blocked_ips.reload(&config).unwrap(); // Start JMAP and SMTP servers servers.bind(&config); @@ -430,6 +426,7 @@ async fn init_jmap_tests(store_id: &str, delete_if_exists: bool) -> JMAPTest { let imap: Arc = IMAP::init(&config) .await .failed("Invalid configuration file"); + jmap.directory.blocked_ips.reload(&config).unwrap(); let (shutdown_tx, _) = servers.spawn(|server, shutdown_rx| { match &server.protocol { diff --git a/tests/src/jmap/push_subscription.rs b/tests/src/jmap/push_subscription.rs index 8979fc97..99b1b436 100644 --- a/tests/src/jmap/push_subscription.rs +++ b/tests/src/jmap/push_subscription.rs @@ -356,6 +356,10 @@ impl utils::listener::SessionManager for SessionManager { fn shutdown(&self) -> impl std::future::Future + Send { async {} } + + fn is_ip_blocked(&self, _: &std::net::IpAddr) -> bool { + false + } } async fn expect_push(event_rx: &mut mpsc::Receiver) -> PushMessage { diff --git a/tests/src/jmap/quota.rs b/tests/src/jmap/quota.rs index 7bacc321..ade4c2f1 100644 --- a/tests/src/jmap/quota.rs +++ b/tests/src/jmap/quota.rs @@ -341,6 +341,15 @@ pub async fn test(params: &mut JMAPTest) { params.client.set_default_account_id(account_id.to_string()); destroy_all_mailboxes(params).await; } + for event in server.smtp.next_event().await { + server + .smtp + .read_message(event.queue_id) + .await + .unwrap() + .remove(&server.smtp, event.due) + .await; + } assert_is_empty(server).await; } diff --git a/tests/src/smtp/config.rs b/tests/src/smtp/config.rs index 75a54134..2082763f 100644 --- a/tests/src/smtp/config.rs +++ b/tests/src/smtp/config.rs @@ -21,7 +21,7 @@ * for more details. */ -use std::{fs, net::IpAddr, path::PathBuf, sync::Arc, time::Duration}; +use std::{fs, net::IpAddr, path::PathBuf, time::Duration}; use store::config::ConfigStore; use tokio::net::TcpSocket; @@ -364,7 +364,6 @@ fn parse_servers() { tls_implicit: false, max_connections: 8192, proxy_networks: vec![], - blocked_ips: Arc::new(Default::default()), }, Server { id: "smtps".to_string(), @@ -394,7 +393,6 @@ fn parse_servers() { tls_implicit: true, max_connections: 1024, proxy_networks: vec![], - blocked_ips: Arc::new(Default::default()), }, Server { id: "submission".to_string(), @@ -414,7 +412,6 @@ fn parse_servers() { tls_implicit: true, max_connections: 8192, proxy_networks: vec![], - blocked_ips: Arc::new(Default::default()), }, ]; diff --git a/tests/src/smtp/inbound/auth.rs b/tests/src/smtp/inbound/auth.rs index d4cde99e..c2b89076 100644 --- a/tests/src/smtp/inbound/auth.rs +++ b/tests/src/smtp/inbound/auth.rs @@ -22,10 +22,11 @@ */ use directory::core::config::ConfigDirectory; -use store::{Store, Stores}; -use utils::config::{if_block::IfBlock, Config, Servers}; +use store::Store; +use utils::config::{if_block::IfBlock, Config}; use crate::smtp::{ + inbound::dummy_stores, session::{TestSession, VerifyResponse}, ParseTestConfig, TestConfig, }; @@ -35,6 +36,9 @@ use smtp::{ }; const DIRECTORY: &str = r#" +[storage] +lookup = "dummy" + [directory."local"] type = "memory" @@ -60,7 +64,7 @@ async fn auth() { let mut core = SMTP::test(); core.shared.directories = Config::new(DIRECTORY) .unwrap() - .parse_directory(&Stores::default(), &Servers::default(), Store::default()) + .parse_directory(&dummy_stores(), Store::default()) .await .unwrap() .directories; diff --git a/tests/src/smtp/inbound/data.rs b/tests/src/smtp/inbound/data.rs index 05f2262d..3c3d26d3 100644 --- a/tests/src/smtp/inbound/data.rs +++ b/tests/src/smtp/inbound/data.rs @@ -21,18 +21,23 @@ * for more details. */ +use std::sync::Arc; + use directory::core::config::ConfigDirectory; -use store::{Store, Stores}; -use utils::config::{if_block::IfBlock, Config, Servers}; +use store::Store; +use utils::config::{if_block::IfBlock, Config}; use crate::smtp::{ - inbound::{TestMessage, TestQueueEvent}, + inbound::{dummy_stores, TestMessage}, session::{load_test_message, TestSession, VerifyResponse}, ParseTestConfig, TestConfig, TestSMTP, }; use smtp::core::{Session, SMTP}; const DIRECTORY: &str = r#" +[storage] +lookup = "dummy" + [directory."local"] type = "memory" @@ -77,7 +82,7 @@ async fn data() { let mut qr = core.init_test_queue("smtp_data_test"); core.shared.directories = Config::new(DIRECTORY) .unwrap() - .parse_directory(&Stores::default(), &Servers::default(), Store::default()) + .parse_directory(&dummy_stores(), Store::default()) .await .unwrap() .directories; @@ -116,7 +121,8 @@ async fn data() { .parse_quota(); // Test queue message builder - let mut session = Session::test(core); + let core = Arc::new(core); + let mut session = Session::test(core.clone()); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.test_builder().await; @@ -148,11 +154,10 @@ async fn data() { // No headers should be added to messages from 10.0.0.1 session - .send_message("john@doe.org", &["bill@foobar.org"], "test:no_msgid", "250") + .send_message("john@test.org", &["mike@test.com"], "test:no_msgid", "250") .await; - qr.read_event().await.assert_reload(); assert_eq!( - qr.last_queued_message().await.read_message(&core).await, + qr.expect_message().await.read_message(&qr).await, load_test_message("no_msgid", "messages") ); @@ -167,12 +172,11 @@ async fn data() { session.data.remote_ip_str = "10.0.0.3".to_string(); session.eval_session_params().await; session - .send_message("john@doe.org", &["mike@test.com"], "test:no_msgid", "250") + .send_message("bill@doe.org", &["mike@test.com"], "test:no_msgid", "250") .await; - qr.read_event().await.assert_reload(); - qr.last_queued_message() + qr.expect_message() .await - .read_lines(&core) + .read_lines(&qr) .await .assert_contains("From: ") .assert_contains("To: ") @@ -185,13 +189,11 @@ async fn data() { .assert_contains("Received-SPF: "); // Only one message is allowed in the queue from john@doe.org - let mut queued_messages = vec![]; session.data.remote_ip_str = "10.0.0.2".to_string(); session.eval_session_params().await; session .send_message("john@doe.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; - queued_messages.push(qr.read_event().await); session .send_message( "john@doe.org", @@ -202,7 +204,7 @@ async fn data() { .await; // Release quota - queued_messages.clear(); + qr.clear_queue(&core).await; // Only 1500 bytes are allowed in the queue to domain foobar.org session @@ -213,7 +215,6 @@ async fn data() { "250", ) .await; - queued_messages.push(qr.read_event().await); session .send_message( "jane@foobar.org", @@ -232,7 +233,6 @@ async fn data() { "250", ) .await; - queued_messages.push(qr.read_event().await); session .send_message( "jane@foobar.org", @@ -241,4 +241,11 @@ async fn data() { "452 4.3.1", ) .await; + + // Make sure store is empty + qr.clear_queue(&core).await; + core.shared + .default_data_store + .assert_is_empty(core.shared.default_blob_store.clone()) + .await; } diff --git a/tests/src/smtp/inbound/dmarc.rs b/tests/src/smtp/inbound/dmarc.rs index e5255744..15cf245c 100644 --- a/tests/src/smtp/inbound/dmarc.rs +++ b/tests/src/smtp/inbound/dmarc.rs @@ -34,11 +34,11 @@ use mail_auth::{ report::DmarcResult, spf::Spf, }; -use store::{Store, Stores}; -use utils::config::{if_block::IfBlock, Config, Servers}; +use store::Store; +use utils::config::{if_block::IfBlock, Config}; use crate::smtp::{ - inbound::{sign::TextConfigContext, TestMessage, TestQueueEvent, TestReportingEvent}, + inbound::{dummy_stores, sign::TextConfigContext, TestMessage, TestReportingEvent}, session::{TestSession, VerifyResponse}, ParseTestConfig, TestConfig, TestSMTP, }; @@ -48,6 +48,9 @@ use smtp::{ }; const DIRECTORY: &str = r#" +[storage] +lookup = "dummy" + [directory."local"] type = "memory" @@ -133,7 +136,7 @@ async fn dmarc() { let mut rr = core.init_test_report(); core.shared.directories = Config::new(DIRECTORY) .unwrap() - .parse_directory(&Stores::default(), &Servers::default(), Store::default()) + .parse_directory(&dummy_stores(), Store::default()) .await .unwrap() .directories; @@ -180,14 +183,13 @@ async fn dmarc() { session.mail_from("bill@example.com", "550 5.7.23").await; // Expect SPF auth failure report - qr.read_event().await.assert_reload(); - let message = qr.last_queued_message().await; + let message = qr.expect_message().await; assert_eq!( message.recipients.last().unwrap().address, "spf-failures@example.com" ); message - .read_lines(&core) + .read_lines(&qr) .await .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com;") .assert_contains("To: spf-failures@example.com") @@ -212,14 +214,13 @@ async fn dmarc() { .await; // Expect DKIM auth failure report - qr.read_event().await.assert_reload(); - let message = qr.last_queued_message().await; + let message = qr.expect_message().await; assert_eq!( message.recipients.last().unwrap().address, "dkim-failures@example.com" ); message - .read_lines(&core) + .read_lines(&qr) .await .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com;") .assert_contains("To: dkim-failures@example.com") @@ -264,14 +265,13 @@ async fn dmarc() { .await; // Expect DMARC auth failure report - qr.read_event().await.assert_reload(); - let message = qr.last_queued_message().await; + let message = qr.expect_message().await; assert_eq!( message.recipients.last().unwrap().address, "dmarc-failures@example.com" ); message - .read_lines(&core) + .read_lines(&qr) .await .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com;") .assert_contains("To: dmarc-failures@example.com") @@ -306,10 +306,9 @@ async fn dmarc() { "250", ) .await; - qr.read_event().await.assert_reload(); - qr.last_queued_message() + qr.expect_message() .await - .read_lines(&core) + .read_lines(&qr) .await .assert_contains("dkim=pass") .assert_contains("spf=pass") diff --git a/tests/src/smtp/inbound/milter.rs b/tests/src/smtp/inbound/milter.rs index aca8ee33..3f3dab82 100644 --- a/tests/src/smtp/inbound/milter.rs +++ b/tests/src/smtp/inbound/milter.rs @@ -42,7 +42,7 @@ use tokio::{ use utils::config::if_block::IfBlock; use crate::smtp::{ - inbound::{TestMessage, TestQueueEvent}, + inbound::TestMessage, session::{load_test_message, TestSession, VerifyResponse}, ParseTestConfig, TestConfig, TestSMTP, }; @@ -152,10 +152,9 @@ async fn milter_session() { "250 2.0.0", ) .await; - qr.read_event().await.assert_reload(); - qr.last_queued_message() + qr.expect_message() .await - .read_lines(&core) + .read_lines(&qr) .await .assert_contains("X-Hello: World") .assert_contains("Subject: Is dinner ready?") @@ -170,10 +169,9 @@ async fn milter_session() { "250 2.0.0", ) .await; - qr.read_event().await.assert_reload(); - qr.last_queued_message() + qr.expect_message() .await - .read_lines(&core) + .read_lines(&qr) .await .assert_contains("Subject: [SPAM] Saying Hello") .assert_count("References: ", 1) @@ -188,10 +186,9 @@ async fn milter_session() { "250 2.0.0", ) .await; - qr.read_event().await.assert_reload(); - qr.last_queued_message() + qr.expect_message() .await - .read_lines(&core) + .read_lines(&qr) .await .assert_contains("X-Spam: Yes") .assert_contains("123456"); diff --git a/tests/src/smtp/inbound/mod.rs b/tests/src/smtp/inbound/mod.rs index af037346..f073c299 100644 --- a/tests/src/smtp/inbound/mod.rs +++ b/tests/src/smtp/inbound/mod.rs @@ -24,14 +24,14 @@ use std::time::Duration; use store::{ - write::{key::DeserializeBigEndian, Bincode, QueueClass, QueueEvent, ValueClass}, - Deserialize, IterateParams, ValueKey, U64_LEN, + write::{key::DeserializeBigEndian, Bincode, QueueClass, QueueEvent, ReportEvent, ValueClass}, + Deserialize, IterateParams, Store, Stores, ValueKey, U64_LEN, }; use tokio::sync::mpsc::error::TryRecvError; use smtp::{ core::SMTP, - queue::{self, DeliveryAttempt, Message, OnHold, QueueId}, + queue::{self, spool::QueueEventLock, DeliveryAttempt, Message, OnHold, QueueId}, reporting::{self, DmarcEvent, TlsEvent}, }; @@ -78,24 +78,87 @@ impl QueueReceiver { } } - pub async fn assert_queue_is_empty(&mut self) { + pub async fn assert_queue_is_empty(&self) { assert_eq!(self.read_queued_messages().await, vec![]); assert_eq!(self.read_queued_events().await, vec![]); } + pub async fn assert_report_is_empty(&self) { + assert_eq!(self.read_report_events().await, vec![]); + + for (from_key, to_key) in [ + ( + ValueKey::from(ValueClass::Queue(QueueClass::TlsReportEvent(ReportEvent { + due: 0, + policy_hash: 0, + seq_id: 0, + domain: String::new(), + }))), + ValueKey::from(ValueClass::Queue(QueueClass::TlsReportEvent(ReportEvent { + due: u64::MAX, + policy_hash: 0, + seq_id: 0, + domain: String::new(), + }))), + ), + ( + ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportEvent( + ReportEvent { + due: 0, + policy_hash: 0, + seq_id: 0, + domain: String::new(), + }, + ))), + ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportEvent( + ReportEvent { + due: u64::MAX, + policy_hash: 0, + seq_id: 0, + domain: String::new(), + }, + ))), + ), + ] { + self.store + .iterate( + IterateParams::new(from_key, to_key).ascending().no_values(), + |key, _| { + panic!("Unexpected report event: {key:?}"); + }, + ) + .await + .unwrap(); + } + } + pub async fn expect_message(&mut self) -> Message { self.read_event().await.assert_reload(); self.last_queued_message().await } + pub async fn consume_message(&mut self, core: &SMTP) -> Message { + self.read_event().await.assert_reload(); + let message = self.last_queued_message().await; + message + .clone() + .remove(core, self.last_queued_due().await) + .await; + message + } + pub async fn expect_message_then_deliver(&mut self) -> DeliveryAttempt { let message = self.expect_message().await; - let event = QueueEvent { - due: self.message_due(message.id).await, - queue_id: message.id, - }; - DeliveryAttempt::new(message, event) + self.delivery_attempt(message.id).await + } + + pub async fn delivery_attempt(&mut self, queue_id: u64) -> DeliveryAttempt { + DeliveryAttempt::new(QueueEventLock { + due: self.message_due(queue_id).await, + queue_id, + lock_expiry: 0, + }) } pub async fn read_queued_events(&self) -> Vec { @@ -134,7 +197,7 @@ impl QueueReceiver { self.store .iterate( - IterateParams::new(from_key, to_key).ascending(), + IterateParams::new(from_key, to_key).descending(), |key, value| { let value = Bincode::::deserialize(value)?; assert_eq!(key.deserialize_be_u64(1)?, value.inner.id); @@ -148,6 +211,46 @@ impl QueueReceiver { messages } + pub async fn read_report_events(&self) -> Vec { + let from_key = ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportHeader( + ReportEvent { + due: 0, + policy_hash: 0, + seq_id: 0, + domain: String::new(), + }, + ))); + let to_key = ValueKey::from(ValueClass::Queue(QueueClass::TlsReportHeader( + ReportEvent { + due: u64::MAX, + policy_hash: 0, + seq_id: 0, + domain: String::new(), + }, + ))); + + let mut events = Vec::new(); + self.store + .iterate( + IterateParams::new(from_key, to_key).ascending().no_values(), + |key, _| { + let event = ReportEvent::deserialize(key)?; + // Skip lock + if event.seq_id != 0 { + events.push(if *key.last().unwrap() == 0 { + QueueClass::DmarcReportHeader(event) + } else { + QueueClass::TlsReportHeader(event) + }); + } + Ok(true) + }, + ) + .await + .unwrap(); + events + } + pub async fn last_queued_message(&self) -> Message { self.read_queued_messages() .await @@ -156,6 +259,10 @@ impl QueueReceiver { .expect("No messages found in queue") } + pub async fn last_queued_due(&self) -> u64 { + self.message_due(self.last_queued_message().await.id).await + } + pub async fn message_due(&self, queue_id: QueueId) -> u64 { self.read_queued_events() .await @@ -169,6 +276,13 @@ impl QueueReceiver { }) .expect("No event found in queue for message") } + + pub async fn clear_queue(&self, core: &SMTP) { + for message in self.read_queued_messages().await { + let due = self.message_due(message.id).await; + message.remove(core, due).await; + } + } } impl ReportReceiver { @@ -198,7 +312,7 @@ impl ReportReceiver { pub trait TestQueueEvent { fn assert_reload(self); - fn unwrap_on_hold(self) -> OnHold; + fn unwrap_on_hold(self) -> OnHold; } impl TestQueueEvent for queue::Event { @@ -209,7 +323,7 @@ impl TestQueueEvent for queue::Event { } } - fn unwrap_on_hold(self) -> OnHold { + fn unwrap_on_hold(self) -> OnHold { match self { queue::Event::OnHold(value) => value, e => panic!("Unexpected event: {e:?}"), @@ -238,16 +352,16 @@ impl TestReportingEvent for reporting::Event { } } +#[allow(async_fn_in_trait)] pub trait TestMessage { - async fn read_message(&self, core: &SMTP) -> String; - async fn read_lines(&self, core: &SMTP) -> Vec; + async fn read_message(&self, core: &QueueReceiver) -> String; + async fn read_lines(&self, core: &QueueReceiver) -> Vec; } impl TestMessage for Message { - async fn read_message(&self, core: &SMTP) -> String { + async fn read_message(&self, core: &QueueReceiver) -> String { String::from_utf8( - core.shared - .default_blob_store + core.blob_store .get_blob(self.blob_hash.as_slice(), 0..u32::MAX) .await .unwrap() @@ -256,7 +370,7 @@ impl TestMessage for Message { .unwrap() } - async fn read_lines(&self, core: &SMTP) -> Vec { + async fn read_lines(&self, core: &QueueReceiver) -> Vec { self.read_message(core) .await .split('\n') @@ -264,3 +378,17 @@ impl TestMessage for Message { .collect() } } + +pub fn dummy_stores() -> Stores { + let mut stores = Stores::default(); + let store = Store::default(); + stores.stores.insert("dummy".to_string(), store.clone()); + stores + .lookup_stores + .insert("dummy".to_string(), store.clone().into()); + stores + .fts_stores + .insert("dummy".to_string(), store.clone().into()); + stores.blob_stores.insert("dummy".to_string(), store.into()); + stores +} diff --git a/tests/src/smtp/inbound/rcpt.rs b/tests/src/smtp/inbound/rcpt.rs index c33b6c49..ab0f940b 100644 --- a/tests/src/smtp/inbound/rcpt.rs +++ b/tests/src/smtp/inbound/rcpt.rs @@ -25,16 +25,20 @@ use std::time::Duration; use directory::core::config::ConfigDirectory; use smtp_proto::{RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_SUCCESS}; -use store::{Store, Stores}; -use utils::config::{if_block::IfBlock, Config, Servers}; +use store::Store; +use utils::config::{if_block::IfBlock, Config}; use crate::smtp::{ + inbound::dummy_stores, session::{TestSession, VerifyResponse}, ParseTestConfig, TestConfig, }; use smtp::core::{Session, State, SMTP}; const DIRECTORY: &str = r#" +[storage] +lookup = "dummy" + [directory."local"] type = "memory" @@ -71,7 +75,7 @@ async fn rcpt() { let config_ext = &mut core.session.config.extensions; core.shared.directories = Config::new(DIRECTORY) .unwrap() - .parse_directory(&Stores::default(), &Servers::default(), Store::default()) + .parse_directory(&dummy_stores(), Store::default()) .await .unwrap() .directories; diff --git a/tests/src/smtp/inbound/rewrite.rs b/tests/src/smtp/inbound/rewrite.rs index 9d6bcd32..fc726211 100644 --- a/tests/src/smtp/inbound/rewrite.rs +++ b/tests/src/smtp/inbound/rewrite.rs @@ -21,16 +21,23 @@ * for more details. */ -use crate::smtp::{inbound::sign::TextConfigContext, session::TestSession, TestConfig}; +use crate::smtp::{ + inbound::{dummy_stores, sign::TextConfigContext}, + session::TestSession, + TestConfig, +}; use directory::core::config::ConfigDirectory; use smtp::{ config::{map_expr_token, scripts::ConfigSieve, ConfigContext}, core::{eval::*, Session, SMTP}, }; -use store::{Store, Stores}; -use utils::config::{if_block::IfBlock, utils::NoConstants, Config, Servers}; +use store::Store; +use utils::config::{if_block::IfBlock, utils::NoConstants, Config}; const CONFIG: &str = r#" +[storage] +lookup = "dummy" + [session.mail] rewrite = [ { if = "ends_with(sender_domain, '.foobar.net') & matches('^([^.]+)@([^.]+)\.(.+)$', sender)", then = "$1 + '+' + $2 + '@' + $3"}, { else = false } ] @@ -96,7 +103,7 @@ async fn address_rewrite() { let mut ctx = ConfigContext::new(&[]).parse_signatures(); let settings = Config::new(CONFIG).unwrap(); ctx.directory = settings - .parse_directory(&Stores::default(), &Servers::default(), Store::default()) + .parse_directory(&dummy_stores(), Store::default()) .await .unwrap(); core.sieve = settings.parse_sieve(&mut ctx).unwrap(); diff --git a/tests/src/smtp/inbound/scripts.rs b/tests/src/smtp/inbound/scripts.rs index ba6fe104..b04a6a1d 100644 --- a/tests/src/smtp/inbound/scripts.rs +++ b/tests/src/smtp/inbound/scripts.rs @@ -37,9 +37,12 @@ use smtp::{ }; use store::{config::ConfigStore, Store}; use tokio::runtime::Handle; -use utils::config::{if_block::IfBlock, Config, Servers}; +use utils::config::{if_block::IfBlock, Config}; const CONFIG: &str = r#" +[storage] +lookup = "sql" + [store."sql"] type = "sqlite" path = "%PATH%/smtp_sieve.db" @@ -135,7 +138,7 @@ async fn sieve_scripts() { ctx.stores = config.parse_stores().await.unwrap(); core.shared.lookup_stores = ctx.stores.lookup_stores.clone(); core.shared.directories = config - .parse_directory(&ctx.stores, &Servers::default(), Store::default()) + .parse_directory(&ctx.stores, Store::default()) .await .unwrap() .directories; @@ -228,10 +231,10 @@ async fn sieve_scripts() { // Expect a modified message session.data("test:multipart", "250").await; - qr.read_event().await.assert_reload(); - qr.last_queued_message() + + qr.expect_message() .await - .read_lines(&core) + .read_lines(&qr) .await .assert_contains("X-Part-Number: 5") .assert_contains("THIS IS A PIECE OF HTML TEXT"); @@ -247,6 +250,7 @@ async fn sieve_scripts() { ) .await; qr.assert_no_events(); + qr.clear_queue(&core).await; // Expect message delivery plus a notification session @@ -258,7 +262,11 @@ async fn sieve_scripts() { ) .await; qr.read_event().await.assert_reload(); - let notification = qr.last_queued_message().await; + qr.read_event().await.assert_reload(); + let messages = qr.read_queued_messages().await; + assert_eq!(messages.len(), 2); + let mut messages = messages.into_iter().rev(); + let notification = messages.next().unwrap(); assert_eq!(notification.return_path, ""); assert_eq!(notification.recipients.len(), 2); assert_eq!( @@ -270,7 +278,7 @@ async fn sieve_scripts() { "jane@example.org" ); notification - .read_lines(&core) + .read_lines(&qr) .await .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com;") .assert_contains("From: \"Sieve Daemon\" ") @@ -278,16 +286,18 @@ async fn sieve_scripts() { .assert_contains("Cc: ") .assert_contains("Subject: You have got mail") .assert_contains("One Two Three Four"); - qr.read_event().await.assert_reload(); - qr.last_queued_message() - .await - .read_lines(&core) + + messages + .next() + .unwrap() + .read_lines(&qr) .await .assert_contains("One Two Three Four") .assert_contains("multi-part message in MIME format") .assert_not_contains("X-Part-Number: 5") .assert_not_contains("THIS IS A PIECE OF HTML TEXT"); qr.assert_no_events(); + qr.clear_queue(&core).await; // Expect a modified message delivery plus a notification session @@ -298,11 +308,16 @@ async fn sieve_scripts() { "250", ) .await; - qr.read_event().await.assert_reload(); - qr.last_queued_message() - .await - .read_lines(&core) + qr.read_event().await.assert_reload(); + let messages = qr.read_queued_messages().await; + assert_eq!(messages.len(), 2); + let mut messages = messages.into_iter().rev(); + + messages + .next() + .unwrap() + .read_lines(&qr) .await .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com;") .assert_contains("From: \"Sieve Daemon\" ") @@ -311,14 +326,15 @@ async fn sieve_scripts() { .assert_contains("Subject: You have got mail") .assert_contains("One Two Three Four"); - qr.read_event().await.assert_reload(); - qr.last_queued_message() - .await - .read_lines(&core) + messages + .next() + .unwrap() + .read_lines(&qr) .await .assert_contains("X-Part-Number: 5") .assert_contains("THIS IS A PIECE OF HTML TEXT") .assert_not_contains("X-My-Header: true"); + qr.clear_queue(&core).await; // Expect a modified redirected message session @@ -329,8 +345,8 @@ async fn sieve_scripts() { "250", ) .await; - qr.read_event().await.assert_reload(); - let redirect = qr.last_queued_message().await; + + let redirect = qr.expect_message().await; assert_eq!(redirect.return_path, ""); assert_eq!(redirect.recipients.len(), 1); assert_eq!( @@ -338,7 +354,7 @@ async fn sieve_scripts() { "redirect@here.email" ); redirect - .read_lines(&core) + .read_lines(&qr) .await .assert_contains("From: no-reply@my.domain") .assert_contains("To: Suzie Q ") @@ -356,8 +372,8 @@ async fn sieve_scripts() { "250", ) .await; - qr.read_event().await.assert_reload(); - let redirect = qr.last_queued_message().await; + + let redirect = qr.expect_message().await; assert_eq!(redirect.return_path, ""); assert_eq!(redirect.recipients.len(), 1); assert_eq!( @@ -365,7 +381,7 @@ async fn sieve_scripts() { "redirect@somewhere.email" ); redirect - .read_lines(&core) + .read_lines(&qr) .await .assert_not_contains("From: no-reply@my.domain") .assert_contains("To: Suzie Q ") @@ -385,10 +401,10 @@ async fn sieve_scripts() { "250", ) .await; - qr.read_event().await.assert_reload(); - qr.last_queued_message() + + qr.expect_message() .await - .read_lines(&core) + .read_lines(&qr) .await .assert_contains("X-My-Header: true") .assert_contains("Authentication-Results"); diff --git a/tests/src/smtp/inbound/sign.rs b/tests/src/smtp/inbound/sign.rs index c8e4323d..53e252b1 100644 --- a/tests/src/smtp/inbound/sign.rs +++ b/tests/src/smtp/inbound/sign.rs @@ -28,11 +28,11 @@ use mail_auth::{ common::{parse::TxtRecordParser, verify::DomainKey}, spf::Spf, }; -use store::{Store, Stores}; -use utils::config::{if_block::IfBlock, Config, Servers}; +use store::Store; +use utils::config::{if_block::IfBlock, Config}; use crate::smtp::{ - inbound::{TestMessage, TestQueueEvent}, + inbound::{dummy_stores, TestMessage}, session::{TestSession, VerifyResponse}, ParseTestConfig, TestConfig, TestSMTP, }; @@ -94,6 +94,9 @@ set-body-length = false "; const DIRECTORY: &str = r#" +[storage] +lookup = "dummy" + [directory."local"] type = "memory" @@ -152,7 +155,7 @@ async fn sign_and_seal() { core.shared.directories = Config::new(DIRECTORY) .unwrap() - .parse_directory(&Stores::default(), &Servers::default(), Store::default()) + .parse_directory(&dummy_stores(), Store::default()) .await .unwrap() .directories; @@ -192,10 +195,9 @@ async fn sign_and_seal() { "250", ) .await; - qr.read_event().await.assert_reload(); - qr.last_queued_message() + qr.expect_message() .await - .read_lines(&core) + .read_lines(&qr) .await .assert_contains( "DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com; c=simple/relaxed;", @@ -205,10 +207,9 @@ async fn sign_and_seal() { session .send_message("bill@foobar.org", &["jdoe@example.com"], "test:arc", "250") .await; - qr.read_event().await.assert_reload(); - qr.last_queued_message() + qr.expect_message() .await - .read_lines(&core) + .read_lines(&qr) .await .assert_contains("ARC-Seal: i=3; a=ed25519-sha256; s=ed; d=example.com; cv=pass;") .assert_contains( diff --git a/tests/src/smtp/inbound/throttle.rs b/tests/src/smtp/inbound/throttle.rs index de77c584..472fb4d6 100644 --- a/tests/src/smtp/inbound/throttle.rs +++ b/tests/src/smtp/inbound/throttle.rs @@ -23,12 +23,22 @@ use std::time::Duration; -use crate::smtp::{session::TestSession, ParseTestConfig, TestConfig}; +use crate::smtp::{session::TestSession, ParseTestConfig, TestConfig, TestSMTP}; use smtp::core::{Session, SessionAddress, SMTP}; #[tokio::test] async fn throttle_inbound() { + // Enable logging + /*let disable = "true"; + tracing::subscriber::set_global_default( + tracing_subscriber::FmtSubscriber::builder() + .with_max_level(tracing::Level::TRACE) + .finish(), + ) + .unwrap();*/ + let mut core = SMTP::test(); + let _qr = core.init_test_queue("smtp_inbound_throttle"); let config = &mut core.session.config; config.throttle.connect = r#"[[throttle]] match = "remote_ip = '10.0.0.1'" diff --git a/tests/src/smtp/inbound/vrfy.rs b/tests/src/smtp/inbound/vrfy.rs index 4ec8c394..4147af95 100644 --- a/tests/src/smtp/inbound/vrfy.rs +++ b/tests/src/smtp/inbound/vrfy.rs @@ -22,16 +22,20 @@ */ use directory::core::config::ConfigDirectory; -use store::{Store, Stores}; -use utils::config::{if_block::IfBlock, Config, Servers}; +use store::Store; +use utils::config::{if_block::IfBlock, Config}; use crate::smtp::{ + inbound::dummy_stores, session::{TestSession, VerifyResponse}, ParseTestConfig, TestConfig, }; use smtp::core::{Session, SMTP}; const DIRECTORY: &str = r#" +[storage] +lookup = "dummy" + [directory."local"] type = "memory" @@ -64,7 +68,7 @@ async fn vrfy_expn() { core.shared.directories = Config::new(DIRECTORY) .unwrap() - .parse_directory(&Stores::default(), &Servers::default(), Store::default()) + .parse_directory(&dummy_stores(), Store::default()) .await .unwrap() .directories; diff --git a/tests/src/smtp/lookup/sql.rs b/tests/src/smtp/lookup/sql.rs index 105b5150..7b327836 100644 --- a/tests/src/smtp/lookup/sql.rs +++ b/tests/src/smtp/lookup/sql.rs @@ -26,7 +26,7 @@ use std::time::Duration; use directory::core::config::ConfigDirectory; use smtp_proto::{AUTH_LOGIN, AUTH_PLAIN}; use store::{config::ConfigStore, Store}; -use utils::config::{if_block::IfBlock, Config, Servers}; +use utils::config::{if_block::IfBlock, Config}; use crate::{ directory::DirectoryStore, @@ -42,6 +42,9 @@ use smtp::{ }; const CONFIG: &str = r#" +[storage] +lookup = "sql" + [store."sql"] type = "sqlite" path = "{TMP}/smtp_sql.db" @@ -89,7 +92,7 @@ async fn lookup_sql() { ctx.stores = config.parse_stores().await.unwrap(); core.shared.lookup_stores = ctx.stores.lookup_stores.clone(); core.shared.directories = config - .parse_directory(&ctx.stores, &Servers::default(), Store::default()) + .parse_directory(&ctx.stores, Store::default()) .await .unwrap() .directories; diff --git a/tests/src/smtp/management/queue.rs b/tests/src/smtp/management/queue.rs index 8c46ff6c..405c85b0 100644 --- a/tests/src/smtp/management/queue.rs +++ b/tests/src/smtp/management/queue.rs @@ -31,11 +31,11 @@ use directory::core::config::ConfigDirectory; use mail_auth::MX; use mail_parser::DateTime; use reqwest::{header::AUTHORIZATION, StatusCode}; -use store::{Store, Stores}; -use utils::config::{if_block::IfBlock, Config, ServerProtocol, Servers}; +use store::Store; +use utils::config::{if_block::IfBlock, Config, ServerProtocol}; use crate::smtp::{ - inbound::TestQueueEvent, management::send_manage_request, outbound::start_test_server, + inbound::dummy_stores, management::send_manage_request, outbound::start_test_server, session::TestSession, TestConfig, TestSMTP, }; use smtp::{ @@ -44,6 +44,9 @@ use smtp::{ }; const DIRECTORY: &str = r#" +[storage] +lookup = "dummy" + [directory."local"] type = "memory" @@ -70,7 +73,8 @@ async fn manage_queue() { let mut core = SMTP::test(); core.session.config.rcpt.relay = IfBlock::new(true); let mut remote_qr = core.init_test_queue("smtp_manage_queue_remote"); - let _rx_remote = start_test_server(core.into(), &[ServerProtocol::Smtp]); + let remote_core = Arc::new(core); + let _rx_remote = start_test_server(remote_core.clone(), &[ServerProtocol::Smtp]); // Add mock DNS entries let mut core = SMTP::test(); @@ -92,7 +96,7 @@ async fn manage_queue() { // Start local management interface let directory = Config::new(DIRECTORY) .unwrap() - .parse_directory(&Stores::default(), &Servers::default(), Store::default()) + .parse_directory(&dummy_stores(), Store::default()) .await .unwrap(); core.shared.default_directory = directory.directories.get("local").unwrap().clone(); @@ -169,10 +173,9 @@ async fn manage_queue() { // Expect delivery to success@foobar.org tokio::time::sleep(Duration::from_millis(100)).await; - remote_qr.read_event().await.assert_reload(); assert_eq!( remote_qr - .last_queued_message() + .consume_message(&remote_core) .await .recipients .into_iter() @@ -216,7 +219,9 @@ async fn manage_queue() { let expires = created + 3000 + hold_for; for domain in &message.domains { if env_id == "c" { - test_search = domain.next_retry.as_ref().unwrap().to_rfc3339(); + let mut dt = *domain.next_retry.as_ref().unwrap(); + dt.second -= 1; + test_search = dt.to_rfc3339(); } if env_id != "f" { assert_eq!(domain.retry_num, 0); @@ -320,10 +325,9 @@ async fn manage_queue() { // Expect delivery to john@foobar.org tokio::time::sleep(Duration::from_millis(100)).await; - remote_qr.read_event().await.assert_reload(); assert_eq!( remote_qr - .last_queued_message() + .consume_message(&remote_core) .await .recipients .into_iter() diff --git a/tests/src/smtp/management/report.rs b/tests/src/smtp/management/report.rs index c6e1f2a0..43689a8f 100644 --- a/tests/src/smtp/management/report.rs +++ b/tests/src/smtp/management/report.rs @@ -34,12 +34,12 @@ use mail_auth::{ ActionDisposition, DmarcResult, Record, }, }; -use store::{Store, Stores}; +use store::Store; use tokio::sync::mpsc; -use utils::config::{if_block::IfBlock, Config, ServerProtocol, Servers}; +use utils::config::{if_block::IfBlock, Config, ServerProtocol}; use crate::smtp::{ - make_temp_dir, management::send_manage_request, outbound::start_test_server, TestConfig, + inbound::dummy_stores, management::send_manage_request, outbound::start_test_server, TestConfig, }; use smtp::{ config::AggregateFrequency, @@ -48,6 +48,9 @@ use smtp::{ }; const DIRECTORY: &str = r#" +[storage] +lookup = "dummy" + [directory."local"] type = "memory" @@ -72,13 +75,12 @@ async fn manage_reports() { // Start reporting service let mut core = SMTP::test(); - let temp_dir = make_temp_dir("smtp_report_management_test", true); let config = &mut core.report.config; config.dmarc_aggregate.max_size = IfBlock::new(1024); config.tls.max_size = IfBlock::new(1024); let directory = Config::new(DIRECTORY) .unwrap() - .parse_directory(&Stores::default(), &Servers::default(), Store::default()) + .parse_directory(&dummy_stores(), Store::default()) .await .unwrap(); core.shared.default_directory = directory.directories.get("local").unwrap().clone(); diff --git a/tests/src/smtp/mod.rs b/tests/src/smtp/mod.rs index 98363689..a88da95b 100644 --- a/tests/src/smtp/mod.rs +++ b/tests/src/smtp/mod.rs @@ -33,7 +33,9 @@ use mail_auth::{ use mail_send::smtp::tls::build_tls_connector; use sieve::Runtime; use smtp_proto::{AUTH_LOGIN, AUTH_PLAIN}; -use store::{backend::sqlite::SqliteStore, LookupStore, Store}; +use store::{ + backend::sqlite::SqliteStore, dispatch::blocked::BlockedIps, BlobStore, LookupStore, Store, +}; use tokio::sync::mpsc; use smtp::{ @@ -201,7 +203,7 @@ impl TestConfig for SMTP { catch_all: AddressMapping::Disable, subaddressing: AddressMapping::Disable, cache: None, - blocked_ips: Arc::new(Default::default()), + blocked_ips: Arc::new(BlockedIps::new(store.clone().into())), }), default_lookup_store: LookupStore::Store(store.clone()), default_blob_store: store::BlobStore::Store(store.clone()), @@ -498,6 +500,7 @@ pub fn add_test_certs(config: &str) -> String { pub struct QueueReceiver { _temp_dir: TempDir, store: Store, + blob_store: BlobStore, pub queue_rx: mpsc::Receiver, } @@ -524,10 +527,12 @@ impl TestSMTP for SMTP { let store = Store::SQLite(SqliteStore::open(&config, "store.sqlite").unwrap().into()); self.shared.default_data_store = store.clone(); self.shared.default_blob_store = store.clone().into(); + self.shared.default_lookup_store = store.clone().into(); let (queue_tx, queue_rx) = mpsc::channel(128); self.queue.tx = queue_tx; QueueReceiver { + blob_store: store.clone().into(), store, queue_rx, _temp_dir, diff --git a/tests/src/smtp/outbound/dane.rs b/tests/src/smtp/outbound/dane.rs index 4d2b6a32..e0763923 100644 --- a/tests/src/smtp/outbound/dane.rs +++ b/tests/src/smtp/outbound/dane.rs @@ -57,7 +57,7 @@ use smtp::{ config::{AggregateFrequency, RequireOptional}, core::{Resolvers, Session, SMTP}, outbound::dane::{DnssecResolver, Tlsa, TlsaEntry}, - queue::{manager::Queue, DeliveryAttempt, Error, ErrorDetails, Status}, + queue::{Error, ErrorDetails, Status}, reporting::PolicyType, }; @@ -106,7 +106,6 @@ async fn dane_verify() { core.report.config.tls.send = IfBlock::new(AggregateFrequency::Weekly); let core = Arc::new(core); - let mut queue = Queue::default(); let mut session = Session::test(core.clone()); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; @@ -120,14 +119,14 @@ async fn dane_verify() { .try_deliver(core.clone()) .await; local_qr - .last_queued_message() + .expect_message() .await - .read_lines(&core) + .read_lines(&local_qr) .await .assert_contains(" (DANE failed to authenticate") .assert_contains("No TLSA records found"); local_qr.read_event().await.assert_reload(); - local_qr.assert_queue_is_empty().await; + local_qr.assert_no_events(); // Expect TLS failure report let report = rr.read_report().await.unwrap_tls(); @@ -173,12 +172,12 @@ async fn dane_verify() { local_qr .expect_message() .await - .read_lines(&core) + .read_lines(&local_qr) .await .assert_contains(" (DANE failed to authenticate") .assert_contains("No matching certificates found"); local_qr.read_event().await.assert_reload(); - local_qr.assert_queue_is_empty().await; + local_qr.assert_no_events(); // Expect TLS failure report let report = rr.read_report().await.unwrap_tls(); @@ -217,11 +216,11 @@ async fn dane_verify() { .try_deliver(core.clone()) .await; local_qr.read_event().await.assert_reload(); - local_qr.assert_queue_is_empty().await; + local_qr.assert_no_events(); remote_qr - .last_queued_message() + .expect_message() .await - .read_lines(&core) + .read_lines(&remote_qr) .await .assert_contains("using TLSv1.3 with cipher"); diff --git a/tests/src/smtp/outbound/extensions.rs b/tests/src/smtp/outbound/extensions.rs index 67b886c8..bfe2b286 100644 --- a/tests/src/smtp/outbound/extensions.rs +++ b/tests/src/smtp/outbound/extensions.rs @@ -36,10 +36,7 @@ use crate::smtp::{ session::{TestSession, VerifyResponse}, TestConfig, TestSMTP, }; -use smtp::{ - core::{Session, SMTP}, - queue::{manager::Queue, DeliveryAttempt}, -}; +use smtp::core::{Session, SMTP}; #[tokio::test] #[serial_test::serial] @@ -81,7 +78,7 @@ async fn extensions() { core.session.config.rcpt.relay = IfBlock::new(true); core.session.config.extensions.dsn = IfBlock::new(true); let core = Arc::new(core); - let mut queue = Queue::default(); + //let mut queue = Queue::default(); let mut session = Session::test(core.clone()); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; @@ -94,44 +91,48 @@ async fn extensions() { "250", ) .await; - local_qr.expect_message_then_deliver().await + local_qr + .expect_message_then_deliver() + .await .try_deliver(core.clone()) .await; local_qr - .read_event() + .expect_message() + .await + .read_lines(&local_qr) .await - .unwrap_message() - .read_lines(&core).await .assert_contains(" (delivered to") .assert_contains("Final-Recipient: rfc822;bill@foobar.org") .assert_contains("Action: delivered"); - local_qr.read_event().await.unwrap_done(); + local_qr.read_event().await.assert_reload(); remote_qr - .read_event() + .expect_message() + .await + .read_lines(&remote_qr) .await - .unwrap_message() - .read_lines(&core).await .assert_contains("using TLSv1.3 with cipher"); // Test SIZE extension session .send_message("john@test.org", &["bill@foobar.org"], "test:arc", "250") .await; - local_qr.expect_message_then_deliver().await + local_qr + .expect_message_then_deliver() + .await .try_deliver(core.clone()) .await; local_qr - .read_event() + .expect_message() + .await + .read_lines(&local_qr) .await - .unwrap_message() - .read_lines(&core).await .assert_contains(" (host 'mx.foobar.org' rejected command 'MAIL FROM:") .assert_contains("Action: failed") .assert_contains("Diagnostic-Code: smtp;552") .assert_contains("Status: 5.3.4"); - local_qr.read_event().await.unwrap_done(); - remote_qr.assert_empty_queue(); + local_qr.read_event().await.assert_reload(); + remote_qr.assert_no_events(); // Test DSN, SMTPUTF8 and REQUIRETLS extensions session @@ -142,11 +143,13 @@ async fn extensions() { "250", ) .await; - local_qr.expect_message_then_deliver().await + local_qr + .expect_message_then_deliver() + .await .try_deliver(core.clone()) .await; - local_qr.read_event().await.unwrap_done(); - let message = remote_qr.expect_message().await(); + local_qr.read_event().await.assert_reload(); + let message = remote_qr.expect_message().await; assert_eq!(message.env_id, Some("abc123".to_string())); assert!((message.flags & MAIL_RET_HDRS) != 0); assert!((message.flags & MAIL_REQUIRETLS) != 0); diff --git a/tests/src/smtp/outbound/ip_lookup.rs b/tests/src/smtp/outbound/ip_lookup.rs index e317b1cc..c2f4c2e2 100644 --- a/tests/src/smtp/outbound/ip_lookup.rs +++ b/tests/src/smtp/outbound/ip_lookup.rs @@ -29,14 +29,8 @@ use std::{ use mail_auth::{IpLookupStrategy, MX}; use utils::config::{if_block::IfBlock, ServerProtocol}; -use crate::smtp::{ - inbound::TestQueueEvent, outbound::start_test_server, session::TestSession, TestConfig, - TestSMTP, -}; -use smtp::{ - core::{Session, SMTP}, - queue::{manager::Queue, DeliveryAttempt}, -}; +use crate::smtp::{outbound::start_test_server, session::TestSession, TestConfig, TestSMTP}; +use smtp::core::{Session, SMTP}; #[tokio::test] #[serial_test::serial] @@ -85,7 +79,6 @@ async fn ip_lookup_strategy() { core.session.config.rcpt.relay = IfBlock::new(true); let core = Arc::new(core); - let mut queue = Queue::default(); let mut session = Session::test(core.clone()); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; @@ -93,17 +86,22 @@ async fn ip_lookup_strategy() { session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; - local_qr.expect_message_then_deliver().await + local_qr + .expect_message_then_deliver() + .await .try_deliver(core.clone()) .await; + tokio::time::sleep(Duration::from_millis(100)).await; if matches!(strategy, IpLookupStrategy::Ipv6thenIpv4) { - local_qr.read_event().await.unwrap_done(); - remote_qr.expect_message().await(); + remote_qr.expect_message().await; } else { - let status = local_qr.read_event().await.unwrap_retry().inner.domains[0] - .status - .to_string(); - assert!(status.contains("Connection refused")); + let message = local_qr.last_queued_message().await; + let status = message.domains[0].status.to_string(); + assert!( + status.contains("Connection refused"), + "Message: {:?}", + message + ); } } } diff --git a/tests/src/smtp/outbound/lmtp.rs b/tests/src/smtp/outbound/lmtp.rs index c862527f..ecc76204 100644 --- a/tests/src/smtp/outbound/lmtp.rs +++ b/tests/src/smtp/outbound/lmtp.rs @@ -27,7 +27,7 @@ use std::{ }; use crate::smtp::{ - inbound::{TestMessage, TestQueueEvent}, + inbound::TestMessage, outbound::start_test_server, session::{TestSession, VerifyResponse}, ParseTestConfig, TestConfig, TestSMTP, @@ -35,8 +35,9 @@ use crate::smtp::{ use smtp::{ config::shared::ConfigShared, core::{Session, SMTP}, - queue::{manager::Queue, DeliveryAttempt, Event}, + queue::{DeliveryAttempt, Event}, }; +use store::write::now; use utils::config::{if_block::IfBlock, Config, ServerProtocol}; const REMOTE: &str = " @@ -89,17 +90,16 @@ async fn lmtp_delivery() { core.session.config.rcpt.max_recipients = IfBlock::new(100); core.session.config.extensions.dsn = IfBlock::new(true); let config = &mut core.queue.config; - config.retry = IfBlock::new(Duration::from_millis(100)); - config.notify = r#"[{if = "rcpt_domain = 'foobar.org'", then = "['100ms', '200ms']"}, - {else = ['100ms']}]"# + config.retry = IfBlock::new(Duration::from_secs(1)); + config.notify = r#"[{if = "rcpt_domain = 'foobar.org'", then = "[1s, 2s]"}, + {else = [1s]}]"# .parse_if(); - config.expire = r#"[{if = "rcpt_domain = 'foobar.org'", then = "400ms"}, - {else = "500ms"}]"# + config.expire = r#"[{if = "rcpt_domain = 'foobar.org'", then = "4s"}, + {else = "5s"}]"# .parse_if(); config.timeout.data = IfBlock::new(Duration::from_millis(50)); let core = Arc::new(core); - let mut queue = Queue::default(); let mut session = Session::test(core.clone()); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; @@ -127,37 +127,39 @@ async fn lmtp_delivery() { let mut dsn = Vec::new(); loop { match local_qr.try_read_event().await { - Some(Event::Queue(message)) => { - dsn.push(message.inner); - } - Some(Event::Done(wr)) => match wr { - WorkerResult::Done => { - break; - } - WorkerResult::Retry(retry) => { - queue.schedule(retry); - } - WorkerResult::OnHold(_) => unreachable!(), - }, + Some(Event::Reload) => {} + Some(Event::OnHold(_)) => unreachable!(), None | Some(Event::Stop) => break, - Some(Event::Manage(_)) => unreachable!(), } - if !queue.scheduled.is_empty() { - tokio::time::sleep(queue.wake_up_time()).await; - DeliveryAttempt::from(queue.next_due().unwrap()) - .try_deliver(core.clone()) - .await; + let events = core.next_event().await; + if events.is_empty() { + break; + } + let now = now(); + for event in events { + if event.due > now { + tokio::time::sleep(Duration::from_secs(event.due - now)).await; + } + + let message = core.read_message(event.queue_id).await.unwrap(); + if message.return_path.is_empty() { + message.clone().remove(&core, event.due).await; + dsn.push(message); + } else { + DeliveryAttempt::new(event).try_deliver(core.clone()).await; + tokio::time::sleep(Duration::from_millis(100)).await; + } } } - assert!(queue.scheduled.is_empty()); + local_qr.assert_queue_is_empty().await; assert_eq!(dsn.len(), 4); let mut dsn = dsn.into_iter(); dsn.next() .unwrap() - .read_lines(&core) + .read_lines(&local_qr) .await .assert_contains(" (delivered to") .assert_contains(" (delivered to") @@ -167,30 +169,29 @@ async fn lmtp_delivery() { dsn.next() .unwrap() - .read_lines(&core) + .read_lines(&local_qr) .await .assert_contains(" (host 'lmtp.foobar.org' rejected") .assert_contains("Action: delayed"); dsn.next() .unwrap() - .read_lines(&core) + .read_lines(&local_qr) .await .assert_contains(" (host 'lmtp.foobar.org' rejected") .assert_contains("Action: delayed"); dsn.next() .unwrap() - .read_lines(&core) + .read_lines(&local_qr) .await .assert_contains(" (host 'lmtp.foobar.org' rejected") .assert_contains("Action: failed"); assert_eq!( remote_qr - .read_event() + .expect_message() .await - .unwrap_message() .recipients .into_iter() .map(|r| r.address) @@ -201,5 +202,5 @@ async fn lmtp_delivery() { "john@foobar.org".to_string() ] ); - remote_qr.assert_empty_queue(); + remote_qr.assert_no_events(); } diff --git a/tests/src/smtp/outbound/mta_sts.rs b/tests/src/smtp/outbound/mta_sts.rs index 08b483aa..c96f029f 100644 --- a/tests/src/smtp/outbound/mta_sts.rs +++ b/tests/src/smtp/outbound/mta_sts.rs @@ -44,7 +44,6 @@ use smtp::{ config::{AggregateFrequency, RequireOptional}, core::{Session, SMTP}, outbound::mta_sts::{lookup::STS_TEST_POLICY, Policy}, - queue::{manager::Queue, DeliveryAttempt}, reporting::PolicyType, }; @@ -93,7 +92,7 @@ async fn mta_sts_verify() { core.report.config.tls.send = IfBlock::new(AggregateFrequency::Weekly); let core = Arc::new(core); - let mut queue = Queue::default(); + //let mut queue = Queue::default(); let mut session = Session::test(core.clone()); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; @@ -107,14 +106,13 @@ async fn mta_sts_verify() { .try_deliver(core.clone()) .await; local_qr - .read_event() + .expect_message() .await - .unwrap_message() - .read_lines(&core) + .read_lines(&local_qr) .await .assert_contains(" (MTA-STS failed to authenticate") .assert_contains("Record not found"); - local_qr.read_event().await.unwrap_done(); + local_qr.read_event().await.assert_reload(); // Expect TLS failure report let report = rr.read_report().await.unwrap_tls(); @@ -144,14 +142,13 @@ async fn mta_sts_verify() { .try_deliver(core.clone()) .await; local_qr - .read_event() + .expect_message() .await - .unwrap_message() - .read_lines(&core) + .read_lines(&local_qr) .await .assert_contains(" (MTA-STS failed to authenticate") .assert_contains("No 'mx' entries found"); - local_qr.read_event().await.unwrap_done(); + local_qr.read_event().await.assert_reload(); // Expect TLS failure report let report = rr.read_report().await.unwrap_tls(); @@ -178,14 +175,13 @@ async fn mta_sts_verify() { .try_deliver(core.clone()) .await; local_qr - .read_event() + .expect_message() .await - .unwrap_message() - .read_lines(&core) + .read_lines(&local_qr) .await .assert_contains(" (MTA-STS failed to authenticate") .assert_contains("not authorized by policy"); - local_qr.read_event().await.unwrap_done(); + local_qr.read_event().await.assert_reload(); // Expect TLS failure report let report = rr.read_report().await.unwrap_tls(); @@ -203,7 +199,7 @@ async fn mta_sts_verify() { report.failure.as_ref().unwrap().result_type, ResultType::ValidationFailure ); - remote_qr.assert_empty_queue(); + remote_qr.assert_no_events(); // MTA-STS successful validation core.resolvers.dns.txt_add( @@ -227,12 +223,11 @@ async fn mta_sts_verify() { .await .try_deliver(core.clone()) .await; - local_qr.read_event().await.unwrap_done(); + local_qr.read_event().await.assert_reload(); remote_qr - .read_event() + .expect_message() .await - .unwrap_message() - .read_lines(&core) + .read_lines(&remote_qr) .await .assert_contains("using TLSv1.3 with cipher"); diff --git a/tests/src/smtp/outbound/smtp.rs b/tests/src/smtp/outbound/smtp.rs index 5d867e31..2c55839e 100644 --- a/tests/src/smtp/outbound/smtp.rs +++ b/tests/src/smtp/outbound/smtp.rs @@ -27,6 +27,7 @@ use std::{ }; use mail_auth::MX; +use store::write::now; use utils::config::{if_block::IfBlock, ServerProtocol}; use crate::smtp::{ @@ -37,7 +38,7 @@ use crate::smtp::{ }; use smtp::{ core::{Session, SMTP}, - queue::{manager::Queue, DeliveryAttempt, Event}, + queue::{DeliveryAttempt, Event}, }; const SMUGGLER: &str = r#"From: Joe SixPack @@ -77,7 +78,8 @@ async fn smtp_delivery() { core.session.config.extensions.dsn = IfBlock::new(true); core.session.config.extensions.chunking = IfBlock::new(false); let mut remote_qr = core.init_test_queue("smtp_delivery_remote"); - let _rx = start_test_server(core.into(), &[ServerProtocol::Smtp]); + let remote_core = Arc::new(core); + let _rx = start_test_server(remote_core.clone(), &[ServerProtocol::Smtp]); // Add mock DNS entries let mut core = SMTP::test(); @@ -108,17 +110,16 @@ async fn smtp_delivery() { core.session.config.rcpt.max_recipients = IfBlock::new(100); core.session.config.extensions.dsn = IfBlock::new(true); let config = &mut core.queue.config; - config.retry = IfBlock::new(Duration::from_millis(100)); - config.notify = r#"[{if = "rcpt_domain = 'foobar.org'", then = "['100ms', '200ms']"}, - {if = "rcpt_domain = 'foobar.com'", then = "['500ms', '600ms']"}, - {else = ['100ms']}]"# + config.retry = IfBlock::new(Duration::from_secs(1)); + config.notify = r#"[{if = "rcpt_domain = 'foobar.org'", then = "[1s, 2s]"}, + {if = "rcpt_domain = 'foobar.com'", then = "[5s, 6s]"}, + {else = [1s]}]"# .parse_if(); - config.expire = r#"[{if = "rcpt_domain = 'foobar.org'", then = "650ms"}, - {else = "750ms"}]"# + config.expire = r#"[{if = "rcpt_domain = 'foobar.org'", then = "6s"}, + {else = "7s"}]"# .parse_if(); let core = Arc::new(core); - let mut queue = Queue::default(); let mut session = Session::test(core.clone()); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; @@ -139,40 +140,44 @@ async fn smtp_delivery() { "250", ) .await; - let message = local_qr.expect_message().await(); + let message = local_qr.expect_message().await; let num_domains = message.domains.len(); assert_eq!(num_domains, 3); - DeliveryAttempt::from(message) + local_qr + .delivery_attempt(message.id) + .await .try_deliver(core.clone()) .await; let mut dsn = Vec::new(); let mut domain_retries = vec![0; num_domains]; loop { match local_qr.try_read_event().await { - Some(Event::Queue(message)) => { - dsn.push(message.inner); - } - Some(Event::Done(wr)) => match wr { - WorkerResult::Done => { - break; - } - WorkerResult::Retry(retry) => { - for (idx, domain) in retry.inner.domains.iter().enumerate() { - domain_retries[idx] = domain.retry.inner; - } - queue.schedule(retry); - } - WorkerResult::OnHold(_) => unreachable!(), - }, + Some(Event::Reload) => {} + Some(Event::OnHold(_)) => unreachable!(), None | Some(Event::Stop) => break, - Some(Event::Manage(_)) => unreachable!(), } - if !queue.scheduled.is_empty() { - tokio::time::sleep(queue.wake_up_time()).await; - DeliveryAttempt::from(queue.next_due().unwrap()) - .try_deliver(core.clone()) - .await; + let events = core.next_event().await; + if events.is_empty() { + break; + } + let now = now(); + for event in events { + if event.due > now { + tokio::time::sleep(Duration::from_secs(event.due - now)).await; + } + + let message = core.read_message(event.queue_id).await.unwrap(); + if message.return_path.is_empty() { + message.clone().remove(&core, event.due).await; + dsn.push(message); + } else { + for (idx, domain) in message.domains.iter().enumerate() { + domain_retries[idx] = domain.retry.inner; + } + DeliveryAttempt::new(event).try_deliver(core.clone()).await; + tokio::time::sleep(Duration::from_millis(100)).await; + } } } assert_eq!(domain_retries[0], 0, "retries {domain_retries:?}"); @@ -183,14 +188,15 @@ async fn smtp_delivery() { "retries {domain_retries:?}" ); - assert!(queue.scheduled.is_empty()); + local_qr.assert_queue_is_empty().await; assert_eq!(dsn.len(), 5); let mut dsn = dsn.into_iter(); dsn.next() .unwrap() - .read_lines(&core).await + .read_lines(&local_qr) + .await .assert_contains(" (delivered to") .assert_contains(" (delivered to") .assert_contains(" (failed to lookup") @@ -199,53 +205,54 @@ async fn smtp_delivery() { dsn.next() .unwrap() - .read_lines(&core).await + .read_lines(&local_qr) + .await .assert_contains(" (host ") .assert_contains(" (host ") .assert_contains("Action: delayed"); dsn.next() .unwrap() - .read_lines(&core).await + .read_lines(&local_qr) + .await .assert_contains(" (host ") .assert_contains("Action: delayed"); dsn.next() .unwrap() - .read_lines(&core).await + .read_lines(&local_qr) + .await .assert_contains(" (host "); dsn.next() .unwrap() - .read_lines(&core).await + .read_lines(&local_qr) + .await .assert_contains(" (host ") .assert_contains("Action: failed"); assert_eq!( remote_qr - .read_event() + .consume_message(&remote_core) .await - .unwrap_message() - .recipients - .into_iter() - .map(|r| r.address) - .collect::>(), - vec!["ok@foobar.net".to_string()] - ); - assert_eq!( - remote_qr - .read_event() - .await - .unwrap_message() .recipients .into_iter() .map(|r| r.address) .collect::>(), vec!["ok@foobar.org".to_string()] ); + assert_eq!( + remote_qr + .consume_message(&remote_core) + .await + .recipients + .into_iter() + .map(|r| r.address) + .collect::>(), + vec!["ok@foobar.net".to_string()] + ); - remote_qr.assert_empty_queue(); - local_qr.assert_empty_queue(); + remote_qr.assert_no_events(); // SMTP smuggling for separator in ["\n", "\r"].iter() { @@ -261,18 +268,18 @@ async fn smtp_delivery() { session .send_message("john@doe.org", &["bill@foobar.com"], &message, "250") .await; - local_qr.expect_message_then_deliver().await + local_qr + .expect_message_then_deliver() + .await .try_deliver(core.clone()) .await; - let event = local_qr.read_event().await; + local_qr.read_event().await.assert_reload(); - assert!( - matches!(event, Event::Done(WorkerResult::Done)), - "event: {:?}", - event - ); - - let message = remote_qr.expect_message().await().read_message(); + let message = remote_qr + .consume_message(&remote_core) + .await + .read_message(&remote_qr) + .await; assert!( message.contains("This is a smuggled message"), diff --git a/tests/src/smtp/outbound/throttle.rs b/tests/src/smtp/outbound/throttle.rs index b33d9a67..dc0e8dff 100644 --- a/tests/src/smtp/outbound/throttle.rs +++ b/tests/src/smtp/outbound/throttle.rs @@ -28,6 +28,7 @@ use std::{ }; use mail_auth::MX; +use store::write::now; use utils::config::if_block::IfBlock; use crate::smtp::{ @@ -36,7 +37,7 @@ use crate::smtp::{ }; use smtp::{ core::{Session, SMTP}, - queue::{manager::Queue, DeliveryAttempt, Message, QueueEnvelope}, + queue::{Message, QueueEnvelope}, }; const THROTTLE: &str = r#" @@ -92,7 +93,6 @@ async fn throttle_outbound() { core.queue.config.expire = IfBlock::new(Duration::from_secs(86400)); let core = Arc::new(core); - let mut queue = Queue::default(); let mut session = Session::test(core.clone()); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; @@ -100,6 +100,7 @@ async fn throttle_outbound() { session .send_message("john@foobar.org", &["bill@test.org"], "test:no_dkim", "250") .await; + assert_eq!(local_qr.last_queued_due().await as i64 - now() as i64, 0); // Throttle sender let span = tracing::info_span!("test"); @@ -123,10 +124,9 @@ async fn throttle_outbound() { .await .try_deliver(core.clone()) .await; - local_qr.assert_empty_queue(); + tokio::time::sleep(Duration::from_millis(100)).await; + local_qr.read_event().await.unwrap_on_hold(); in_flight.clear(); - assert!(!queue.on_hold.is_empty()); - queue.next_on_hold().unwrap(); // Expect rate limit throttle for sender domain 'foobar.net' test_message.return_path_domain = "foobar.net".to_string(); @@ -149,16 +149,10 @@ async fn throttle_outbound() { .await .try_deliver(core.clone()) .await; - local_qr.assert_empty_queue(); - assert!([1799, 1800].contains( - &queue - .scheduled - .pop() - .unwrap() - .due - .duration_since(Instant::now()) - .as_secs() - )); + tokio::time::sleep(Duration::from_millis(100)).await; + local_qr.read_event().await.assert_reload(); + let due = local_qr.last_queued_due().await - now(); + assert!(due > 0, "Due: {}", due); // Expect concurrency throttle for recipient domain 'example.org' test_message.return_path_domain = "test.net".to_string(); @@ -186,6 +180,7 @@ async fn throttle_outbound() { .await .try_deliver(core.clone()) .await; + tokio::time::sleep(Duration::from_millis(100)).await; local_qr.read_event().await.unwrap_on_hold(); in_flight.clear(); @@ -214,15 +209,10 @@ async fn throttle_outbound() { .await .try_deliver(core.clone()) .await; - assert!([2399, 2400].contains( - &local_qr - .read_event() - .await - .unwrap_retry() - .due - .duration_since(Instant::now()) - .as_secs() - )); + tokio::time::sleep(Duration::from_millis(100)).await; + local_qr.read_event().await.assert_reload(); + let due = local_qr.last_queued_due().await - now(); + assert!(due > 0, "Due: {}", due); // Expect concurrency throttle for mx 'mx.test.org' core.resolvers.dns.mx_add( @@ -293,15 +283,11 @@ async fn throttle_outbound() { .await .try_deliver(core.clone()) .await; - assert!([2999, 3000].contains( - &local_qr - .read_event() - .await - .unwrap_retry() - .due - .duration_since(Instant::now()) - .as_secs() - )); + + tokio::time::sleep(Duration::from_millis(100)).await; + local_qr.read_event().await.assert_reload(); + let due = local_qr.last_queued_due().await - now(); + assert!(due > 0, "Due: {}", due); } pub trait TestQueueEnvelope<'x> { diff --git a/tests/src/smtp/outbound/tls.rs b/tests/src/smtp/outbound/tls.rs index 3008eb74..fcab4511 100644 --- a/tests/src/smtp/outbound/tls.rs +++ b/tests/src/smtp/outbound/tls.rs @@ -27,10 +27,11 @@ use std::{ }; use mail_auth::MX; +use store::write::now; use utils::config::{if_block::IfBlock, ServerProtocol}; use crate::smtp::{ - inbound::{TestMessage, TestQueueEvent}, + inbound::TestMessage, outbound::start_test_server, session::{TestSession, VerifyResponse}, TestConfig, TestSMTP, @@ -38,7 +39,6 @@ use crate::smtp::{ use smtp::{ config::RequireOptional, core::{Session, SMTP}, - queue::{manager::Queue, DeliveryAttempt}, }; #[tokio::test] @@ -80,7 +80,6 @@ async fn starttls_optional() { core.queue.config.tls.start = IfBlock::new(RequireOptional::Optional); let core = Arc::new(core); - let mut queue = Queue::default(); let mut session = Session::test(core.clone()); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; @@ -93,18 +92,25 @@ async fn starttls_optional() { .await .try_deliver(core.clone()) .await; - let mut retry = local_qr.read_event().await.unwrap_retry(); - assert!(retry.inner.domains[0].disable_tls); - retry.inner.domains[0].retry.due = Instant::now(); - DeliveryAttempt::from(retry.inner) + let mut retry = local_qr.expect_message().await; + assert!(retry.domains[0].disable_tls); + let prev_due = retry.domains[0].retry.due; + let next_due = now(); + let queue_id = retry.id; + retry.domains[0].retry.due = next_due; + retry + .save_changes(&core, prev_due.into(), next_due.into()) + .await; + local_qr + .delivery_attempt(queue_id) + .await .try_deliver(core.clone()) .await; - local_qr.read_event().await.unwrap_done(); + tokio::time::sleep(Duration::from_millis(100)).await; remote_qr - .read_event() + .expect_message() .await - .unwrap_message() - .read_lines(&core) + .read_lines(&remote_qr) .await .assert_not_contains("using TLSv1.3 with cipher"); } diff --git a/tests/src/smtp/queue/concurrent.rs b/tests/src/smtp/queue/concurrent.rs new file mode 100644 index 00000000..9851a262 --- /dev/null +++ b/tests/src/smtp/queue/concurrent.rs @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use mail_auth::MX; +use utils::config::{if_block::IfBlock, ServerProtocol}; + +use crate::smtp::{outbound::start_test_server, session::TestSession, TestConfig, TestSMTP}; +use smtp::{ + core::{Session, SMTP}, + queue::manager::Queue, +}; + +#[tokio::test] +#[serial_test::serial] +async fn concurrent_queue() { + /*let disable = true; + tracing::subscriber::set_global_default( + tracing_subscriber::FmtSubscriber::builder() + .with_max_level(tracing::Level::DEBUG) + .finish(), + ) + .unwrap();*/ + + // Start test server + let mut core = SMTP::test(); + core.session.config.rcpt.relay = IfBlock::new(true); + let remote_qr = core.init_test_queue("smtp_concurrent_queue_remote"); + let _rx = start_test_server(core.into(), &[ServerProtocol::Smtp]); + + // Add mock DNS entries + let mut core = SMTP::test(); + core.resolvers.dns.mx_add( + "foobar.org", + vec![MX { + exchanges: vec!["mx.foobar.org".to_string()], + preference: 10, + }], + Instant::now() + Duration::from_secs(100), + ); + core.resolvers.dns.ipv4_add( + "mx.foobar.org", + vec!["127.0.0.1".parse().unwrap()], + Instant::now() + Duration::from_secs(100), + ); + let local_qr = core.init_test_queue("smtp_concurrent_queue_local"); + core.session.config.rcpt.relay = IfBlock::new(true); + core.session.config.data.max_messages = IfBlock::new(200); + + let core = Arc::new(core); + let mut session = Session::test(core.clone()); + session.data.remote_ip_str = "10.0.0.1".to_string(); + session.eval_session_params().await; + session.ehlo("mx.test.org").await; + + // Send 100 test messages + for _ in 0..100 { + session + .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") + .await; + } + + // Spawn 20 concurrent queues at different times + for _ in 0..10 { + let core = core.clone(); + tokio::spawn(async move { + Queue::new(core).process_events().await; + }); + } + tokio::time::sleep(Duration::from_millis(500)).await; + for _ in 0..10 { + let core = core.clone(); + tokio::spawn(async move { + Queue::new(core).process_events().await; + }); + } + tokio::time::sleep(Duration::from_millis(1500)).await; + + local_qr.assert_queue_is_empty().await; + let remote_messages = remote_qr.read_queued_messages().await; + assert_eq!(remote_messages.len(), 100); + + // Make sure local store is queue + core.shared + .default_data_store + .assert_is_empty(core.shared.default_blob_store.clone()) + .await; +} diff --git a/tests/src/smtp/queue/dsn.rs b/tests/src/smtp/queue/dsn.rs index 49ae936c..bc416991 100644 --- a/tests/src/smtp/queue/dsn.rs +++ b/tests/src/smtp/queue/dsn.rs @@ -21,26 +21,19 @@ * for more details. */ -use std::{ - fs, - path::PathBuf, - time::{Duration, Instant, SystemTime}, -}; +use std::{fs, path::PathBuf, time::SystemTime}; use smtp_proto::{Response, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_SUCCESS}; -use tokio::{fs::File, io::AsyncReadExt}; +use store::write::now; +use utils::BlobHash; use crate::smtp::{ - inbound::{sign::TextConfigContext, TestQueueEvent}, - ParseTestConfig, TestConfig, TestSMTP, + inbound::sign::TextConfigContext, ParseTestConfig, QueueReceiver, TestConfig, TestSMTP, }; use smtp::{ config::ConfigContext, core::SMTP, - queue::{ - DeliveryAttempt, Domain, Error, ErrorDetails, HostResponse, Message, Recipient, Schedule, - Status, - }, + queue::{Domain, Error, ErrorDetails, HostResponse, Message, Recipient, Schedule, Status}, }; #[tokio::test] @@ -51,12 +44,12 @@ async fn generate_dsn() { path.push("dsn"); path.push("original.txt"); let size = fs::metadata(&path).unwrap().len() as usize; + let dsn_original = fs::read_to_string(&path).unwrap(); let flags = RCPT_NOTIFY_FAILURE | RCPT_NOTIFY_DELAY | RCPT_NOTIFY_SUCCESS; - let message = Box::new(Message { + let mut message = Message { size, id: 0, - path, created: SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()), @@ -85,25 +78,20 @@ async fn generate_dsn() { domain: "example.org".to_string(), retry: Schedule::now(), notify: Schedule::now(), - expires: Instant::now() + Duration::from_secs(10), + expires: now() + 10, status: Status::TemporaryFailure(Error::ConnectionError(ErrorDetails { entity: "mx.domain.org".to_string(), details: "Connection timeout".to_string(), })), disable_tls: false, - changed: false, }], flags: 0, env_id: None, priority: 0, - - queue_refs: vec![], - }); - let mut attempt = DeliveryAttempt { - span: tracing::span!(tracing::Level::INFO, "hi"), - message, - in_flight: vec![], + blob_hash: BlobHash::from(dsn_original.as_bytes()), + quota_keys: vec![], }; + let span = tracing::span!(tracing::Level::INFO, "hi"); // Load config let mut core = SMTP::test(); @@ -113,18 +101,24 @@ async fn generate_dsn() { // Create temp dir for queue let mut qr = core.init_test_queue("smtp_dsn_test"); + qr.blob_store + .put_blob(message.blob_hash.as_slice(), dsn_original.as_bytes()) + .await + .unwrap(); // Disabled DSN - core.send_dsn(&mut attempt).await; - qr.assert_empty_queue(); + core.send_dsn(&mut message, &span).await; + qr.assert_no_events(); + qr.assert_queue_is_empty().await; // Failure DSN - attempt.message.recipients[0].flags = flags; - core.send_dsn(&mut attempt).await; - compare_dsn(qr.expect_message().await(), "failure.eml").await; + message.recipients[0].flags = flags; + core.send_dsn(&mut message, &span).await; + let dsn_message = qr.expect_message().await; + qr.compare_dsn(dsn_message, "failure.eml").await; // Success DSN - attempt.message.recipients.push(Recipient { + message.recipients.push(Recipient { domain_idx: 0, address: "jane@example.org".to_string(), address_lcase: "jane@example.org".to_string(), @@ -139,11 +133,12 @@ async fn generate_dsn() { flags, orcpt: None, }); - core.send_dsn(&mut attempt).await; - compare_dsn(qr.expect_message().await(), "success.eml").await; + core.send_dsn(&mut message, &span).await; + let dsn_message = qr.expect_message().await; + qr.compare_dsn(dsn_message, "success.eml").await; // Delay DSN - attempt.message.recipients.push(Recipient { + message.recipients.push(Recipient { domain_idx: 0, address: "john.doe@example.org".to_string(), address_lcase: "john.doe@example.org".to_string(), @@ -151,49 +146,52 @@ async fn generate_dsn() { flags, orcpt: "jdoe@example.org".to_string().into(), }); - core.send_dsn(&mut attempt).await; - compare_dsn(qr.expect_message().await(), "delay.eml").await; + core.send_dsn(&mut message, &span).await; + let dsn_message = qr.expect_message().await; + qr.compare_dsn(dsn_message, "delay.eml").await; // Mixed DSN - for rcpt in &mut attempt.message.recipients { + for rcpt in &mut message.recipients { rcpt.flags = flags; } - attempt.message.domains[0].notify.due = Instant::now(); - core.send_dsn(&mut attempt).await; - compare_dsn(qr.expect_message().await(), "mixed.eml").await; + message.domains[0].notify.due = now(); + core.send_dsn(&mut message, &span).await; + let dsn_message = qr.expect_message().await; + qr.compare_dsn(dsn_message, "mixed.eml").await; // Load queue - let queue = core.queue.read_queue().await; - assert_eq!(queue.scheduled.len(), 4); + let queue = qr.read_queued_messages().await; + assert_eq!(queue.len(), 4); } -async fn compare_dsn(message: Box, test: &str) { - let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - path.push("resources"); - path.push("smtp"); - path.push("dsn"); - path.push(test); +impl QueueReceiver { + async fn compare_dsn(&self, message: Message, test: &str) { + let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + path.push("resources"); + path.push("smtp"); + path.push("dsn"); + path.push(test); - let mut bytes = vec![0u8; message.size]; - File::open(&message.path) - .await - .unwrap() - .read_exact(&mut bytes) - .await - .unwrap(); + let bytes = self + .blob_store + .get_blob(message.blob_hash.as_slice(), 0..u32::MAX) + .await + .unwrap() + .unwrap(); - let dsn = remove_ids(bytes); - let dsn_expected = fs::read_to_string(&path).unwrap(); + let dsn = remove_ids(bytes); + let dsn_expected = fs::read_to_string(&path).unwrap(); - if dsn != dsn_expected { - let mut failed = PathBuf::from(&path); - failed.set_extension("failed"); - fs::write(&failed, dsn.as_bytes()).unwrap(); - panic!( - "Failed for {}, output saved to {}", - path.display(), - failed.display() - ); + if dsn != dsn_expected { + let mut failed = PathBuf::from(&path); + failed.set_extension("failed"); + fs::write(&failed, dsn.as_bytes()).unwrap(); + panic!( + "Failed for {}, output saved to {}", + path.display(), + failed.display() + ); + } } } diff --git a/tests/src/smtp/queue/manager.rs b/tests/src/smtp/queue/manager.rs index 527219e1..7b443877 100644 --- a/tests/src/smtp/queue/manager.rs +++ b/tests/src/smtp/queue/manager.rs @@ -21,49 +21,57 @@ * for more details. */ -use std::time::{Duration, Instant}; +use std::{sync::Arc, time::Duration}; use mail_auth::hickory_resolver::proto::op::ResponseCode; -use smtp::queue::{manager::Queue, Domain, Message, Schedule, Status}; +use smtp::{ + core::SMTP, + queue::{Domain, Message, Schedule, Status}, +}; +use store::write::now; -#[test] -fn queue_due() { - let mut queue = Queue::default(); +use crate::smtp::{TestConfig, TestSMTP}; + +#[tokio::test] +async fn queue_due() { + let mut core = SMTP::test(); + let qr = core.init_test_queue("smtp_queue_due_test"); + let core = Arc::new(core); let mut message = new_message(0); message.domains.push(domain("c", 3, 8, 9)); - queue.schedule(Schedule { - due: message.next_delivery_event(), - inner: message, - }); + let due = message.next_delivery_event(); + message.save_changes(&core, 0.into(), due.into()).await; let mut message = new_message(1); message.domains.push(domain("b", 2, 6, 7)); - queue.schedule(Schedule { - due: message.next_delivery_event(), - inner: message, - }); + let due = message.next_delivery_event(); + message.save_changes(&core, 0.into(), due.into()).await; let mut message = new_message(2); message.domains.push(domain("a", 1, 4, 5)); - queue.schedule(Schedule { - due: message.next_delivery_event(), - inner: message, - }); + let due = message.next_delivery_event(); + message.save_changes(&core, 0.into(), due.into()).await; for domain in vec!["a", "b", "c"].into_iter() { - let wake_up = queue.wake_up_time(); - assert!( - (900..=1000).contains(&wake_up.as_millis()), - "{}", - wake_up.as_millis() - ); - std::thread::sleep(wake_up); - queue.next_due().unwrap().domain(domain); + let now = now(); + for queue_event in core.next_event().await { + if queue_event.due > now { + let wake_up = queue_event.due - now; + assert_eq!(wake_up, 1); + std::thread::sleep(Duration::from_secs(wake_up)); + } + if let Some(message) = core.read_message(queue_event.queue_id).await { + message.domain(domain); + message.remove(&core, queue_event.due).await; + } else { + panic!("Message not found"); + } + } } - assert!(queue.next_due().is_none()); + qr.assert_queue_is_empty().await; } #[test] @@ -127,11 +135,10 @@ fn delivery_events() { assert!(message.next_event().is_none()); } -pub fn new_message(id: u64) -> Box { - Box::new(Message { +pub fn new_message(id: u64) -> Message { + Message { size: 0, id, - path: Default::default(), created: 0, return_path: "sender@foobar.org".to_string(), return_path_lcase: "".to_string(), @@ -141,8 +148,9 @@ pub fn new_message(id: u64) -> Box { flags: 0, env_id: None, priority: 0, - queue_refs: vec![], - }) + quota_keys: vec![], + blob_hash: Default::default(), + } } fn domain(domain: &str, retry: u64, notify: u64, expires: u64) -> Domain { @@ -150,10 +158,9 @@ fn domain(domain: &str, retry: u64, notify: u64, expires: u64) -> Domain { domain: domain.to_string(), retry: Schedule::later(Duration::from_secs(retry)), notify: Schedule::later(Duration::from_secs(notify)), - expires: Instant::now() + Duration::from_secs(expires), + expires: now() + expires, status: Status::Scheduled, disable_tls: false, - changed: false, } } @@ -164,7 +171,10 @@ pub trait TestMessage { impl TestMessage for Message { fn domain(&self, name: &str) -> &Domain { - self.domains.iter().find(|d| d.domain == name).unwrap() + self.domains + .iter() + .find(|d| d.domain == name) + .unwrap_or_else(|| panic!("Expected domain {name} not found in {:?}", self.domains)) } fn domain_mut(&mut self, name: &str) -> &mut Domain { diff --git a/tests/src/smtp/queue/mod.rs b/tests/src/smtp/queue/mod.rs index c39cd621..259512f6 100644 --- a/tests/src/smtp/queue/mod.rs +++ b/tests/src/smtp/queue/mod.rs @@ -21,7 +21,7 @@ * for more details. */ +pub mod concurrent; pub mod dsn; pub mod manager; pub mod retry; -pub mod serialize; diff --git a/tests/src/smtp/queue/retry.rs b/tests/src/smtp/queue/retry.rs index b8b26a40..a49c38f3 100644 --- a/tests/src/smtp/queue/retry.rs +++ b/tests/src/smtp/queue/retry.rs @@ -21,10 +21,7 @@ * for more details. */ -use std::{ - sync::Arc, - time::{Duration, Instant}, -}; +use std::{sync::Arc, time::Duration}; use crate::smtp::{ inbound::{TestMessage, TestQueueEvent}, @@ -33,8 +30,9 @@ use crate::smtp::{ }; use smtp::{ core::{Session, SMTP}, - queue::{manager::Queue, DeliveryAttempt, Event}, + queue::{DeliveryAttempt, Event}, }; +use store::write::now; use utils::config::if_block::IfBlock; #[tokio::test] @@ -57,17 +55,17 @@ async fn queue_retry() { config.deliver_by = IfBlock::new(Duration::from_secs(86400)); config.future_release = IfBlock::new(Duration::from_secs(86400)); let config = &mut core.queue.config; - config.retry = r#""[100ms, 200ms, 300ms]""#.parse_if(); - config.notify = r#"[{if = "sender_domain = 'test.org'", then = "['150ms', '200ms']"}, + config.retry = r#""[1s, 2s, 3s]""#.parse_if(); + config.notify = r#"[{if = "sender_domain = 'test.org'", then = "[1s, 2s]"}, {else = ['15h', '22h']}]"# .parse_if(); - config.expire = r#"[{if = "sender_domain = 'test.org'", then = "600ms"}, + config.expire = r#"[{if = "sender_domain = 'test.org'", then = "6s"}, {else = '1d'}]"# .parse_if(); // Create test message let core = Arc::new(core); - let mut queue = Queue::default(); + let mut session = Session::test(core.clone()); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; @@ -75,22 +73,22 @@ async fn queue_retry() { session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; - let attempt = DeliveryAttempt::from(qr.expect_message().await()); + let attempt = qr.expect_message_then_deliver().await; // Expect a failed DSN - let path = attempt.message.path.clone(); attempt.try_deliver(core.clone()).await; - let message = qr.expect_message().await(); + let message = qr.expect_message().await; assert_eq!(message.return_path, ""); assert_eq!(message.domains.first().unwrap().domain, "test.org"); assert_eq!(message.recipients.first().unwrap().address, "john@test.org"); message - .read_lines(&core).await + .read_lines(&qr) + .await .assert_contains("Content-Type: multipart/report") .assert_contains("Final-Recipient: rfc822;bill@foobar.org") .assert_contains("Action: failed"); - qr.read_event().await.unwrap_done(); - assert!(!path.exists()); + qr.read_event().await.assert_reload(); + qr.clear_queue(&core).await; // Expect a failed DSN for foobar.org, followed by two delayed DSN and // a final failed DSN for _dns_error.org. @@ -102,65 +100,71 @@ async fn queue_retry() { "250", ) .await; - let attempt = DeliveryAttempt::from(qr.expect_message().await()); - let path = attempt.message.path.clone(); + let attempt = qr.expect_message_then_deliver().await; let mut dsn = Vec::new(); - let mut num_retries = 0; + let mut retries = Vec::new(); attempt.try_deliver(core.clone()).await; loop { match qr.try_read_event().await { - Some(Event::Queue(message)) => { - dsn.push(message.inner); - } - Some(Event::Done(wr)) => match wr { - WorkerResult::Done => break, - WorkerResult::Retry(retry) => { - queue.schedule(retry); - num_retries += 1; - } - WorkerResult::OnHold(_) => unreachable!(), - }, + Some(Event::Reload) => {} + Some(Event::OnHold(_)) => unreachable!(), None | Some(Event::Stop) => break, - Some(Event::Manage(_)) => unreachable!(), } - if !queue.scheduled.is_empty() { - tokio::time::sleep(queue.wake_up_time()).await; - DeliveryAttempt::from(queue.next_due().unwrap()) - .try_deliver(core.clone()) - .await; + let now = now(); + let events = core.next_event().await; + if events.is_empty() { + break; + } + for event in events { + if event.due > now { + tokio::time::sleep(Duration::from_secs(event.due - now)).await; + } + + let message = core.read_message(event.queue_id).await.unwrap(); + if message.return_path.is_empty() { + message.clone().remove(&core, event.due).await; + dsn.push(message); + } else { + retries.push(event.due - now); + DeliveryAttempt::new(event).try_deliver(core.clone()).await; + tokio::time::sleep(Duration::from_millis(100)).await; + } } } - assert!(queue.scheduled.is_empty()); - assert_eq!(num_retries, 3); + qr.assert_queue_is_empty().await; + assert_eq!(retries, vec![1, 2, 3]); assert_eq!(dsn.len(), 4); - assert!(!path.exists()); let mut dsn = dsn.into_iter(); dsn.next() .unwrap() - .read_lines(&core).await + .read_lines(&qr) + .await .assert_contains(" (failed to lookup 'foobar.org'") .assert_contains("Final-Recipient: rfc822;bill@foobar.org") .assert_contains("Action: failed"); dsn.next() .unwrap() - .read_lines(&core).await + .read_lines(&qr) + .await .assert_contains(" (failed to lookup '_dns_error.org'") .assert_contains("Final-Recipient: rfc822;jane@_dns_error.org") .assert_contains("Action: delayed"); dsn.next() .unwrap() - .read_lines(&core).await + .read_lines(&qr) + .await .assert_contains(" (failed to lookup '_dns_error.org'") .assert_contains("Final-Recipient: rfc822;jane@_dns_error.org") .assert_contains("Action: delayed"); dsn.next() .unwrap() - .read_lines(&core).await + .read_lines(&qr) + .await .assert_contains(" (failed to lookup '_dns_error.org'") .assert_contains("Final-Recipient: rfc822;jane@_dns_error.org") .assert_contains("Action: failed"); @@ -176,37 +180,12 @@ async fn queue_retry() { "250", ) .await; - let now = Instant::now(); - let schedule = qr.read_event().await.unwrap_schedule(); - assert!([59, 60].contains(&schedule.due.duration_since(now).as_secs())); - assert!([59, 60].contains( - &schedule - .inner - .next_delivery_event() - .duration_since(now) - .as_secs() - )); - assert!([3599, 3600].contains( - &schedule - .inner - .domains - .first() - .unwrap() - .expires - .duration_since(now) - .as_secs() - )); - assert!([54059, 54060].contains( - &schedule - .inner - .domains - .first() - .unwrap() - .notify - .due - .duration_since(now) - .as_secs() - )); + let now_ = now(); + let message = qr.expect_message().await; + assert!([59, 60].contains(&(qr.message_due(message.id).await - now_))); + assert!([59, 60].contains(&(message.next_delivery_event() - now_))); + assert!([3599, 3600].contains(&(message.domains.first().unwrap().expires - now_))); + assert!([54059, 54060].contains(&(message.domains.first().unwrap().notify.due - now_))); // Test DELIVERBY (NOTIFY) session @@ -217,17 +196,6 @@ async fn queue_retry() { "250", ) .await; - let now = Instant::now(); - let schedule = qr.read_event().await.unwrap_schedule(); - assert!([3599, 3600].contains( - &schedule - .inner - .domains - .first() - .unwrap() - .notify - .due - .duration_since(now) - .as_secs() - )); + let schedule = qr.expect_message().await; + assert!([3599, 3600].contains(&(schedule.domains.first().unwrap().notify.due - now()))); } diff --git a/tests/src/smtp/queue/serialize.rs b/tests/src/smtp/queue/serialize.rs deleted file mode 100644 index 76b461f0..00000000 --- a/tests/src/smtp/queue/serialize.rs +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::{ - path::PathBuf, - time::{Duration, Instant}, -}; - -use smtp_proto::{Response, MAIL_REQUIRETLS, MAIL_SMTPUTF8, RCPT_CONNEG, RCPT_NOTIFY_FAILURE}; - -use smtp::{ - core::SMTP, - queue::{ - Domain, Error, ErrorDetails, HostResponse, Message, Recipient, Schedule, Status, - RCPT_STATUS_CHANGED, - }, -}; - -use crate::smtp::{inbound::TestQueueEvent, TestConfig, TestSMTP}; - -#[tokio::test] -async fn queue_serialize() { - let mut core = SMTP::test(); - - // Create temp dir for queue - let mut qr = core.init_test_queue("smtp_queue_serialize_test"); - - // Create test message - let message = Message { - size: 0, - id: 0, - path: PathBuf::new(), - created: 123456, - return_path: "sender@FooBar.org".to_string(), - return_path_lcase: "sender@foobar.org".to_string(), - return_path_domain: "foobar.org".to_string(), - recipients: vec![ - Recipient { - domain_idx: 0, - address: "FOOBAR@example.org".to_string(), - address_lcase: "foobar@example.org".to_string(), - status: Status::Scheduled, - flags: RCPT_CONNEG, - orcpt: None, - }, - Recipient { - domain_idx: 1, - address: "FOOBAR@example.org".to_string(), - address_lcase: "foobar@example.org".to_string(), - status: Status::Scheduled, - flags: RCPT_NOTIFY_FAILURE, - orcpt: None, - }, - ], - domains: vec![ - Domain { - domain: "example.org".to_string(), - retry: Schedule::now(), - notify: Schedule::now(), - expires: Instant::now() + Duration::from_secs(10), - status: Status::Scheduled, - disable_tls: false, - changed: false, - }, - Domain { - domain: "example.com".to_string(), - retry: Schedule::now(), - notify: Schedule::now(), - expires: Instant::now() + Duration::from_secs(10), - status: Status::Scheduled, - disable_tls: false, - changed: false, - }, - ], - flags: MAIL_REQUIRETLS | MAIL_SMTPUTF8, - env_id: "hello".to_string().into(), - priority: -1, - - queue_refs: vec![], - }; - - // Queue message - assert!( - core.queue - .queue_message( - Box::new(message), - (&b"From: test@foobar.org\r\n"[..]).into(), - b"Subject: test\r\n\n\ntest", - &tracing::info_span!("hi") - ) - .await - ); - let mut message = qr.expect_message().await(); - - // Deserialize - assert_msg_eq( - &message, - &Message::from_path(message.path.clone()).await.unwrap(), - ); - - // Write update - message.recipients[0].status = Status::PermanentFailure(HostResponse { - hostname: ErrorDetails { - entity: "mx.example.org".to_string(), - details: "RCPT TO:".to_string(), - }, - response: Response { - code: 550, - esc: [5, 1, 2], - message: "User does not exist\nplease contact support for details\n".to_string(), - }, - }); - message.recipients[0].flags |= RCPT_STATUS_CHANGED; - - message.recipients[1].status = Status::Completed(HostResponse { - hostname: "smtp.foo.bar".to_string(), - response: Response { - code: 250, - esc: [2, 1, 5], - message: "Great success!".to_string(), - }, - }); - message.recipients[1].flags |= RCPT_STATUS_CHANGED; - - message.domains[0].status = Status::TemporaryFailure(Error::UnexpectedResponse(HostResponse { - hostname: ErrorDetails { - entity: "mx2.example.org".to_string(), - details: "DATA".to_string(), - }, - response: Response { - code: 450, - esc: [4, 3, 1], - message: "Can't accept mail at this moment".to_string(), - }, - })); - message.domains[0].changed = true; - - message.domains[1].status = Status::TemporaryFailure(Error::ConnectionError(ErrorDetails { - entity: "mx.domain.org".to_string(), - details: "Connection timeout".to_string(), - })); - message.domains[1].changed = true; - message.domains[1].notify = Schedule::later(Duration::from_secs(30)); - message.domains[1].notify.inner = 321; - message.domains[1].retry = Schedule::later(Duration::from_secs(62)); - message.domains[1].retry.inner = 678; - - // Save changes - message.save_changes().await; - assert!(message.serialize_changes().is_empty()); - assert_msg_eq( - &message, - &Message::from_path(message.path.clone()).await.unwrap(), - ); - - // Remove - message.remove().await; - assert!(!message.path.exists()); -} - -fn assert_msg_eq(msg: &Message, other: &Message) { - assert_eq!(msg.id, other.id); - assert_eq!(msg.created, other.created); - assert_eq!(msg.path, other.path); - assert_eq!(msg.return_path, other.return_path); - assert_eq!(msg.return_path_lcase, other.return_path_lcase); - assert_eq!(msg.return_path_domain, other.return_path_domain); - assert_eq!(msg.recipients, other.recipients); - assert_eq!(msg.domains.len(), other.domains.len()); - for (domain, other) in msg.domains.iter().zip(other.domains.iter()) { - assert_eq!(domain.domain, other.domain); - assert_eq!(domain.retry.inner, other.retry.inner); - assert_eq!(domain.notify.inner, other.notify.inner); - assert_eq!(domain.status, other.status); - assert_instant_eq(domain.expires, other.expires); - assert_instant_eq(domain.retry.due, other.retry.due); - assert_instant_eq(domain.notify.due, other.notify.due); - } - assert_eq!(msg.flags, other.flags); - assert_eq!(msg.env_id, other.env_id); - assert_eq!(msg.priority, other.priority); - assert_eq!(msg.size, other.size); -} - -fn assert_instant_eq(instant: Instant, other: Instant) { - let dur = if instant > other { - instant - other - } else { - other - instant - } - .as_secs(); - assert!(dur <= 1, "dur {dur}"); -} diff --git a/tests/src/smtp/reporting/analyze.rs b/tests/src/smtp/reporting/analyze.rs index fb17d4c0..2bb82779 100644 --- a/tests/src/smtp/reporting/analyze.rs +++ b/tests/src/smtp/reporting/analyze.rs @@ -78,7 +78,7 @@ async fn report_analyze() { "250", ) .await; - qr.assert_empty_queue(); + qr.assert_no_events(); ac += 1; } } diff --git a/tests/src/smtp/reporting/dmarc.rs b/tests/src/smtp/reporting/dmarc.rs index a7262b46..bb928c9f 100644 --- a/tests/src/smtp/reporting/dmarc.rs +++ b/tests/src/smtp/reporting/dmarc.rs @@ -32,11 +32,11 @@ use mail_auth::{ dmarc::Dmarc, report::{ActionDisposition, Disposition, DmarcResult, Record, Report}, }; +use store::write::QueueClass; use utils::config::if_block::IfBlock; use crate::smtp::{ - inbound::{sign::TextConfigContext, TestMessage, TestQueueEvent}, - make_temp_dir, + inbound::{sign::TextConfigContext, TestMessage}, session::VerifyResponse, ParseTestConfig, TestConfig, TestSMTP, }; @@ -58,7 +58,6 @@ async fn report_dmarc() { // Create scheduler let mut core = SMTP::test(); core.shared.signers = ConfigContext::new(&[]).parse_signatures().signers; - let temp_dir = make_temp_dir("smtp_report_dmarc_test", true); let config = &mut core.report.config; config.dmarc_aggregate.sign = "\"['rsa']\"".parse_if(); config.dmarc_aggregate.max_size = IfBlock::new(4096); @@ -113,20 +112,19 @@ async fn report_dmarc() { interval: AggregateFrequency::Weekly, })) .await; - assert_eq!(scheduler.reports.len(), 1); tokio::time::sleep(Duration::from_millis(200)).await; - let report_path; - match scheduler.reports.into_iter().next().unwrap() { - (ReportType::Dmarc(domain), ReportType::Dmarc(path)) => { - report_path = path.path.clone(); - core.generate_dmarc_report(domain, path); + let reports = qr.read_report_events().await; + assert_eq!(reports.len(), 1); + match reports.into_iter().next().unwrap() { + QueueClass::DmarcReportHeader(event) => { + core.generate_dmarc_report(event).await; } _ => unreachable!(), } // Expect report - let message = qr.expect_message().await(); - qr.assert_empty_queue(); + let message = qr.expect_message().await; + qr.assert_no_events(); assert_eq!(message.recipients.len(), 1); assert_eq!( message.recipients.last().unwrap().address, @@ -134,14 +132,15 @@ async fn report_dmarc() { ); assert_eq!(message.return_path, "reports@example.org"); message - .read_lines(&core).await + .read_lines(&qr) + .await .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com;") .assert_contains("To: ") .assert_contains("Report Domain: foobar.org") .assert_contains("Submitter: mx.example.org"); // Verify generated report - let report = Report::parse_rfc5322(message.read_message().as_bytes()).unwrap(); + let report = Report::parse_rfc5322(message.read_message(&qr).await.as_bytes()).unwrap(); assert_eq!(report.domain(), "foobar.org"); assert_eq!(report.email(), "reports@example.org"); assert_eq!(report.org_name(), "Foobar, Inc."); @@ -166,6 +165,5 @@ async fn report_dmarc() { panic!("unexpected ip {source_ip}"); } } - - assert!(!report_path.exists()); + qr.assert_report_is_empty().await; } diff --git a/tests/src/smtp/reporting/scheduler.rs b/tests/src/smtp/reporting/scheduler.rs index 402b08cb..caccbdea 100644 --- a/tests/src/smtp/reporting/scheduler.rs +++ b/tests/src/smtp/reporting/scheduler.rs @@ -29,10 +29,10 @@ use mail_auth::{ mta_sts::TlsRpt, report::{ActionDisposition, Alignment, Disposition, DmarcResult, PolicyPublished, Record}, }; -use tokio::fs; +use store::write::QueueClass; use utils::config::if_block::IfBlock; -use crate::smtp::{make_temp_dir, TestConfig}; +use crate::smtp::{TestConfig, TestSMTP}; use smtp::{ config::AggregateFrequency, core::SMTP, @@ -50,7 +50,7 @@ async fn report_scheduler() { // Create scheduler let mut core = SMTP::test(); - let temp_dir = make_temp_dir("smtp_report_scheduler_test", true); + let qr = core.init_test_queue("smtp_report_queue_test"); let config = &mut core.report.config; config.dmarc_aggregate.max_size = IfBlock::new(500); config.tls.max_size = IfBlock::new(550); @@ -143,55 +143,27 @@ async fn report_scheduler() { let mut total_tls = 0; let mut total_tls_policies = 0; let mut total_dmarc_policies = 0; - for report in scheduler.reports.values() { + let mut last_domain = String::new(); + for report in qr.read_report_events().await { match report { - ReportType::Dmarc(r) => { - assert!(r.size <= 550, "{}", r.size); - assert_eq!(fs::metadata(&r.path).await.unwrap().len() as usize, r.size); - assert_eq!(r.deliver_at, AggregateFrequency::Weekly); + QueueClass::DmarcReportHeader(event) => { total_dmarc_policies += 1; + assert_eq!(event.due - event.seq_id, 7 * 86400); } - ReportType::Tls(r) => { - total_tls += 1; - total_tls_policies += r.path.len(); - assert!(r.size <= 550); - assert_eq!(r.deliver_at, AggregateFrequency::Daily); - let mut sizes = 0; - for p in &r.path { - sizes += fs::metadata(&p.inner).await.unwrap().len() as usize; + QueueClass::TlsReportHeader(event) => { + if event.domain != last_domain { + last_domain = event.domain.clone(); + total_tls += 1; } - assert_eq!(r.size, sizes); + total_tls_policies += 1; + assert_eq!(event.due - event.seq_id, 86400); } + _ => unreachable!(), } } assert_eq!(total_tls, 1); assert_eq!(total_tls_policies, 3); assert_eq!(total_dmarc_policies, 2); - - // Verify deserialized report queue - let mut scheduler_deser = core.report.read_reports().await; - for (key, value) in scheduler.reports { - let a = Some(value); - let b = scheduler_deser.reports.remove(&key); - match (&a, &b) { - (Some(ReportType::Tls(a)), Some(ReportType::Tls(b))) => { - assert_eq!(a.created, b.created); - assert_eq!(a.size, b.size); - assert_eq!(a.deliver_at, b.deliver_at); - assert_eq!(a.path.len(), b.path.len()); - for p in &a.path { - assert!(b.path.contains(p)); - } - for p in &b.path { - assert!(a.path.contains(p)); - } - } - _ => { - assert_eq!(a, b, "failed for {key:?}"); - } - } - } - assert_eq!(scheduler.main.len(), scheduler_deser.main.len()); } #[test] diff --git a/tests/src/smtp/reporting/tls.rs b/tests/src/smtp/reporting/tls.rs index 3bd0246f..1e6e7f72 100644 --- a/tests/src/smtp/reporting/tls.rs +++ b/tests/src/smtp/reporting/tls.rs @@ -29,11 +29,11 @@ use mail_auth::{ mta_sts::TlsRpt, report::tlsrpt::{FailureDetails, PolicyType, ResultType, TlsReport}, }; +use store::write::QueueClass; use utils::config::if_block::IfBlock; use crate::smtp::{ - inbound::{sign::TextConfigContext, TestMessage, TestQueueEvent}, - make_temp_dir, + inbound::{sign::TextConfigContext, TestMessage}, session::VerifyResponse, ParseTestConfig, TestConfig, TestSMTP, }; @@ -45,7 +45,8 @@ use smtp::{ #[tokio::test] async fn report_tls() { - /*tracing::subscriber::set_global_default( + /*let disable = "true"; + tracing::subscriber::set_global_default( tracing_subscriber::FmtSubscriber::builder() .with_max_level(tracing::Level::DEBUG) .finish(), @@ -55,17 +56,13 @@ async fn report_tls() { // Create scheduler let mut core = SMTP::test(); core.shared.signers = ConfigContext::new(&[]).parse_signatures().signers; - let temp_dir = make_temp_dir("smtp_report_tls_test", true); let config = &mut core.report.config; - config.path = temp_dir.temp_dir.clone(); - config.hash = IfBlock::new(16); config.tls.sign = "\"['rsa']\"".parse_if(); - config.tls.max_size = IfBlock::new(4096); + config.tls.max_size = IfBlock::new(1532); config.submitter = IfBlock::new("mx.example.org".to_string()); config.tls.address = IfBlock::new("reports@example.org".to_string()); config.tls.org_name = IfBlock::new("Foobar, Inc.".to_string()); config.tls.contact_info = IfBlock::new("https://foobar.org/contact".to_string()); - let mut scheduler = Scheduler::default(); // Create temp dir for queue let mut qr = core.init_test_queue("smtp_report_tls_test"); @@ -76,18 +73,14 @@ async fn report_tls() { for _ in 0..2 { // Add two successful records - scheduler - .schedule_tls( - Box::new(TlsEvent { - domain: "foobar.org".to_string(), - policy: smtp::reporting::PolicyType::None, - failure: None, - tls_record: tls_record.clone(), - interval: AggregateFrequency::Daily, - }), - &core, - ) - .await; + core.schedule_tls(Box::new(TlsEvent { + domain: "foobar.org".to_string(), + policy: smtp::reporting::PolicyType::None, + failure: None, + tls_record: tls_record.clone(), + interval: AggregateFrequency::Daily, + })) + .await; } for (policy, rt) in [ @@ -107,52 +100,54 @@ async fn report_tls() { smtp::reporting::PolicyType::Sts(None), ResultType::StsPolicyInvalid, ), + ( + smtp::reporting::PolicyType::Sts(None), // Quota limited at 1532 bytes, this should not be included in the report. + ResultType::StsWebpkiInvalid, + ), ] { - scheduler - .schedule_tls( - Box::new(TlsEvent { - domain: "foobar.org".to_string(), - policy, - failure: FailureDetails::new(rt).into(), - tls_record: tls_record.clone(), - interval: AggregateFrequency::Daily, - }), - &core, - ) - .await; + core.schedule_tls(Box::new(TlsEvent { + domain: "foobar.org".to_string(), + policy, + failure: FailureDetails::new(rt).into(), + tls_record: tls_record.clone(), + interval: AggregateFrequency::Daily, + })) + .await; } // Wait for flush tokio::time::sleep(Duration::from_millis(200)).await; - - assert_eq!(scheduler.reports.len(), 1); - let mut report_path = Vec::new(); - match scheduler.reports.into_iter().next().unwrap() { - (ReportType::Tls(domain), ReportType::Tls(path)) => { - for p in &path.path { - report_path.push(p.inner.clone()); + let reports = qr.read_report_events().await; + assert_eq!(reports.len(), 3); + let mut tls_reports = Vec::with_capacity(3); + for report in reports { + match report { + QueueClass::TlsReportHeader(event) => { + tls_reports.push(event); } - core.generate_tls_report(domain, path); + _ => unreachable!(), } - _ => unreachable!(), } + core.generate_tls_report(tls_reports.first().unwrap().domain.clone(), tls_reports) + .await; // Expect report - let message = qr.expect_message().await(); + let message = qr.expect_message().await; assert_eq!( message.recipients.last().unwrap().address, "reports@foobar.org" ); assert_eq!(message.return_path, "reports@example.org"); message - .read_lines(&core).await + .read_lines(&qr) + .await .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com;") .assert_contains("To: ") .assert_contains("Report Domain: foobar.org") .assert_contains("Submitter: mx.example.org"); // Verify generated report - let report = TlsReport::parse_rfc5322(message.read_message().as_bytes()).unwrap(); + let report = TlsReport::parse_rfc5322(message.read_message(&qr).await.as_bytes()).unwrap(); assert_eq!(report.organization_name.unwrap(), "Foobar, Inc."); assert_eq!(report.contact_info.unwrap(), "https://foobar.org/contact"); assert_eq!(report.policies.len(), 3); @@ -204,53 +199,42 @@ async fn report_tls() { assert!(seen[1]); assert!(seen[2]); - for path in report_path { - assert!(!path.exists()); - } - // Schedule TLS reports to be delivered via https - let mut scheduler = Scheduler::default(); let tls_record = Arc::new(TlsRpt::parse(b"v=TLSRPTv1;rua=https://127.0.0.1/tls").unwrap()); for _ in 0..2 { // Add two successful records - scheduler - .schedule_tls( - Box::new(TlsEvent { - domain: "foobar.org".to_string(), - policy: smtp::reporting::PolicyType::None, - failure: None, - tls_record: tls_record.clone(), - interval: AggregateFrequency::Daily, - }), - &core, - ) - .await; + core.schedule_tls(Box::new(TlsEvent { + domain: "foobar.org".to_string(), + policy: smtp::reporting::PolicyType::None, + failure: None, + tls_record: tls_record.clone(), + interval: AggregateFrequency::Daily, + })) + .await; } - let mut report_path = Vec::new(); - match scheduler.reports.into_iter().next().unwrap() { - (ReportType::Tls(domain), ReportType::Tls(path)) => { - for p in &path.path { - report_path.push(p.inner.clone()); - } - core.generate_tls_report(domain, path); + let reports = qr.read_report_events().await; + assert_eq!(reports.len(), 1); + match reports.into_iter().next().unwrap() { + QueueClass::TlsReportHeader(event) => { + core.generate_tls_report(event.domain.clone(), vec![event]) + .await; } _ => unreachable!(), } tokio::time::sleep(Duration::from_millis(200)).await; // Uncompress report - let gz_report = TLS_HTTP_REPORT.lock(); - let mut file = GzDecoder::new(&gz_report[..]); - let mut buf = Vec::new(); - file.read_to_end(&mut buf).unwrap(); - let report = TlsReport::parse_json(&buf).unwrap(); - assert_eq!(report.organization_name.unwrap(), "Foobar, Inc."); - assert_eq!(report.contact_info.unwrap(), "https://foobar.org/contact"); - assert_eq!(report.policies.len(), 1); - - for path in report_path { - assert!(!path.exists()); + { + let gz_report = TLS_HTTP_REPORT.lock(); + let mut file = GzDecoder::new(&gz_report[..]); + let mut buf = Vec::new(); + file.read_to_end(&mut buf).unwrap(); + let report = TlsReport::parse_json(&buf).unwrap(); + assert_eq!(report.organization_name.unwrap(), "Foobar, Inc."); + assert_eq!(report.contact_info.unwrap(), "https://foobar.org/contact"); + assert_eq!(report.policies.len(), 1); } + qr.assert_report_is_empty().await; } diff --git a/tests/src/smtp/session.rs b/tests/src/smtp/session.rs index 17a19a6c..d656492d 100644 --- a/tests/src/smtp/session.rs +++ b/tests/src/smtp/session.rs @@ -372,7 +372,6 @@ impl TestServerInstance for ServerInstance { limiter: ConcurrencyLimiter::new(100), shutdown_rx, proxy_networks: vec![], - blocked_ips: Arc::new(Default::default()), } } } diff --git a/tests/src/store/lookup.rs b/tests/src/store/lookup.rs index e11a04b4..0d97ccf1 100644 --- a/tests/src/store/lookup.rs +++ b/tests/src/store/lookup.rs @@ -21,8 +21,10 @@ * for more details. */ +use std::time::Duration; + use store::{config::ConfigStore, LookupStore}; -use utils::config::Config; +use utils::config::{Config, Rate}; use crate::store::{TempDir, CONFIG}; @@ -32,9 +34,10 @@ pub async fn lookup_tests() { let config = Config::new(&CONFIG.replace("{TMP}", temp_dir.path.as_path().to_str().unwrap())).unwrap(); let stores = config.parse_stores().await.unwrap(); - - let todo = "test expiry counter + ratelimit"; - let todo = "use lookup ratelimit everywhere"; + let rate = Rate { + requests: 1, + period: Duration::from_secs(1), + }; for (store_id, store) in stores.lookup_stores { println!("Testing lookup store {}...", store_id); @@ -83,5 +86,38 @@ pub async fn lookup_tests() { assert_eq!(1, store.counter_get(key.clone()).await.unwrap()); store.counter_incr(key.clone(), 2, None).await.unwrap(); assert_eq!(3, store.counter_get(key.clone()).await.unwrap()); + store.counter_incr(key.clone(), -3, None).await.unwrap(); + assert_eq!(0, store.counter_get(key.clone()).await.unwrap()); + + // Test counter expiry + let key = "fgh".as_bytes().to_vec(); + store.counter_incr(key.clone(), 1, 1.into()).await.unwrap(); + assert_eq!(1, store.counter_get(key.clone()).await.unwrap()); + tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + store.purge_expired().await.unwrap(); + assert_eq!(0, store.counter_get(key.clone()).await.unwrap()); + + // Test rate limiter + assert!(store + .is_rate_allowed("rate".as_bytes(), &rate, false) + .await + .unwrap() + .is_none()); + assert!(store + .is_rate_allowed("rate".as_bytes(), &rate, false) + .await + .unwrap() + .is_some()); + tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + assert!(store + .is_rate_allowed("rate".as_bytes(), &rate, false) + .await + .unwrap() + .is_none()); + tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + store.purge_expired().await.unwrap(); + if let LookupStore::Store(store) = &store { + store.assert_is_empty(store.clone().into()).await; + } } }