diff --git a/Cargo.lock b/Cargo.lock index 8569b69c..307b9c91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,6 +23,12 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + [[package]] name = "aead" version = "0.5.2" @@ -180,9 +186,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" [[package]] name = "arbitrary" @@ -284,7 +290,7 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", "synstructure 0.13.1", ] @@ -307,7 +313,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -353,7 +359,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -375,7 +381,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -386,7 +392,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -671,7 +677,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -691,7 +697,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -709,7 +715,7 @@ dependencies = [ "regex", "rustc-hash 2.1.0", "shlex", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -891,7 +897,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -943,9 +949,9 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.20.0" +version = "1.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b37c88a63ffd85d15b406896cc343916d7cf57838a847b3a6f2ca5d39a5695a" +checksum = "ef657dfab802224e671f5818e9a4935f9b1957ed18e58292690cc39e7a4092a3" [[package]] name = "byteorder" @@ -1010,9 +1016,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.4" +version = "1.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9157bbaa6b165880c27a4293a474c91cdcf265cc68cc829bf10be0964a391caf" +checksum = "c31a0499c1dc64f458ad13872de75c0eb7e3fdb0e67964610c914b034fc5956e" dependencies = [ "jobserver", "libc", @@ -1146,7 +1152,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -1346,6 +1352,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" +dependencies = [ + "memchr", +] + [[package]] name = "cpufeatures" version = "0.2.16" @@ -1534,7 +1549,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -1582,7 +1597,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -1604,9 +1619,15 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core 0.20.10", "quote", - "syn 2.0.90", + "syn 2.0.91", ] +[[package]] +name = "dary_heap" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728" + [[package]] name = "dashmap" version = "6.1.0" @@ -1661,9 +1682,9 @@ dependencies = [ [[package]] name = "deadpool-postgres" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ab8a4ea925ce79678034870834602a2980f4b88c09e97feb266496dbb4493d2" +checksum = "3d697d376cbfa018c23eb4caab1fd1883dd9c906a8c034e8d9a3cb06a7e0bef9" dependencies = [ "async-trait", "deadpool 0.12.1", @@ -1756,38 +1777,7 @@ checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", -] - -[[package]] -name = "derive_builder" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" -dependencies = [ - "darling 0.20.10", - "proc-macro2", - "quote", - "syn 2.0.90", -] - -[[package]] -name = "derive_builder_macro" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" -dependencies = [ - "derive_builder_core", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -1889,7 +1879,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -2108,7 +2098,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -2150,7 +2140,7 @@ version = "0.10.7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -2244,9 +2234,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "foldhash" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" +checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" [[package]] name = "foreign-types" @@ -2326,7 +2316,7 @@ checksum = "afc84a5ff0dba78222551017f5625f3365aa09551c78cfaa44136fc6818c2611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", "try_map", ] @@ -2369,7 +2359,7 @@ checksum = "e99b8b3c28ae0e84b604c75f721c21dc77afb3706076af5e8216d15fd1deaae3" dependencies = [ "frunk_proc_macro_helpers", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -2381,7 +2371,7 @@ dependencies = [ "frunk_core", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -2393,7 +2383,7 @@ dependencies = [ "frunk_core", "frunk_proc_macro_helpers", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -2473,7 +2463,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -2997,9 +2987,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.4" +version = "0.27.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6884a48c6826ec44f524c7456b163cebe9e55a18d7b5e307cb4f100371cc767" +checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", "http 1.2.0", @@ -3183,7 +3173,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -3267,6 +3257,29 @@ dependencies = [ "trc", ] +[[package]] +name = "include-flate" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df49c16750695486c1f34de05da5b7438096156466e7f76c38fcdf285cf0113e" +dependencies = [ + "include-flate-codegen", + "lazy_static", + "libflate", +] + +[[package]] +name = "include-flate-codegen" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c5b246c6261be723b85c61ecf87804e8ea4a35cb68be0ff282ed84b95ffe7d7" +dependencies = [ + "libflate", + "proc-macro2", + "quote", + "syn 2.0.91", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -3432,17 +3445,26 @@ dependencies = [ ] [[package]] -name = "jieba-rs" -version = "0.7.0" +name = "jieba-macros" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1e2b0210dc78b49337af9e49d7ae41a39dceac6e5985613f1cf7763e2f76a25" +checksum = "7c676b32a471d3cfae8dac2ad2f8334cd52e53377733cca8c1fb0a5062fec192" +dependencies = [ + "phf_codegen", +] + +[[package]] +name = "jieba-rs" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a77d0ae8831f870c4f6ffce310f708b5273ea2e7a88e6af770a10d1b4876311" dependencies = [ "cedarwood", - "derive_builder", "fxhash", + "include-flate", + "jieba-macros", "lazy_static", "phf", - "phf_codegen", "regex", ] @@ -3675,9 +3697,33 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.168" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" + +[[package]] +name = "libflate" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45d9dfdc14ea4ef0900c1cddbc8dcd553fbaacd8a4a282cf4018ae9dd04fb21e" +dependencies = [ + "adler32", + "core2", + "crc32fast", + "dary_heap", + "libflate_lz77", +] + +[[package]] +name = "libflate_lz77" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e0d73b369f386f1c44abd9c570d5318f55ccde816ff4b562fa452e5182863d" +dependencies = [ + "core2", + "hashbrown 0.14.5", + "rle-decode-fast", +] [[package]] name = "libloading" @@ -3707,14 +3753,13 @@ dependencies = [ [[package]] name = "librocksdb-sys" -version = "0.16.0+8.10.0" +version = "0.17.1+9.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce3d60bc059831dc1c83903fb45c103f75db65c5a7bf22272764d9cc683e348c" +checksum = "2b7869a512ae9982f4d46ba482c2a304f1efd80c6412a3d4bf57bb79a619679f" dependencies = [ "bindgen 0.69.5", "bzip2-sys", "cc", - "glob", "libc", "libz-sys", "lz4-sys", @@ -3952,7 +3997,7 @@ checksum = "5cf92c10c7e361d6b99666ec1c6f9805b0bea2c3bd8c78dc6fe98ac5bd78db11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -4069,7 +4114,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", "termcolor", "thiserror 1.0.69", ] @@ -4298,9 +4343,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.36.5" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "memchr", ] @@ -4358,7 +4403,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -4644,7 +4689,7 @@ dependencies = [ "phf_shared 0.11.2", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -4682,7 +4727,7 @@ checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -4818,7 +4863,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -4882,7 +4927,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -4900,7 +4945,7 @@ version = "0.10.7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -4937,7 +4982,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -4952,9 +4997,9 @@ dependencies = [ [[package]] name = "psl" -version = "2.1.72" +version = "2.1.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "699f79b15ea465a0a1ef3c7f0ef2b6b6782a2f720f3587428f0eea37f5c1d3b9" +checksum = "a62360a10ed773da9a36aa1b5817c5f505b1f35728e5ea6e4e524a13fc10778c" dependencies = [ "psl-types", ] @@ -5057,7 +5102,7 @@ dependencies = [ "rustc-hash 2.1.0", "rustls 0.23.20", "socket2", - "thiserror 2.0.8", + "thiserror 2.0.9", "tokio", "tracing", ] @@ -5076,7 +5121,7 @@ dependencies = [ "rustls 0.23.20", "rustls-pki-types", "slab", - "thiserror 2.0.8", + "thiserror 2.0.9", "tinyvec", "tracing", "web-time", @@ -5098,9 +5143,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -5293,9 +5338,9 @@ dependencies = [ [[package]] name = "rcgen" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54077e1872c46788540de1ea3d7f4ccb1983d12f9aa909b234468676c1a36779" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" dependencies = [ "pem", "ring 0.17.8", @@ -5306,9 +5351,9 @@ dependencies = [ [[package]] name = "redis" -version = "0.26.1" +version = "0.27.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e902a69d09078829137b4a5d9d082e0490393537badd7c91a3d69d14639e115f" +checksum = "09d8f99a4090c89cc489a94833c901ead69bfbf3877b4867d5482e321ee875bc" dependencies = [ "arc-swap", "async-trait", @@ -5317,6 +5362,7 @@ dependencies = [ "crc16", "futures", "futures-util", + "itertools 0.13.0", "itoa", "log", "num-bigint", @@ -5455,7 +5501,7 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "hyper 1.5.2", - "hyper-rustls 0.27.4", + "hyper-rustls 0.27.5", "hyper-util", "ipnet", "js-sys", @@ -5583,6 +5629,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "rle-decode-fast" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" + [[package]] name = "roaring" version = "0.10.9" @@ -5595,9 +5647,9 @@ dependencies = [ [[package]] name = "rocksdb" -version = "0.22.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bd13e55d6d7b8cd0ea569161127567cd587676c99f4472f779a0279aa60a7a7" +checksum = "26ec73b20525cb235bad420f911473b69f9fe27cc856c5461bccd7e4af037f43" dependencies = [ "libc", "librocksdb-sys", @@ -6139,14 +6191,14 @@ checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] name = "serde_json" -version = "1.0.133" +version = "1.0.134" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" dependencies = [ "indexmap 2.7.0", "itoa", @@ -6222,7 +6274,7 @@ checksum = "5d69265a08751de7844521fd15003ae0a888e035773ba05695c5c759a6f89eef" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -6658,9 +6710,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.90" +version = "2.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" +checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035" dependencies = [ "proc-macro2", "quote", @@ -6702,7 +6754,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -6817,11 +6869,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.8" +version = "2.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f5383f3e0071702bf93ab5ee99b52d26936be9dedd9413067cbdcddcb6141a" +checksum = "f072643fd0190df67a8bab670c20ef5d8737177d6ac6b2e9a236cb096206b2cc" dependencies = [ - "thiserror-impl 2.0.8", + "thiserror-impl 2.0.9", ] [[package]] @@ -6832,18 +6884,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] name = "thiserror-impl" -version = "2.0.8" +version = "2.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f357fcec90b3caef6623a099691be676d033b40a058ac95d2a6ade6fa0c943" +checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -6899,9 +6951,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" dependencies = [ "tinyvec_macros", ] @@ -6939,7 +6991,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -7178,7 +7230,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -7308,9 +7360,9 @@ checksum = "e36a83ea2b3c704935a01b4642946aadd445cea40b10935e3f8bd8052b8193d6" [[package]] name = "unicase" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" [[package]] name = "unicode-bidi" @@ -7457,7 +7509,7 @@ dependencies = [ "privdrop", "psl", "rand 0.8.5", - "rcgen 0.13.1", + "rcgen 0.13.2", "regex", "reqwest 0.12.9", "ring 0.17.8", @@ -7566,7 +7618,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", "wasm-bindgen-shared", ] @@ -7601,7 +7653,7 @@ checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -8018,9 +8070,9 @@ checksum = "ea8b391c9a790b496184c29f7f93b9ed5b16abb306c05415b68bcc16e4d06432" [[package]] name = "xxhash-rust" -version = "0.8.12" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a5cbf750400958819fb6178eaa83bee5cd9c29a26a40cc241df8c70fdd46984" +checksum = "d7d48f1b18be023c95e7b75f481cac649d74be7c507ff4a407c55cfb957f7934" [[package]] name = "yasna" @@ -8051,7 +8103,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", "synstructure 0.13.1", ] @@ -8073,7 +8125,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -8093,7 +8145,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", "synstructure 0.13.1", ] @@ -8114,7 +8166,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -8136,7 +8188,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -8161,7 +8213,7 @@ dependencies = [ "pbkdf2", "rand 0.8.5", "sha1", - "thiserror 2.0.8", + "thiserror 2.0.9", "time", "zeroize", "zopfli", diff --git a/crates/common/src/ipc.rs b/crates/common/src/ipc.rs index b21e4b5a..d39f95cc 100644 --- a/crates/common/src/ipc.rs +++ b/crates/common/src/ipc.rs @@ -67,8 +67,14 @@ pub enum HousekeeperEvent { pub enum PurgeType { Data(Store), - Blobs { store: Store, blob_store: BlobStore }, - Lookup(InMemoryStore), + Blobs { + store: Store, + blob_store: BlobStore, + }, + Lookup { + store: InMemoryStore, + prefix: Option>, + }, Account(Option), } diff --git a/crates/jmap/src/api/management/principal.rs b/crates/jmap/src/api/management/principal.rs index f1bc2877..f8d87d4b 100644 --- a/crates/jmap/src/api/management/principal.rs +++ b/crates/jmap/src/api/management/principal.rs @@ -6,7 +6,7 @@ use std::sync::{atomic::Ordering, Arc}; -use common::{auth::AccessToken, Server}; +use common::{auth::AccessToken, Server, KV_BAYES_MODEL_USER}; use directory::{ backend::internal::{ lookup::DirectoryStore, @@ -355,6 +355,23 @@ impl PrincipalManager for Server { self.core.storage.fts.remove_all(account_id).await?; } + // Delete bayes model + if self + .core + .spam + .bayes + .as_ref() + .map_or(false, |c| c.account_classify) + { + let mut key = Vec::with_capacity(std::mem::size_of::() + 1); + key.push(KV_BAYES_MODEL_USER); + key.extend_from_slice(&account_id.to_be_bytes()); + + if let Err(err) = self.in_memory_store().key_delete_prefix(&key).await { + trc::error!(err.details("Failed to delete user bayes model")); + } + } + // Remove entries from cache self.inner .data diff --git a/crates/jmap/src/api/management/stores.rs b/crates/jmap/src/api/management/stores.rs index 6e2a5648..6224bcb0 100644 --- a/crates/jmap/src/api/management/stores.rs +++ b/crates/jmap/src/api/management/stores.rs @@ -9,7 +9,7 @@ use common::{ auth::AccessToken, ipc::{HousekeeperEvent, PurgeType}, manager::webadmin::Resource, - Server, + *, }; use directory::{ backend::internal::manage::{self, ManageDirectory}, @@ -108,7 +108,7 @@ impl ManageStore for Server { // Validate the access token access_token.assert_has_permission(Permission::PurgeDataStore)?; - let store = if let Some(id) = id { + let store = if let Some(id) = id.filter(|id| *id != "default") { if let Some(store) = self.core.storage.stores.get(id) { store.clone() } else { @@ -121,11 +121,11 @@ impl ManageStore for Server { self.housekeeper_request(HousekeeperEvent::Purge(PurgeType::Data(store))) .await } - (Some("purge"), Some("lookup"), id, &Method::GET) => { + (Some("purge"), Some("in-memory"), id, &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::PurgeInMemoryStore)?; - let store = if let Some(id) = id { + let store = if let Some(id) = id.filter(|id| *id != "default") { if let Some(store) = self.core.storage.lookups.get(id) { store.clone() } else { @@ -135,8 +135,57 @@ impl ManageStore for Server { self.core.storage.lookup.clone() }; - self.housekeeper_request(HousekeeperEvent::Purge(PurgeType::Lookup(store))) - .await + let prefix = match path.get(4).copied() { + Some("acme") => vec![KV_ACME].into(), + Some("oauth") => vec![KV_OAUTH].into(), + Some("rate-rcpt") => vec![KV_RATE_LIMIT_RCPT].into(), + Some("rate-scan") => vec![KV_RATE_LIMIT_SCAN].into(), + Some("rate-loiter") => vec![KV_RATE_LIMIT_LOITER].into(), + Some("rate-auth") => vec![KV_RATE_LIMIT_AUTH].into(), + Some("rate-hash") => vec![KV_RATE_LIMIT_HASH].into(), + Some("rate-contact") => vec![KV_RATE_LIMIT_CONTACT].into(), + Some("rate-jmap") => vec![KV_RATE_LIMIT_JMAP].into(), + Some("rate-jmap-auth") => vec![KV_RATE_LIMIT_JMAP_AUTH].into(), + Some("rate-http-anonymous") => vec![KV_RATE_LIMIT_HTTP_ANONYM].into(), + Some("rate-imap") => vec![KV_RATE_LIMIT_IMAP].into(), + Some("reputation-ip") => vec![KV_REPUTATION_IP].into(), + Some("reputation-from") => vec![KV_REPUTATION_FROM].into(), + Some("reputation-domain") => vec![KV_REPUTATION_DOMAIN].into(), + Some("reputation-asn") => vec![KV_REPUTATION_ASN].into(), + Some("greylist") => vec![KV_GREYLIST].into(), + Some("bayes-account") => { + if let Some(account) = path.get(5).copied() { + let account_id = self + .core + .storage + .data + .get_principal_id(decode_path_element(account).as_ref()) + .await? + .ok_or_else(|| trc::ManageEvent::NotFound.into_err())?; + + let mut key = Vec::with_capacity(std::mem::size_of::() + 1); + key.push(KV_BAYES_MODEL_USER); + key.extend_from_slice(&account_id.to_be_bytes()); + key.into() + } else { + vec![KV_BAYES_MODEL_USER].into() + } + } + Some("bayes-global") => vec![KV_BAYES_MODEL_GLOBAL].into(), + Some("trusted-reply") => vec![KV_TRUSTED_REPLY].into(), + Some("lock-purge-account") => vec![KV_LOCK_PURGE_ACCOUNT].into(), + Some("lock-queue-message") => vec![KV_LOCK_QUEUE_MESSAGE].into(), + Some("lock-queue-report") => vec![KV_LOCK_QUEUE_REPORT].into(), + Some("lock-email-task") => vec![KV_LOCK_EMAIL_TASK].into(), + Some("lock-housekeeper") => vec![KV_LOCK_HOUSEKEEPER].into(), + _ => None, + }; + + self.housekeeper_request(HousekeeperEvent::Purge(PurgeType::Lookup { + store, + prefix, + })) + .await } (Some("purge"), Some("account"), id, &Method::GET) => { // Validate the access token diff --git a/crates/jmap/src/services/housekeeper.rs b/crates/jmap/src/services/housekeeper.rs index 7a153a27..d0af0ef3 100644 --- a/crates/jmap/src/services/housekeeper.rs +++ b/crates/jmap/src/services/housekeeper.rs @@ -375,7 +375,10 @@ pub fn spawn_housekeeper(inner: Arc, mut rx: mpsc::Receiver { - PurgeType::Lookup(in_memory_store) + PurgeType::Lookup { + store: in_memory_store, + prefix: None, + } } }, idx as u32, @@ -638,7 +641,7 @@ impl Purge for Server { .collect::>() .into(), ), - PurgeType::Lookup(_) => ( + PurgeType::Lookup { prefix: None, .. } => ( "in-memory", [2u8] .into_iter() @@ -646,6 +649,7 @@ impl Purge for Server { .collect::>() .into(), ), + PurgeType::Lookup { .. } => ("in-memory-prefix", None), PurgeType::Account(_) => ("account", None), }; if let Some(lock_name) = &lock_name { @@ -719,8 +723,14 @@ impl Purge for Server { trc::error!(err.details("Failed to purge blob store")); } } - PurgeType::Lookup(store) => { - if let Err(err) = store.purge_in_memory_store().await { + PurgeType::Lookup { store, prefix } => { + if let Some(prefix) = prefix { + if let Err(err) = store.key_delete_prefix(&prefix).await { + trc::error!(err + .details("Failed to delete key prefix") + .ctx(trc::Key::Key, prefix)); + } + } else if let Err(err) = store.purge_in_memory_store().await { trc::error!(err.details("Failed to purge lookup store")); } } diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 2444af73..a14df1b7 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -8,7 +8,7 @@ resolver = "2" utils = { path = "../utils" } nlp = { path = "../nlp" } trc = { path = "../trc" } -rocksdb = { version = "0.22", optional = true, features = ["multi-threaded-cf"] } +rocksdb = { version = "0.23", optional = true, features = ["multi-threaded-cf"] } foundationdb = { version = "0.9.0", features = ["embedded-fdb-include", "fdb-7_1"], optional = true } rusqlite = { version = "0.32", features = ["bundled"], optional = true } rust-s3 = { version = "=0.35.0-alpha.2", default-features = false, features = ["tokio-rustls-tls", "no-verify-ssl"], optional = true } @@ -44,7 +44,7 @@ serde_json = {version = "1.0.64", optional = true } regex = "1.7.0" flate2 = "1.0" async-trait = "0.1.68" -redis = { version = "0.26", features = [ "tokio-comp", "tokio-rustls-comp", "tls-rustls-insecure", "tls-rustls-webpki-roots", "cluster-async"], optional = true } +redis = { version = "0.27", features = [ "tokio-comp", "tokio-rustls-comp", "tls-rustls-insecure", "tls-rustls-webpki-roots", "cluster-async"], optional = true } deadpool = { version = "0.12", features = ["managed"], optional = true } bincode = "1.3.3" arc-swap = "1.6.0" diff --git a/crates/store/src/backend/redis/lookup.rs b/crates/store/src/backend/redis/lookup.rs index c541d8bb..d60b4fda 100644 --- a/crates/store/src/backend/redis/lookup.rs +++ b/crates/store/src/backend/redis/lookup.rs @@ -11,12 +11,7 @@ use crate::Deserialize; use super::{into_error, RedisPool, RedisStore}; impl RedisStore { - pub async fn key_set( - &self, - key: Vec, - value: Vec, - expires: Option, - ) -> trc::Result<()> { + pub async fn key_set(&self, key: &[u8], value: &[u8], expires: Option) -> trc::Result<()> { match &self.pool { RedisPool::Single(pool) => { self.key_set_( @@ -39,12 +34,7 @@ impl RedisStore { } } - pub async fn key_incr( - &self, - key: Vec, - value: i64, - expires: Option, - ) -> trc::Result { + pub async fn key_incr(&self, key: &[u8], value: i64, expires: Option) -> trc::Result { match &self.pool { RedisPool::Single(pool) => { self.key_incr_( @@ -67,7 +57,7 @@ impl RedisStore { } } - pub async fn key_delete(&self, key: Vec) -> trc::Result<()> { + pub async fn key_delete(&self, key: &[u8]) -> trc::Result<()> { match &self.pool { RedisPool::Single(pool) => { self.key_delete_(pool.get().await.map_err(into_error)?.as_mut(), key) @@ -80,9 +70,22 @@ impl RedisStore { } } + pub async fn key_delete_prefix(&self, prefix: &[u8]) -> trc::Result<()> { + match &self.pool { + RedisPool::Single(pool) => { + self.key_delete_prefix_(pool.get().await.map_err(into_error)?.as_mut(), prefix) + .await + } + RedisPool::Cluster(pool) => { + self.key_delete_prefix_(pool.get().await.map_err(into_error)?.as_mut(), prefix) + .await + } + } + } + pub async fn key_get( &self, - key: Vec, + key: &[u8], ) -> trc::Result> { match &self.pool { RedisPool::Single(pool) => { @@ -96,7 +99,7 @@ impl RedisStore { } } - pub async fn counter_get(&self, key: Vec) -> trc::Result { + pub async fn counter_get(&self, key: &[u8]) -> trc::Result { match &self.pool { RedisPool::Single(pool) => { self.counter_get_(pool.get().await.map_err(into_error)?.as_mut(), key) @@ -109,7 +112,7 @@ impl RedisStore { } } - pub async fn key_exists(&self, key: Vec) -> trc::Result { + pub async fn key_exists(&self, key: &[u8]) -> trc::Result { match &self.pool { RedisPool::Single(pool) => { self.key_exists_(pool.get().await.map_err(into_error)?.as_mut(), key) @@ -125,7 +128,7 @@ impl RedisStore { async fn key_get_( &self, conn: &mut impl AsyncCommands, - key: Vec, + key: &[u8], ) -> trc::Result> { if let Some(value) = redis::cmd("GET") .arg(key) @@ -139,7 +142,7 @@ impl RedisStore { } } - async fn counter_get_(&self, conn: &mut impl AsyncCommands, key: Vec) -> trc::Result { + async fn counter_get_(&self, conn: &mut impl AsyncCommands, key: &[u8]) -> trc::Result { redis::cmd("GET") .arg(key) .query_async::>(conn) @@ -148,15 +151,15 @@ impl RedisStore { .map_err(into_error) } - async fn key_exists_(&self, conn: &mut impl AsyncCommands, key: Vec) -> trc::Result { + async fn key_exists_(&self, conn: &mut impl AsyncCommands, key: &[u8]) -> trc::Result { conn.exists(key).await.map_err(into_error) } async fn key_set_( &self, conn: &mut impl AsyncCommands, - key: Vec, - value: Vec, + key: &[u8], + value: &[u8], expires: Option, ) -> trc::Result<()> { if let Some(expires) = expires { @@ -169,26 +172,59 @@ impl RedisStore { async fn key_incr_( &self, conn: &mut impl AsyncCommands, - key: Vec, + key: &[u8], value: i64, expires: Option, ) -> trc::Result { if let Some(expires) = expires { redis::pipe() .atomic() - .incr(&key, value) - .expire(&key, expires as i64) + .incr(key, value) + .expire(key, expires as i64) .ignore() .query_async::>(conn) .await .map_err(into_error) .map(|v| v.first().copied().unwrap_or(0)) } else { - conn.incr(&key, value).await.map_err(into_error) + conn.incr(key, value).await.map_err(into_error) } } - async fn key_delete_(&self, conn: &mut impl AsyncCommands, key: Vec) -> trc::Result<()> { + async fn key_delete_(&self, conn: &mut impl AsyncCommands, key: &[u8]) -> trc::Result<()> { conn.del(key).await.map_err(into_error) } + + async fn key_delete_prefix_( + &self, + conn: &mut impl AsyncCommands, + prefix: &[u8], + ) -> trc::Result<()> { + let mut pattern = Vec::with_capacity(prefix.len() + 1); + pattern.extend_from_slice(prefix); + pattern.push(b'*'); + + let mut cursor = 0; + loop { + let (new_cursor, keys): (u64, Vec>) = redis::cmd("SCAN") + .cursor_arg(cursor) + .arg("MATCH") + .arg(&pattern) + .arg("COUNT") + .arg(100) + .query_async(conn) + .await + .map_err(into_error)?; + + if !keys.is_empty() { + conn.del::<_, ()>(&keys).await.map_err(into_error)?; + } + + if new_cursor != 0 { + cursor = new_cursor; + } else { + return Ok(()); + } + } + } } diff --git a/crates/store/src/backend/rocksdb/write.rs b/crates/store/src/backend/rocksdb/write.rs index 8c060ae2..64ce46a2 100644 --- a/crates/store/src/backend/rocksdb/write.rs +++ b/crates/store/src/backend/rocksdb/write.rs @@ -70,30 +70,13 @@ impl RocksDbStore { pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> { let db = self.db.clone(); self.spawn_worker(move || { - let cf = db - .cf_handle(std::str::from_utf8(&[from.subspace()]).unwrap()) - .unwrap(); - - // TODO use delete_range when implemented (see https://github.com/rust-rocksdb/rust-rocksdb/issues/839) - let from = from.serialize(0); - let to = to.serialize(0); - let mut delete_keys = Vec::new(); - let it_mode = IteratorMode::From(&from, Direction::Forward); - - for row in db.iterator_cf(&cf, it_mode) { - let (key, _) = row.map_err(into_error)?; - - if key.as_ref() < from.as_slice() || key.as_ref() >= to.as_slice() { - break; - } - delete_keys.push(key); - } - - for k in delete_keys { - db.delete_cf(&cf, &k).map_err(into_error)?; - } - - Ok(()) + db.delete_range_cf( + &db.cf_handle(std::str::from_utf8(&[from.subspace()]).unwrap()) + .unwrap(), + from.serialize(0), + to.serialize(0), + ) + .map_err(into_error) }) .await } diff --git a/crates/store/src/config.rs b/crates/store/src/config.rs index 15e1cf3b..7be6fc8a 100644 --- a/crates/store/src/config.rs +++ b/crates/store/src/config.rs @@ -323,7 +323,9 @@ impl Stores { } } for (store_id, store) in &self.in_memory_stores { - if matches!(store, InMemoryStore::Store(_)) { + if matches!(store, InMemoryStore::Store(_)) + && config.is_active_in_memory_store(store_id) + { self.purge_schedules.push(PurgeSchedule { cron: config .property_or_default::( @@ -342,6 +344,7 @@ impl Stores { #[allow(dead_code)] trait IsActiveStore { fn is_active_store(&self, id: &str) -> bool; + fn is_active_in_memory_store(&self, id: &str) -> bool; } impl IsActiveStore for Config { @@ -363,4 +366,9 @@ impl IsActiveStore for Config { false } + + fn is_active_in_memory_store(&self, id: &str) -> bool { + self.value("storage.lookup") + .map_or(false, |store_id| store_id == id) + } } diff --git a/crates/store/src/dispatch/lookup.rs b/crates/store/src/dispatch/lookup.rs index a39b9d72..c0dc89e2 100644 --- a/crates/store/src/dispatch/lookup.rs +++ b/crates/store/src/dispatch/lookup.rs @@ -43,7 +43,7 @@ impl InMemoryStore { store.write(batch.build()).await.map(|_| ()) } #[cfg(feature = "redis")] - InMemoryStore::Redis(store) => store.key_set(kv.key, kv.value, kv.expires).await, + InMemoryStore::Redis(store) => store.key_set(&kv.key, &kv.value, kv.expires).await, InMemoryStore::Static(_) | InMemoryStore::Http(_) => { Err(trc::StoreEvent::NotSupported.into_err()) } @@ -80,7 +80,7 @@ impl InMemoryStore { .and_then(|r| r.last_counter_id()) } #[cfg(feature = "redis")] - InMemoryStore::Redis(store) => store.key_incr(kv.key, kv.value, kv.expires).await, + InMemoryStore::Redis(store) => store.key_incr(&kv.key, kv.value, kv.expires).await, InMemoryStore::Static(_) | InMemoryStore::Http(_) => { Err(trc::StoreEvent::NotSupported.into_err()) } @@ -88,18 +88,18 @@ impl InMemoryStore { .caused_by(trc::location!()) } - pub async fn key_delete(&self, key: Vec) -> trc::Result<()> { + pub async fn key_delete(&self, key: impl Into>) -> trc::Result<()> { match self { InMemoryStore::Store(store) => { let mut batch = BatchBuilder::new(); batch.ops.push(Operation::Value { - class: ValueClass::Lookup(LookupClass::Key(key)), + class: ValueClass::Lookup(LookupClass::Key(key.into().into_bytes())), op: ValueOp::Clear, }); store.write(batch.build()).await.map(|_| ()) } #[cfg(feature = "redis")] - InMemoryStore::Redis(store) => store.key_delete(key).await, + InMemoryStore::Redis(store) => store.key_delete(key.into().as_bytes()).await, InMemoryStore::Static(_) | InMemoryStore::Http(_) => { Err(trc::StoreEvent::NotSupported.into_err()) } @@ -107,18 +107,55 @@ impl InMemoryStore { .caused_by(trc::location!()) } - pub async fn counter_delete(&self, key: Vec) -> trc::Result<()> { + pub async fn counter_delete(&self, key: impl Into>) -> trc::Result<()> { match self { InMemoryStore::Store(store) => { let mut batch = BatchBuilder::new(); batch.ops.push(Operation::Value { - class: ValueClass::Lookup(LookupClass::Counter(key)), + class: ValueClass::Lookup(LookupClass::Counter(key.into().into_bytes())), op: ValueOp::Clear, }); store.write(batch.build()).await.map(|_| ()) } #[cfg(feature = "redis")] - InMemoryStore::Redis(store) => store.key_delete(key).await, + InMemoryStore::Redis(store) => store.key_delete(key.into().as_bytes()).await, + InMemoryStore::Static(_) | InMemoryStore::Http(_) => { + Err(trc::StoreEvent::NotSupported.into_err()) + } + } + .caused_by(trc::location!()) + } + + pub async fn key_delete_prefix(&self, prefix: &[u8]) -> trc::Result<()> { + match self { + InMemoryStore::Store(store) => { + if prefix.is_empty() { + return Ok(()); + } + + let from_range = prefix.to_vec(); + let mut to_range = Vec::with_capacity(prefix.len() + 3); + to_range.extend_from_slice(prefix); + to_range.extend_from_slice([u8::MAX, u8::MAX, u8::MAX].as_ref()); + + store + .delete_range( + ValueKey::from(ValueClass::Lookup(LookupClass::Counter( + from_range.clone(), + ))), + ValueKey::from(ValueClass::Lookup(LookupClass::Counter(to_range.clone()))), + ) + .await?; + + store + .delete_range( + ValueKey::from(ValueClass::Lookup(LookupClass::Key(from_range))), + ValueKey::from(ValueClass::Lookup(LookupClass::Key(to_range))), + ) + .await + } + #[cfg(feature = "redis")] + InMemoryStore::Redis(store) => store.key_delete_prefix(prefix).await, InMemoryStore::Static(_) | InMemoryStore::Http(_) => { Err(trc::StoreEvent::NotSupported.into_err()) } @@ -138,7 +175,7 @@ impl InMemoryStore { .await .map(|value| value.and_then(|v| v.into())), #[cfg(feature = "redis")] - InMemoryStore::Redis(store) => store.key_get(key.into().into_bytes()).await, + InMemoryStore::Redis(store) => store.key_get(key.into().as_bytes()).await, InMemoryStore::Static(store) => Ok(store .get(key.into().as_str()) .map(|value| T::from(value.clone()))), @@ -149,17 +186,17 @@ impl InMemoryStore { .caused_by(trc::location!()) } - pub async fn counter_get(&self, key: Vec) -> trc::Result { + pub async fn counter_get(&self, key: impl Into>) -> trc::Result { match self { InMemoryStore::Store(store) => { store .get_counter(ValueKey::from(ValueClass::Lookup(LookupClass::Counter( - key, + key.into().into_bytes(), )))) .await } #[cfg(feature = "redis")] - InMemoryStore::Redis(store) => store.counter_get(key).await, + InMemoryStore::Redis(store) => store.counter_get(key.into().as_bytes()).await, InMemoryStore::Static(_) | InMemoryStore::Http(_) => { Err(trc::StoreEvent::NotSupported.into_err()) } @@ -176,7 +213,7 @@ impl InMemoryStore { .await .map(|value| matches!(value, Some(LookupValue::Value(())))), #[cfg(feature = "redis")] - InMemoryStore::Redis(store) => store.key_exists(key.into().into_bytes()).await, + InMemoryStore::Redis(store) => store.key_exists(key.into().as_bytes()).await, InMemoryStore::Static(store) => Ok(store.get(key.into().as_str()).is_some()), InMemoryStore::Http(store) => Ok(store.contains(key.into().as_str())), } @@ -386,6 +423,15 @@ impl LookupKey<'_> { LookupKey::BytesRef(bytes) => bytes.to_vec(), } } + + pub fn as_bytes(&self) -> &[u8] { + match self { + LookupKey::String(string) => string.as_bytes(), + LookupKey::StringRef(string) => string.as_bytes(), + LookupKey::Bytes(bytes) => bytes.as_slice(), + LookupKey::BytesRef(bytes) => bytes, + } + } } impl KeyValue { diff --git a/tests/Cargo.toml b/tests/Cargo.toml index a92fb932..2ce1678f 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -6,7 +6,7 @@ resolver = "2" [features] #default = ["sqlite", "postgres", "mysql", "rocks", "elastic", "s3", "redis", "azure", "foundationdb"] -default = ["rocks", "sqlite"] +default = ["rocks", "redis", "postgres", "mysql", "foundationdb"] sqlite = ["store/sqlite"] foundationdb = ["store/foundation", "common/foundation"] postgres = ["store/postgres"] diff --git a/tests/src/store/lookup.rs b/tests/src/store/lookup.rs index 6ab529fe..42f53f27 100644 --- a/tests/src/store/lookup.rs +++ b/tests/src/store/lookup.rs @@ -28,6 +28,7 @@ pub async fn lookup_tests() { }; for (store_id, store) in stores.in_memory_stores { + let is_mysql = store_id == "mysql"; println!("Testing lookup store {}...", store_id); if let InMemoryStore::Store(store) = &store { store.destroy().await; @@ -80,10 +81,15 @@ pub async fn lookup_tests() { .await .unwrap(); assert_eq!(3, store.counter_get(key.clone()).await.unwrap()); - store - .counter_incr(KeyValue::new(key.clone(), -3)) - .await - .unwrap(); + if !is_mysql { + store + .counter_incr(KeyValue::new(key.clone(), -3)) + .await + .unwrap(); + } else { + // TODO: Detect mySQL version and use RETURNING + store.counter_delete(key.clone()).await.unwrap(); + } assert_eq!(0, store.counter_get(key.clone()).await.unwrap()); // Test counter expiry @@ -119,5 +125,138 @@ pub async fn lookup_tests() { if let InMemoryStore::Store(store) = &store { store.assert_is_empty(store.clone().into()).await; } + + // Test prefix delete + store + .key_set(KeyValue::with_prefix( + 1, + [0], + "hello".to_string().into_bytes(), + )) + .await + .unwrap(); + for v in 0u32..2020u32 { + store + .key_set(KeyValue::with_prefix( + 0, + pack_u32(0, v), + "world".to_string().into_bytes(), + )) + .await + .unwrap(); + store + .counter_incr(KeyValue::with_prefix(0, pack_u32(1, v), 123).expires(3600)) + .await + .unwrap(); + } + + // Make sure the keys are there + assert_eq!( + Some("hello"), + store + .key_get::(KeyValue::<()>::build_key(1, [0])) + .await + .unwrap() + .as_deref() + ); + for v in [0, 1000, 1001, 2000, 2001] { + assert_eq!( + Some("world"), + store + .key_get::(KeyValue::<()>::build_key(0, pack_u32(0, v))) + .await + .unwrap() + .as_deref() + ); + } + for v in [0, 1000, 1001, 2000, 2001] { + assert_ne!( + 0, + store + .counter_get(KeyValue::<()>::build_key(0, pack_u32(1, v))) + .await + .unwrap() + ); + } + + // Delete [0, 0, 0, 0, 1] prefix and make sure only the keys with that prefix are gone + store + .key_delete_prefix(&KeyValue::<()>::build_key(0, 1u32.to_be_bytes())) + .await + .unwrap(); + + assert_eq!( + Some("hello"), + store + .key_get::(KeyValue::<()>::build_key(1, [0])) + .await + .unwrap() + .as_deref() + ); + for v in [0, 1000, 1001, 2000, 2001] { + assert_eq!( + Some("world"), + store + .key_get::(KeyValue::<()>::build_key(0, pack_u32(0, v))) + .await + .unwrap() + .as_deref() + ); + } + + for v in [0, 1000, 1001, 2000, 2001] { + assert_eq!( + 0, + store + .counter_get(KeyValue::<()>::build_key(0, pack_u32(1, v))) + .await + .unwrap() + ); + } + + // Delete [0, 0, 0, 0, 0] prefix and make sure only the keys with that prefix are gone + store + .key_delete_prefix(&KeyValue::<()>::build_key(0, 0u32.to_be_bytes())) + .await + .unwrap(); + + assert_eq!( + Some("hello"), + store + .key_get::(KeyValue::<()>::build_key(1, [0])) + .await + .unwrap() + .as_deref() + ); + for v in [0, 1000, 1001, 2000, 2001] { + assert_eq!( + None, + store + .key_get::(KeyValue::<()>::build_key(0, pack_u32(0, v))) + .await + .unwrap() + .as_deref() + ); + } + + // Delete [1, ...] prefix and make sure it's all gone + store.key_delete_prefix(&[1u8]).await.unwrap(); + + assert_eq!( + None, + store + .key_get::(KeyValue::<()>::build_key(1, [0])) + .await + .unwrap() + .as_deref() + ); + + if let InMemoryStore::Store(store) = &store { + store.assert_is_empty(store.clone().into()).await; + } } } + +fn pack_u32(a: u32, b: u32) -> Vec { + ((a as u64) << 32 | b as u64).to_be_bytes().to_vec() +} diff --git a/tests/src/store/mod.rs b/tests/src/store/mod.rs index 503eb1a2..76c3bb2b 100644 --- a/tests/src/store/mod.rs +++ b/tests/src/store/mod.rs @@ -67,6 +67,11 @@ type = "redis" urls = "redis://127.0.0.1" redis-type = "single" +[storage] +lookup = "mysql" +data = "postgresql" +blob = "sqlite" + "#; #[tokio::test(flavor = "multi_thread")]