From 4a00bbb79ee2c6ed4ee66b882b4235843385d750 Mon Sep 17 00:00:00 2001 From: mdecimus Date: Fri, 1 Dec 2023 07:49:12 +0100 Subject: [PATCH] mySQL backend implementation --- Cargo.lock | 549 +++++++++++++++++- crates/jmap/src/lib.rs | 18 +- crates/store/Cargo.toml | 1 + .../store/src/backend/foundationdb/write.rs | 31 +- crates/store/src/backend/mod.rs | 1 + crates/store/src/backend/mysql/blob.rs | 77 +++ crates/store/src/backend/mysql/main.rs | 131 +++++ crates/store/src/backend/mysql/mod.rs | 47 ++ crates/store/src/backend/mysql/purge.rs | 64 ++ crates/store/src/backend/mysql/read.rs | 378 ++++++++++++ crates/store/src/backend/mysql/tls.rs | 152 +++++ crates/store/src/backend/mysql/write.rs | 313 ++++++++++ crates/store/src/backend/postgres/purge.rs | 2 +- crates/store/src/dispatch.rs | 25 + crates/store/src/lib.rs | 12 +- crates/store/src/write/bitmap.rs | 28 + tests/src/imap/mod.rs | 11 +- tests/src/jmap/mod.rs | 11 +- tests/src/store/blob.rs | 20 +- tests/src/store/mod.rs | 18 +- 20 files changed, 1856 insertions(+), 33 deletions(-) create mode 100644 crates/store/src/backend/mysql/blob.rs create mode 100644 crates/store/src/backend/mysql/main.rs create mode 100644 crates/store/src/backend/mysql/mod.rs create mode 100644 crates/store/src/backend/mysql/purge.rs create mode 100644 crates/store/src/backend/mysql/read.rs create mode 100644 crates/store/src/backend/mysql/tls.rs create mode 100644 crates/store/src/backend/mysql/write.rs diff --git a/Cargo.lock b/Cargo.lock index c38d1d7f..23938a71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -453,6 +453,19 @@ dependencies = [ "tokio", ] +[[package]] +name = "bigdecimal" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06619be423ea5bb86c95f087d5707942791a08a85530df0db2209a3ecfb8bc9" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "bincode" version = "1.3.3" @@ -627,6 +640,39 @@ dependencies = [ "cipher 0.4.4", ] +[[package]] +name = "borsh" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf617fabf5cdbdc92f774bfe5062d870f228b80056d41180797abf48bed4056e" +dependencies = [ + "borsh-derive", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f404657a7ea7b5249e36808dff544bc88a28f26e0ac40009f674b7a009d14be3" +dependencies = [ + "once_cell", + "proc-macro-crate 2.0.0", + "proc-macro2", + "quote", + "syn 2.0.39", + "syn_derive", +] + +[[package]] +name = "btoi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd6407f73a9b8b6162d8a2ef999fe6afd7cc15902ebf42c5cd296addf17e0ad" +dependencies = [ + "num-traits", +] + [[package]] name = "buffered-reader" version = "1.3.0" @@ -643,6 +689,28 @@ version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +[[package]] +name = "bytecheck" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6372023ac861f6e6dc89c8344a8f398fb42aaba2b5dbc649ca0c0e9dbcb627" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7ec4c6f261935ad534c0c22dbef2201b45918860eb1c574b972bd213a76af61" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "bytemuck" version = "1.14.0" @@ -764,6 +832,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + [[package]] name = "chrono" version = "0.4.31" @@ -861,6 +935,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "cmake" +version = "0.1.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.0" @@ -953,6 +1036,20 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" +dependencies = [ + "cfg-if", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + [[package]] name = "crossbeam-channel" version = "0.5.8" @@ -1103,6 +1200,41 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "darling" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.39", +] + +[[package]] +name = "darling_macro" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.39", +] + [[package]] name = "dashmap" version = "5.5.3" @@ -1764,6 +1896,58 @@ dependencies = [ "bindgen 0.65.1", ] +[[package]] +name = "frunk" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11a351b59e12f97b4176ee78497dff72e4276fb1ceb13e19056aca7fa0206287" +dependencies = [ + "frunk_core", + "frunk_derives", + "frunk_proc_macros", +] + +[[package]] +name = "frunk_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af2469fab0bd07e64ccf0ad57a1438f63160c69b2e57f04a439653d68eb558d6" + +[[package]] +name = "frunk_derives" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fa992f1656e1707946bbba340ad244f0814009ef8c0118eb7b658395f19a2e" +dependencies = [ + "frunk_proc_macro_helpers", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "frunk_proc_macro_helpers" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35b54add839292b743aeda6ebedbd8b11e93404f902c56223e51b9ec18a13d2c" +dependencies = [ + "frunk_core", + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "frunk_proc_macros" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71b85a1d4a9a6b300b41c05e8e13ef2feca03e0334127f29eca9506a7fe13a93" +dependencies = [ + "frunk_core", + "frunk_proc_macro_helpers", + "quote", + "syn 2.0.39", +] + [[package]] name = "funty" version = "2.0.0" @@ -2327,6 +2511,12 @@ dependencies = [ "cipher 0.4.4", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.4.0" @@ -2646,6 +2836,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "keyed_priority_queue" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d" +dependencies = [ + "indexmap 2.1.0", +] + [[package]] name = "konst" version = "0.3.6" @@ -2845,6 +3044,15 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "lru" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2994eeba8ed550fd9b47a0b38f0242bc3344e496483c6180b69139cc2fa5d1d7" +dependencies = [ + "hashbrown 0.14.3", +] + [[package]] name = "lru-cache" version = "0.1.2" @@ -3100,10 +3308,105 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" dependencies = [ "libc", + "log", "wasi", "windows-sys 0.48.0", ] +[[package]] +name = "mysql-common-derive" +version = "0.30.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56b0d8a0db9bf6d2213e11f2c701cb91387b0614361625ab7b9743b41aa4938f" +dependencies = [ + "darling", + "heck", + "num-bigint", + "proc-macro-crate 1.3.1", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.39", + "termcolor", + "thiserror", +] + +[[package]] +name = "mysql_async" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6750b17ce50f8f112ef1a8394121090d47c596b56a6a17569ca680a9626e2ef2" +dependencies = [ + "bytes", + "crossbeam", + "flate2", + "futures-core", + "futures-sink", + "futures-util", + "keyed_priority_queue", + "lazy_static", + "lru", + "mio", + "mysql_common", + "once_cell", + "pem", + "percent-encoding", + "pin-project", + "rand", + "rustls 0.21.9", + "rustls-pemfile", + "serde", + "serde_json", + "socket2 0.5.5", + "thiserror", + "tokio", + "tokio-rustls", + "tokio-util", + "twox-hash", + "url", + "webpki", + "webpki-roots 0.25.3", +] + +[[package]] +name = "mysql_common" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06f19e4cfa0ab5a76b627cec2d81331c49b034988eaf302c3bafeada684eadef" +dependencies = [ + "base64 0.21.5", + "bigdecimal", + "bindgen 0.65.1", + "bitflags 2.4.1", + "bitvec", + "btoi", + "byteorder", + "bytes", + "cc", + "cmake", + "crc32fast", + "flate2", + "frunk", + "lazy_static", + "mysql-common-derive", + "num-bigint", + "num-traits", + "rand", + "regex", + "rust_decimal", + "saturating", + "serde", + "serde_json", + "sha1", + "sha2 0.10.8", + "smallvec", + "subprocess", + "thiserror", + "time", + "uuid", + "zstd 0.12.4", +] + [[package]] name = "new_debug_unreachable" version = "1.0.4" @@ -3542,6 +3845,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" +[[package]] +name = "pem" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3163d2912b7c3b52d651a055f2c7eec9ba5cd22d26ef75b8dd3a59980b185923" +dependencies = [ + "base64 0.21.5", + "serde", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -3801,6 +4114,49 @@ dependencies = [ "nix", ] +[[package]] +name = "proc-macro-crate" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" +dependencies = [ + "once_cell", + "toml_edit 0.19.15", +] + +[[package]] +name = "proc-macro-crate" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8366a6159044a37876a2b9817124296703c586a5c92e2c53751fa06d8d43e8" +dependencies = [ + "toml_edit 0.20.7", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro2" version = "1.0.70" @@ -3833,6 +4189,26 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "pwhash" version = "1.0.0" @@ -4084,6 +4460,15 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +[[package]] +name = "rend" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2571463863a6bd50c32f94402933f03457a3fbaf697a707c5be741e459f08fd" +dependencies = [ + "bytecheck", +] + [[package]] name = "reqwest" version = "0.11.22" @@ -4191,6 +4576,34 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "rkyv" +version = "0.7.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0200c8230b013893c0b2d6213d6ec64ed2b9be2e0e016682b7224ff82cff5c58" +dependencies = [ + "bitvec", + "bytecheck", + "hashbrown 0.12.3", + "ptr_meta", + "rend", + "rkyv_derive", + "seahash", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.7.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2e06b915b5c230a17d7a736d1e2e63ee753c256a8614ef3f5147b13a4f5541d" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "roaring" version = "0.10.2" @@ -4319,6 +4732,22 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "rust_decimal" +version = "1.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06676aec5ccb8fc1da723cc8c0f9a46549f21ebb8753d3915c6c41db1e7f1dc4" +dependencies = [ + "arrayvec", + "borsh", + "bytes", + "num-traits", + "rand", + "rkyv", + "serde", + "serde_json", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -4438,6 +4867,12 @@ dependencies = [ "cipher 0.4.4", ] +[[package]] +name = "saturating" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71" + [[package]] name = "schannel" version = "0.1.22" @@ -4484,6 +4919,12 @@ dependencies = [ "untrusted 0.9.0", ] +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "sec1" version = "0.7.3" @@ -4771,6 +5212,12 @@ dependencies = [ "rand_core", ] +[[package]] +name = "simdutf8" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" + [[package]] name = "siphasher" version = "0.3.11" @@ -5191,6 +5638,7 @@ dependencies = [ "lazy_static", "lru-cache", "lz4_flex", + "mysql_async", "nlp", "num_cpus", "parking_lot", @@ -5242,6 +5690,16 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "subprocess" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2e86926081dda636c546d8c5e641661049d7562a68f5488be4a1f7f66f6086" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "subtle" version = "2.4.1" @@ -5270,6 +5728,18 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1329189c02ff984e9736652b1631330da25eaa6bc639089ed4915d25446cbe7b" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "sync_wrapper" version = "0.1.2" @@ -5350,6 +5820,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "termcolor" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" +dependencies = [ + "winapi-util", +] + [[package]] name = "tests" version = "0.1.0" @@ -5608,6 +6087,34 @@ dependencies = [ "tracing", ] +[[package]] +name = "toml_datetime" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" + +[[package]] +name = "toml_edit" +version = "0.19.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap 2.1.0", + "toml_datetime", + "winnow", +] + +[[package]] +name = "toml_edit" +version = "0.20.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" +dependencies = [ + "indexmap 2.1.0", + "toml_datetime", + "winnow", +] + [[package]] name = "tonic" version = "0.9.2" @@ -5819,6 +6326,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", + "rand", "static_assertions", ] @@ -6222,6 +6730,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -6369,6 +6886,15 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "winnow" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.50.0" @@ -6494,7 +7020,7 @@ dependencies = [ "pbkdf2 0.11.0", "sha1", "time", - "zstd", + "zstd 0.11.2+zstd.1.5.2", ] [[package]] @@ -6514,7 +7040,16 @@ version = "0.11.2+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" dependencies = [ - "zstd-safe", + "zstd-safe 5.0.2+zstd.1.5.2", +] + +[[package]] +name = "zstd" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" +dependencies = [ + "zstd-safe 6.0.6", ] [[package]] @@ -6527,6 +7062,16 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "zstd-safe" +version = "6.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" +dependencies = [ + "libc", + "zstd-sys", +] + [[package]] name = "zstd-sys" version = "2.0.9+zstd.1.5.5" diff --git a/crates/jmap/src/lib.rs b/crates/jmap/src/lib.rs index 027691ae..a85f0602 100644 --- a/crates/jmap/src/lib.rs +++ b/crates/jmap/src/lib.rs @@ -48,7 +48,10 @@ use services::{ }; use smtp::core::SMTP; use store::{ - backend::{foundationdb::FdbStore, fs::FsStore, postgres::PostgresStore, sqlite::SqliteStore}, + backend::{ + foundationdb::FdbStore, fs::FsStore, mysql::MysqlStore, postgres::PostgresStore, + sqlite::SqliteStore, + }, fts::FtsFilter, parking_lot::Mutex, query::{sort::Pagination, Comparator, Filter, ResultSet, SortedResultSet}, @@ -196,21 +199,26 @@ impl JMAP { .property::("global.shared-map.shard")? .unwrap_or(32) .next_power_of_two() as usize; - let store = Store::PostgreSQL(Arc::new( + /*let store = Store::PostgreSQL(Arc::new( PostgresStore::open(config) .await .failed("Unable to open database"), )); - /*let store = Store::SQLite(Arc::new( + let store = Store::SQLite(Arc::new( SqliteStore::open(config) .await .failed("Unable to open database"), - ));*/ - /*let store = Store::FoundationDb(Arc::new( + )); + let store = Store::FoundationDb(Arc::new( FdbStore::open(config) .await .failed("Unable to open database"), ));*/ + let store = Store::MySQL(Arc::new( + MysqlStore::open(config) + .await + .failed("Unable to open database"), + )); let blob_store = store.clone().into(); /*let blob_store = BlobStore::Fs(Arc::new( FsStore::open(config) diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 3ac40bec..3ba3ce77 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -34,6 +34,7 @@ tokio-postgres = "0.7.10" tokio-rustls = { version = "0.24.0"} rustls = "0.21.0" ring = "0.17" +mysql_async = { version = "*", default-features = false, features = ["default-rustls"] } [dev-dependencies] tokio = { version = "1.23", features = ["full"] } diff --git a/crates/store/src/backend/foundationdb/write.rs b/crates/store/src/backend/foundationdb/write.rs index 89eb776c..4908fb00 100644 --- a/crates/store/src/backend/foundationdb/write.rs +++ b/crates/store/src/backend/foundationdb/write.rs @@ -28,7 +28,10 @@ use foundationdb::{options::MutationType, FdbError}; use rand::Rng; use crate::{ - write::{bitmap::DenseBitmap, Batch, Operation, ValueOp, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME}, + write::{ + bitmap::{block_contains, DenseBitmap}, + Batch, BitmapClass, Operation, ValueClass, ValueOp, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, + }, BitmapKey, BlobKey, IndexKey, Key, LogKey, ValueKey, }; @@ -96,6 +99,28 @@ impl FdbStore { if let ValueOp::Set(value) = op { trx.set(&key, value); + + if matches!(class, ValueClass::ReservedId) { + let block_num = DenseBitmap::block_num(document_id); + if let Ok(Some(bytes)) = trx + .get( + &BitmapKey { + account_id, + collection, + class: BitmapClass::DocumentIds, + block_num, + } + .serialize(true), + true, + ) + .await + { + if block_contains(&bytes, block_num, document_id) { + trx.cancel(); + return Err(crate::Error::AssertValueFailed); + } + } + } } else { trx.clear(&key); } @@ -205,7 +230,7 @@ impl FdbStore { match trx.commit().await { Ok(_) => { - #[cfg(feature = "test_mode")] + /*#[cfg(feature = "test_mode")] { for op in &batch.ops { match op { @@ -255,7 +280,7 @@ impl FdbStore { _ => {} } } - } + }*/ return Ok(()); } diff --git a/crates/store/src/backend/mod.rs b/crates/store/src/backend/mod.rs index 850cbc02..9ab4c108 100644 --- a/crates/store/src/backend/mod.rs +++ b/crates/store/src/backend/mod.rs @@ -24,6 +24,7 @@ #[cfg(feature = "foundation")] pub mod foundationdb; pub mod fs; +pub mod mysql; pub mod postgres; #[cfg(feature = "rocks")] pub mod rocksdb; diff --git a/crates/store/src/backend/mysql/blob.rs b/crates/store/src/backend/mysql/blob.rs new file mode 100644 index 00000000..be837c93 --- /dev/null +++ b/crates/store/src/backend/mysql/blob.rs @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::ops::Range; + +use mysql_async::prelude::Queryable; + +use super::MysqlStore; + +impl MysqlStore { + pub(crate) async fn get_blob( + &self, + key: &[u8], + range: Range, + ) -> crate::Result>> { + let mut conn = self.conn_pool.get_conn().await?; + let s = conn.prep("SELECT v FROM t WHERE k = ?").await?; + conn.exec_first::, _, _>(&s, (key,)) + .await + .map(|bytes| { + if range.start == 0 && range.end == u32::MAX { + bytes + } else { + bytes.map(|bytes| { + bytes + .get( + range.start as usize + ..std::cmp::min(bytes.len(), range.end as usize), + ) + .unwrap_or_default() + .to_vec() + }) + } + }) + .map_err(Into::into) + } + + pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> crate::Result<()> { + let mut conn = self.conn_pool.get_conn().await?; + let s = conn + .prep("INSERT INTO t (k, v) VALUES (?, ?) ON DUPLICATE KEY UPDATE v = VALUES(v)") + .await?; + conn.exec_drop(&s, (key, data)) + .await + .map_err(|e| crate::Error::InternalError(format!("Failed to insert blob: {}", e))) + .map(|_| ()) + } + + pub(crate) async fn delete_blob(&self, key: &[u8]) -> crate::Result { + let mut conn = self.conn_pool.get_conn().await?; + let s = conn.prep("DELETE FROM t WHERE k = ?").await?; + conn.exec_iter(&s, (key,)) + .await + .map_err(|e| crate::Error::InternalError(format!("Failed to delete blob: {}", e))) + .map(|hits| hits.affected_rows() > 0) + } +} diff --git a/crates/store/src/backend/mysql/main.rs b/crates/store/src/backend/mysql/main.rs new file mode 100644 index 00000000..57ae3b4e --- /dev/null +++ b/crates/store/src/backend/mysql/main.rs @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use mysql_async::{prelude::Queryable, OptsBuilder, Pool, PoolConstraints, PoolOpts}; + +use crate::{ + SUBSPACE_BITMAPS, SUBSPACE_BLOBS, SUBSPACE_BLOB_DATA, SUBSPACE_COUNTERS, SUBSPACE_INDEXES, + SUBSPACE_INDEX_VALUES, SUBSPACE_LOGS, SUBSPACE_VALUES, +}; + +use super::MysqlStore; + +impl MysqlStore { + pub async fn open(config: &utils::config::Config) -> crate::Result { + let mut opts = OptsBuilder::default() + .ip_or_hostname(config.value_require("store.db.host")?.to_string()) + .user(config.value("store.db.user").map(|s| s.to_string())) + .pass(config.value("store.db.password").map(|s| s.to_string())) + .db_name( + config + .value_require("store.db.database")? + .to_string() + .into(), + ) + .wait_timeout(config.property("store.db.timeout")?); + if let Some(port) = config.property("store.db.port")? { + opts = opts.tcp_port(port); + } + + // Configure connection pool + let mut pool_min = PoolConstraints::default().min(); + let mut pool_max = PoolConstraints::default().max(); + if let Some(n_size) = config.property::("store.db.pool.min-connections")? { + pool_min = n_size; + } + if let Some(n_size) = config.property::("store.db.pool.max-connections")? { + pool_max = n_size; + } + opts = opts.pool_opts( + PoolOpts::default().with_constraints(PoolConstraints::new(pool_min, pool_max).unwrap()), + ); + + let db = Self { + conn_pool: Pool::new(opts), + }; + + db.create_tables().await?; + + Ok(db) + } + + pub(super) async fn create_tables(&self) -> crate::Result<()> { + let mut conn = self.conn_pool.get_conn().await?; + + for table in [SUBSPACE_VALUES, SUBSPACE_LOGS, SUBSPACE_INDEX_VALUES] { + let table = char::from(table); + conn.query_drop(&format!( + "CREATE TABLE IF NOT EXISTS {table} ( + k TINYBLOB, + v MEDIUMBLOB NOT NULL, + PRIMARY KEY (k(255)) + ) ENGINE=InnoDB" + )) + .await?; + } + + conn.query_drop(&format!( + "CREATE TABLE IF NOT EXISTS {} ( + k TINYBLOB, + v LONGBLOB NOT NULL, + PRIMARY KEY (k(255)) + ) ENGINE=InnoDB", + char::from(SUBSPACE_BLOB_DATA), + )) + .await?; + + for table in [SUBSPACE_INDEXES, SUBSPACE_BITMAPS] { + let table = char::from(table); + conn.query_drop(&format!( + "CREATE TABLE IF NOT EXISTS {table} ( + k BLOB, + PRIMARY KEY (k(400)) + ) ENGINE=InnoDB" + )) + .await?; + } + + for table in [SUBSPACE_BLOBS] { + let table = char::from(table); + conn.query_drop(&format!( + "CREATE TABLE IF NOT EXISTS {table} ( + k TINYBLOB, + PRIMARY KEY (k(255)) + ) ENGINE=InnoDB" + )) + .await?; + } + + conn.query_drop(&format!( + "CREATE TABLE IF NOT EXISTS {} ( + k TINYBLOB, + v BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (k(255)) + ) ENGINE=InnoDB", + char::from(SUBSPACE_COUNTERS) + )) + .await?; + + Ok(()) + } +} diff --git a/crates/store/src/backend/mysql/mod.rs b/crates/store/src/backend/mysql/mod.rs new file mode 100644 index 00000000..c307781c --- /dev/null +++ b/crates/store/src/backend/mysql/mod.rs @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use mysql_async::Pool; + +pub mod blob; +pub mod main; +pub mod purge; +pub mod read; +pub mod tls; +pub mod write; + +pub struct MysqlStore { + pub(crate) conn_pool: Pool, +} + +impl From for crate::Error { + fn from(err: mysql_async::Error) -> Self { + Self::InternalError(format!("mySQL error: {}", err)) + } +} + +impl From for crate::Error { + fn from(err: mysql_async::FromValueError) -> Self { + Self::InternalError(format!("mySQL value conversion error: {}", err)) + } +} diff --git a/crates/store/src/backend/mysql/purge.rs b/crates/store/src/backend/mysql/purge.rs new file mode 100644 index 00000000..7a794984 --- /dev/null +++ b/crates/store/src/backend/mysql/purge.rs @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use mysql_async::prelude::Queryable; + +use crate::{ + write::key::KeySerializer, SUBSPACE_BITMAPS, SUBSPACE_INDEXES, SUBSPACE_INDEX_VALUES, + SUBSPACE_LOGS, SUBSPACE_VALUES, U32_LEN, +}; + +use super::MysqlStore; + +impl MysqlStore { + pub(crate) async fn purge_bitmaps(&self) -> crate::Result<()> { + // Not needed for PostgreSQL + Ok(()) + } + + pub(crate) async fn purge_account(&self, account_id: u32) -> crate::Result<()> { + let mut conn = self.conn_pool.get_conn().await?; + let from_key = KeySerializer::new(U32_LEN).write(account_id).finalize(); + let to_key = KeySerializer::new(U32_LEN).write(account_id + 1).finalize(); + + for (table, i) in [ + (SUBSPACE_BITMAPS, 'z'), + (SUBSPACE_VALUES, 'k'), + (SUBSPACE_LOGS, 'k'), + (SUBSPACE_INDEXES, 'k'), + (SUBSPACE_INDEX_VALUES, 'k'), + ] { + let s = conn + .prep(&format!( + "DELETE FROM {} WHERE {} >= ? AND {} < ?", + char::from(table), + i, + i + )) + .await?; + conn.exec_drop(&s, (&from_key, &to_key)).await?; + } + + Ok(()) + } +} diff --git a/crates/store/src/backend/mysql/read.rs b/crates/store/src/backend/mysql/read.rs new file mode 100644 index 00000000..e91a6a00 --- /dev/null +++ b/crates/store/src/backend/mysql/read.rs @@ -0,0 +1,378 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use futures::TryStreamExt; +use mysql_async::{prelude::Queryable, Row}; +use roaring::RoaringBitmap; + +use crate::{ + query::{self, Operator}, + write::{ + key::{DeserializeBigEndian, KeySerializer}, + BitmapClass, ValueClass, + }, + BitmapKey, Deserialize, IndexKey, IndexKeyPrefix, IterateParams, Key, ValueKey, U32_LEN, +}; + +use super::MysqlStore; + +impl MysqlStore { + pub(crate) async fn get_value(&self, key: impl Key) -> crate::Result> + where + U: Deserialize + 'static, + { + let key = key.serialize(false); + let mut conn = self.conn_pool.get_conn().await?; + let s = conn.prep("SELECT v FROM v WHERE k = ?").await?; + conn.exec_first::, _, _>(&s, (key,)) + .await + .map_err(Into::into) + .and_then(|r| { + if let Some(r) = r { + Ok(Some(U::deserialize(&r)?)) + } else { + Ok(None) + } + }) + } + + pub(crate) async fn get_bitmap( + &self, + mut key: BitmapKey, + ) -> crate::Result> { + let begin = key.serialize(false); + key.block_num = u32::MAX; + let key_len = begin.len(); + let end = key.serialize(false); + let mut conn = self.conn_pool.get_conn().await?; + + let mut bm = RoaringBitmap::new(); + let s = conn.prep("SELECT k FROM b WHERE k >= ? AND k <= ?").await?; + let mut rows = conn.exec_stream::, _, _>(&s, (begin, end)).await?; + + while let Some(key) = rows.try_next().await? { + if key.len() == key_len { + bm.insert(key.as_slice().deserialize_be_u32(key.len() - U32_LEN)?); + } + } + Ok(if !bm.is_empty() { Some(bm) } else { None }) + } + + pub(crate) async fn range_to_bitmap( + &self, + account_id: u32, + collection: u8, + field: u8, + value: &[u8], + op: query::Operator, + ) -> crate::Result> { + let mut conn = self.conn_pool.get_conn().await?; + let k1 = + KeySerializer::new(std::mem::size_of::>() + value.len() + 1 + U32_LEN) + .write(account_id) + .write(collection) + .write(field); + let k2 = + KeySerializer::new(std::mem::size_of::>() + value.len() + 1 + U32_LEN) + .write(account_id) + .write(collection) + .write( + field + matches!(op, Operator::GreaterThan | Operator::GreaterEqualThan) as u8, + ); + + let (query, begin, end) = match op { + Operator::LowerThan => ( + ("SELECT k FROM i WHERE k >= ? AND k < ?"), + (k1.finalize()), + (k2.write(value).write(0u32).finalize()), + ), + Operator::LowerEqualThan => ( + ("SELECT k FROM i WHERE k >= ? AND k <= ?"), + (k1.finalize()), + (k2.write(value).write(u32::MAX).finalize()), + ), + Operator::GreaterThan => ( + ("SELECT k FROM i WHERE k > ? AND k <= ?"), + (k1.write(value).write(u32::MAX).finalize()), + (k2.finalize()), + ), + Operator::GreaterEqualThan => ( + ("SELECT k FROM i WHERE k >= ? AND k <= ?"), + (k1.write(value).write(0u32).finalize()), + (k2.finalize()), + ), + Operator::Equal => ( + ("SELECT k FROM i WHERE k >= ? AND k <= ?"), + (k1.write(value).write(0u32).finalize()), + (k2.write(value).write(u32::MAX).finalize()), + ), + }; + + let mut bm = RoaringBitmap::new(); + let s = conn.prep(query).await?; + let key_len = begin.len(); + let mut rows = conn.exec_stream::, _, _>(&s, (begin, end)).await?; + + if op != Operator::Equal { + while let Some(key) = rows.try_next().await? { + bm.insert(key.as_slice().deserialize_be_u32(key.len() - U32_LEN)?); + } + } else { + while let Some(key) = rows.try_next().await? { + if key.len() == key_len { + bm.insert(key.as_slice().deserialize_be_u32(key.len() - U32_LEN)?); + } + } + } + + Ok(Some(bm)) + } + + pub(crate) async fn sort_index( + &self, + account_id: u32, + collection: impl Into + Sync + Send, + field: impl Into + Sync + Send, + ascending: bool, + mut cb: impl for<'x> FnMut(&'x [u8], u32) -> crate::Result + Sync + Send, + ) -> crate::Result<()> { + let collection = collection.into(); + let field = field.into(); + + let mut conn = self.conn_pool.get_conn().await?; + let begin = IndexKeyPrefix { + account_id, + collection, + field, + } + .serialize(false); + let end = IndexKeyPrefix { + account_id, + collection, + field: field + 1, + } + .serialize(false); + let prefix_len = begin.len(); + let s = conn + .prep(if ascending { + "SELECT k FROM i WHERE k >= ? AND k < ? ORDER BY k ASC" + } else { + "SELECT k FROM i WHERE k >= ? AND k < ? ORDER BY k DESC" + }) + .await?; + let mut rows = conn.exec_stream::, _, _>(&s, (begin, end)).await?; + + while let Some(key) = rows.try_next().await? { + let id_pos = key.len() - U32_LEN; + if !cb( + key.get(prefix_len..id_pos).ok_or_else(|| { + crate::Error::InternalError("Invalid key found in index".to_string()) + })?, + key.as_slice().deserialize_be_u32(id_pos)?, + )? { + return Ok(()); + } + } + + Ok(()) + } + + pub(crate) async fn iterate( + &self, + params: IterateParams, + mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> crate::Result + Sync + Send, + ) -> crate::Result<()> { + let mut conn = self.conn_pool.get_conn().await?; + let table = char::from(params.begin.subspace()); + let begin = params.begin.serialize(false); + let end = params.end.serialize(false); + let keys = if params.values { "k, v" } else { "k" }; + + let s = conn + .prep(&match (params.first, params.ascending) { + (true, true) => { + format!( + "SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC LIMIT 1" + ) + } + (true, false) => { + format!( + "SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC LIMIT 1" + ) + } + (false, true) => { + format!("SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC") + } + (false, false) => { + format!("SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC") + } + }) + .await?; + let mut rows = conn.exec_stream::(&s, (begin, end)).await?; + + if params.values { + while let Some(mut row) = rows.try_next().await? { + let value = row + .take_opt::, _>(1) + .unwrap_or_else(|| Ok(vec![]))?; + let key = row + .take_opt::, _>(0) + .unwrap_or_else(|| Ok(vec![]))?; + + if !cb(&key, &value)? { + break; + } + } + } else { + while let Some(mut row) = rows.try_next().await? { + if !cb( + &row.take_opt::, _>(0) + .unwrap_or_else(|| Ok(vec![]))?, + b"", + )? { + break; + } + } + } + + Ok(()) + } + + pub(crate) async fn get_counter( + &self, + key: impl Into> + Sync + Send, + ) -> crate::Result { + let key = key.into().serialize(false); + let mut conn = self.conn_pool.get_conn().await?; + let s = conn.prep("SELECT v FROM c WHERE k = ?").await?; + match conn.exec_first::(&s, (key,)).await { + Ok(Some(num)) => Ok(num), + Ok(None) => Ok(0), + Err(e) => Err(e.into()), + } + } + + #[cfg(feature = "test_mode")] + pub(crate) async fn assert_is_empty(&self) { + let mut conn = self.conn_pool.get_conn().await.unwrap(); + + // Values + let mut has_errors = false; + for table in [ + crate::SUBSPACE_VALUES, + crate::SUBSPACE_INDEX_VALUES, + crate::SUBSPACE_COUNTERS, + crate::SUBSPACE_BLOB_DATA, + ] { + let table = char::from(table); + let s = conn + .prep(&format!("SELECT k, v FROM {table}")) + .await + .unwrap(); + let mut rows = conn.exec_stream::(&s, ()).await.unwrap(); + + while let Some(mut row) = rows.try_next().await.unwrap() { + let key = row + .take_opt::, _>(0) + .unwrap_or_else(|| Ok(vec![])) + .unwrap(); + if table != 'c' { + let value = row + .take_opt::, _>(1) + .unwrap_or_else(|| Ok(vec![])) + .unwrap(); + + if key[0..4] != u32::MAX.to_be_bytes() { + eprintln!("Table {table:?} is not empty: {key:?} {value:?}"); + has_errors = true; + } + } else { + let value = row.take_opt::(1).unwrap_or(Ok(0)).unwrap(); + if value != 0 { + eprintln!( + "Table counter is not empty, account {:?}, quota: {}", + key, value, + ); + has_errors = true; + } + } + } + } + + // Indexes + for table in [crate::SUBSPACE_INDEXES, crate::SUBSPACE_BLOBS] { + let table = char::from(table); + let s = conn.prep(&format!("SELECT k FROM {table}")).await.unwrap(); + let mut rows = conn.exec_stream::(&s, ()).await.unwrap(); + while let Some(mut row) = rows.try_next().await.unwrap() { + let key = row + .take_opt::, _>(0) + .unwrap_or_else(|| Ok(vec![])) + .unwrap(); + + if table == 'i' { + eprintln!( + "Table index is not empty, account {}, collection {}, document {}, property {}, value {:?}: {:?}", + u32::from_be_bytes(key[0..4].try_into().unwrap()), + key[4], + u32::from_be_bytes(key[key.len()-4..].try_into().unwrap()), + key[5], + String::from_utf8_lossy(&key[6..key.len()-4]), + key + ); + } else { + eprintln!("Table {table:?} is not empty: {key:?}"); + } + has_errors = true; + } + } + + // Bitmaps + let s = conn + .prep(&format!( + "SELECT k FROM {}", + char::from(crate::SUBSPACE_BITMAPS) + )) + .await + .unwrap(); + let mut rows = conn.exec_stream::(&s, ()).await.unwrap(); + while let Some(mut row) = rows.try_next().await.unwrap() { + let key = row + .take_opt::, _>(0) + .unwrap_or_else(|| Ok(vec![])) + .unwrap(); + if key[0..4] != u32::MAX.to_be_bytes() { + eprintln!("Table bitmaps failed to purge, found key: {key:?}"); + has_errors = true; + } + } + drop(rows); + + // Delete logs + conn.exec_drop("DELETE FROM l", ()).await.unwrap(); + + if has_errors { + panic!("Database is not empty"); + } + } +} diff --git a/crates/store/src/backend/mysql/tls.rs b/crates/store/src/backend/mysql/tls.rs new file mode 100644 index 00000000..76805aeb --- /dev/null +++ b/crates/store/src/backend/mysql/tls.rs @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +// Credits: https://github.com/jbg/tokio-postgres-rustls + +use std::{ + convert::TryFrom, + future::Future, + io, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; + +use futures::future::{FutureExt, TryFutureExt}; +use ring::digest; +use rustls::{ClientConfig, ServerName}; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; +use tokio_postgres::tls::{ChannelBinding, MakeTlsConnect, TlsConnect}; +use tokio_rustls::{client::TlsStream, TlsConnector}; + +#[derive(Clone)] +pub struct MakeRustlsConnect { + config: Arc, +} + +impl MakeRustlsConnect { + pub fn new(config: ClientConfig) -> Self { + Self { + config: Arc::new(config), + } + } +} + +impl MakeTlsConnect for MakeRustlsConnect +where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, +{ + type Stream = RustlsStream; + type TlsConnect = RustlsConnect; + type Error = io::Error; + + fn make_tls_connect(&mut self, hostname: &str) -> io::Result { + ServerName::try_from(hostname) + .map(|dns_name| { + RustlsConnect(Some(RustlsConnectData { + hostname: dns_name, + connector: Arc::clone(&self.config).into(), + })) + }) + .or(Ok(RustlsConnect(None))) + } +} + +pub struct RustlsConnect(Option); + +struct RustlsConnectData { + hostname: ServerName, + connector: TlsConnector, +} + +impl TlsConnect for RustlsConnect +where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, +{ + type Stream = RustlsStream; + type Error = io::Error; + type Future = Pin>> + Send>>; + + fn connect(self, stream: S) -> Self::Future { + match self.0 { + None => Box::pin(core::future::ready(Err(io::ErrorKind::InvalidInput.into()))), + Some(c) => c + .connector + .connect(c.hostname, stream) + .map_ok(|s| RustlsStream(Box::pin(s))) + .boxed(), + } + } +} + +pub struct RustlsStream(Pin>>); + +impl tokio_postgres::tls::TlsStream for RustlsStream +where + S: AsyncRead + AsyncWrite + Unpin, +{ + fn channel_binding(&self) -> ChannelBinding { + let (_, session) = self.0.get_ref(); + match session.peer_certificates() { + Some(certs) if !certs.is_empty() => { + let sha256 = digest::digest(&digest::SHA256, certs[0].as_ref()); + ChannelBinding::tls_server_end_point(sha256.as_ref().into()) + } + _ => ChannelBinding::none(), + } + } +} + +impl AsyncRead for RustlsStream +where + S: AsyncRead + AsyncWrite + Unpin, +{ + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + self.0.as_mut().poll_read(cx, buf) + } +} + +impl AsyncWrite for RustlsStream +where + S: AsyncRead + AsyncWrite + Unpin, +{ + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context, + buf: &[u8], + ) -> Poll> { + self.0.as_mut().poll_write(cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + self.0.as_mut().poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + self.0.as_mut().poll_shutdown(cx) + } +} diff --git a/crates/store/src/backend/mysql/write.rs b/crates/store/src/backend/mysql/write.rs new file mode 100644 index 00000000..6f9753b5 --- /dev/null +++ b/crates/store/src/backend/mysql/write.rs @@ -0,0 +1,313 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::time::{Duration, Instant}; + +use ahash::AHashMap; +use mysql_async::{params, prelude::Queryable, Conn, Error, IsolationLevel, Row, TxOpts}; +use rand::Rng; + +use crate::{ + write::{ + Batch, BitmapClass, Operation, ValueClass, ValueOp, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, + }, + BitmapKey, BlobKey, IndexKey, Key, LogKey, ValueKey, +}; + +use super::MysqlStore; + +impl MysqlStore { + pub(crate) async fn write(&self, batch: Batch) -> crate::Result<()> { + let start = Instant::now(); + let mut retry_count = 0; + let mut conn = self.conn_pool.get_conn().await?; + + loop { + match self.write_trx(&mut conn, &batch).await { + Ok(success) => { + return if success { + Ok(()) + } else { + Err(crate::Error::AssertValueFailed) + }; + } + Err(Error::Server(err)) + if [1062, 1213].contains(&err.code) + && retry_count < MAX_COMMIT_ATTEMPTS + && start.elapsed() < MAX_COMMIT_TIME => + { + let backoff = rand::thread_rng().gen_range(50..=300); + tokio::time::sleep(Duration::from_millis(backoff)).await; + retry_count += 1; + } + Err(err) => { + return Err(err.into()); + } + } + } + } + + async fn write_trx(&self, conn: &mut Conn, batch: &Batch) -> Result { + let mut account_id = u32::MAX; + let mut collection = u8::MAX; + let mut document_id = u32::MAX; + let mut asserted_values = AHashMap::new(); + let mut tx_opts = TxOpts::default(); + tx_opts + .with_consistent_snapshot(false) + .with_isolation_level(IsolationLevel::ReadCommitted); + let mut trx = conn.start_transaction(tx_opts).await?; + + for op in &batch.ops { + match op { + Operation::AccountId { + account_id: account_id_, + } => { + account_id = *account_id_; + } + Operation::Collection { + collection: collection_, + } => { + collection = *collection_; + } + Operation::DocumentId { + document_id: document_id_, + } => { + document_id = *document_id_; + } + Operation::Value { + class, + op: ValueOp::Add(by), + } => { + let key = ValueKey { + account_id, + collection, + document_id, + class, + } + .serialize(false); + + if *by >= 0 { + let s = trx + .prep(concat!( + "INSERT INTO c (k, v) VALUES (?, ?) ", + "ON DUPLICATE KEY UPDATE v = v + VALUES(v)" + )) + .await?; + trx.exec_drop(&s, (key, by)).await?; + } else { + let s = trx.prep("UPDATE c SET v = v + ? WHERE k = ?").await?; + trx.exec_drop(&s, (by, key)).await?; + } + } + Operation::Value { class, op } => { + let key = ValueKey { + account_id, + collection, + document_id, + class, + }; + let table = char::from(key.subspace()); + let key = key.serialize(false); + + if let ValueOp::Set(value) = op { + let s = if let Some(exists) = asserted_values.get(&key) { + if *exists { + trx.prep(&format!("UPDATE {} SET v = :v WHERE k = :k", table)) + .await? + } else { + trx.prep(&format!("INSERT INTO {} (k, v) VALUES (:k, :v)", table)) + .await? + } + } else { + trx + .prep( + &format!("INSERT INTO {} (k, v) VALUES (:k, :v) ON DUPLICATE KEY UPDATE v = VALUES(v)", table), + ) + .await? + }; + + match trx.exec_drop(&s, params! {"k" => key, "v" => value}).await { + Ok(_) => { + if trx.affected_rows() == 0 { + trx.rollback().await?; + return Ok(false); + } + } + Err(err) => { + trx.rollback().await?; + return Err(err); + } + } + + if matches!(class, ValueClass::ReservedId) { + // Make sure the reserved id is not already in use + let s = trx.prep("SELECT 1 FROM b WHERE k = ?").await?; + let key = BitmapKey { + account_id, + collection, + class: BitmapClass::DocumentIds, + block_num: document_id, + } + .serialize(false); + if trx.exec_first::(&s, (key,)).await?.is_some() { + trx.rollback().await?; + return Ok(false); + } + } + } else { + let s = trx + .prep(&format!("DELETE FROM {} WHERE k = ?", table)) + .await?; + trx.exec_drop(&s, (key,)).await?; + } + } + Operation::Index { field, key, set } => { + let key = IndexKey { + account_id, + collection, + document_id, + field: *field, + key, + } + .serialize(false); + + let s = if *set { + trx.prep("INSERT IGNORE INTO i (k) VALUES (?)").await? + } else { + trx.prep("DELETE FROM i WHERE k = ?").await? + }; + trx.exec_drop(&s, (key,)).await?; + } + Operation::Bitmap { class, set } => { + let key = BitmapKey { + account_id, + collection, + class, + block_num: document_id, + } + .serialize(false); + + let s = if *set { + if matches!(class, BitmapClass::DocumentIds) { + trx.prep("INSERT INTO b (k) VALUES (?)").await? + } else { + trx.prep("INSERT IGNORE INTO b (k) VALUES (?)").await? + } + } else { + trx.prep("DELETE FROM b WHERE k = ?").await? + }; + trx.exec_drop(&s, (key,)).await?; + } + Operation::Blob { hash, op, set } => { + let key = BlobKey { + account_id, + collection, + document_id, + hash, + op: *op, + } + .serialize(false); + + let s = if *set { + trx.prep("INSERT IGNORE INTO o (k) VALUES (?)").await? + } else { + trx.prep("DELETE FROM o WHERE k = ?").await? + }; + trx.exec_drop(&s, (key,)).await?; + } + Operation::Log { + collection, + change_id, + set, + } => { + let key = LogKey { + account_id, + collection: *collection, + change_id: *change_id, + } + .serialize(false); + + let s = trx + .prep("INSERT INTO l (k, v) VALUES (?, ?) ON DUPLICATE KEY UPDATE v = VALUES(v)") + .await?; + trx.exec_drop(&s, (key, set)).await?; + } + Operation::AssertValue { + class, + assert_value, + } => { + let key = ValueKey { + account_id, + collection, + document_id, + class, + }; + let table = char::from(key.subspace()); + let key = key.serialize(false); + + let s = trx + .prep(&format!("SELECT v FROM {} WHERE k = ? FOR UPDATE", table)) + .await?; + let (exists, matches) = trx + .exec_first::, _, _>(&s, (&key,)) + .await? + .map(|bytes| (true, assert_value.matches(&bytes))) + .unwrap_or_else(|| (false, assert_value.is_none())); + if !matches { + trx.rollback().await?; + return Ok(false); + } + asserted_values.insert(key, exists); + } + } + } + + trx.commit().await.map(|_| true) + } + + #[cfg(feature = "test_mode")] + pub(crate) async fn destroy(&self) { + use crate::{ + SUBSPACE_BITMAPS, SUBSPACE_BLOBS, SUBSPACE_BLOB_DATA, SUBSPACE_COUNTERS, + SUBSPACE_INDEXES, SUBSPACE_INDEX_VALUES, SUBSPACE_LOGS, SUBSPACE_VALUES, + }; + + let mut conn = self.conn_pool.get_conn().await.unwrap(); + for table in [ + SUBSPACE_VALUES, + SUBSPACE_LOGS, + SUBSPACE_BITMAPS, + SUBSPACE_INDEXES, + SUBSPACE_BLOBS, + SUBSPACE_INDEX_VALUES, + SUBSPACE_COUNTERS, + SUBSPACE_BLOB_DATA, + ] { + conn.exec_drop(&format!("DROP TABLE {}", char::from(table)), ()) + .await + .unwrap(); + } + self.create_tables().await.unwrap(); + } +} diff --git a/crates/store/src/backend/postgres/purge.rs b/crates/store/src/backend/postgres/purge.rs index 73dbb838..7e355d3a 100644 --- a/crates/store/src/backend/postgres/purge.rs +++ b/crates/store/src/backend/postgres/purge.rs @@ -48,7 +48,7 @@ impl PostgresStore { ] { let s = conn .prepare_cached(&format!( - "DELETE FROM {} WHERE {} >= ? AND {} < ?", + "DELETE FROM {} WHERE {} >= $1 AND {} < $2", char::from(table), i, i diff --git a/crates/store/src/dispatch.rs b/crates/store/src/dispatch.rs index e035f6eb..fed77e59 100644 --- a/crates/store/src/dispatch.rs +++ b/crates/store/src/dispatch.rs @@ -44,6 +44,7 @@ impl Store { Self::SQLite(store) => store.get_value(key).await, Self::FoundationDb(store) => store.get_value(key).await, Self::PostgreSQL(store) => store.get_value(key).await, + Self::MySQL(store) => store.get_value(key).await, } } @@ -68,6 +69,7 @@ impl Store { Self::SQLite(store) => store.get_bitmap(key).await, Self::FoundationDb(store) => store.get_bitmap(key).await, Self::PostgreSQL(store) => store.get_bitmap(key).await, + Self::MySQL(store) => store.get_bitmap(key).await, } } @@ -117,6 +119,11 @@ impl Store { .range_to_bitmap(account_id, collection, field, value, op) .await } + Self::MySQL(store) => { + store + .range_to_bitmap(account_id, collection, field, value, op) + .await + } } } @@ -144,6 +151,11 @@ impl Store { .sort_index(account_id, collection, field, ascending, cb) .await } + Self::MySQL(store) => { + store + .sort_index(account_id, collection, field, ascending, cb) + .await + } } } @@ -156,6 +168,7 @@ impl Store { Self::SQLite(store) => store.iterate(params, cb).await, Self::FoundationDb(store) => store.iterate(params, cb).await, Self::PostgreSQL(store) => store.iterate(params, cb).await, + Self::MySQL(store) => store.iterate(params, cb).await, } } @@ -167,6 +180,7 @@ impl Store { Self::SQLite(store) => store.get_counter(key).await, Self::FoundationDb(store) => store.get_counter(key).await, Self::PostgreSQL(store) => store.get_counter(key).await, + Self::MySQL(store) => store.get_counter(key).await, } } @@ -175,6 +189,7 @@ impl Store { Self::SQLite(store) => store.write(batch).await, Self::FoundationDb(store) => store.write(batch).await, Self::PostgreSQL(store) => store.write(batch).await, + Self::MySQL(store) => store.write(batch).await, } } @@ -183,6 +198,7 @@ impl Store { Self::SQLite(store) => store.purge_bitmaps().await, Self::FoundationDb(store) => store.purge_bitmaps().await, Self::PostgreSQL(store) => store.purge_bitmaps().await, + Self::MySQL(store) => store.purge_bitmaps().await, } } pub async fn purge_account(&self, account_id: u32) -> crate::Result<()> { @@ -190,6 +206,7 @@ impl Store { Self::SQLite(store) => store.purge_account(account_id).await, Self::FoundationDb(store) => store.purge_account(account_id).await, Self::PostgreSQL(store) => store.purge_account(account_id).await, + Self::MySQL(store) => store.purge_account(account_id).await, } } @@ -198,6 +215,7 @@ impl Store { Self::SQLite(store) => store.get_blob(key, range).await, Self::FoundationDb(store) => store.get_blob(key, range).await, Self::PostgreSQL(store) => store.get_blob(key, range).await, + Self::MySQL(store) => store.get_blob(key, range).await, } } @@ -206,6 +224,7 @@ impl Store { Self::SQLite(store) => store.put_blob(key, data).await, Self::FoundationDb(store) => store.put_blob(key, data).await, Self::PostgreSQL(store) => store.put_blob(key, data).await, + Self::MySQL(store) => store.put_blob(key, data).await, } } @@ -214,6 +233,7 @@ impl Store { Self::SQLite(store) => store.delete_blob(key).await, Self::FoundationDb(store) => store.delete_blob(key).await, Self::PostgreSQL(store) => store.delete_blob(key).await, + Self::MySQL(store) => store.delete_blob(key).await, } } @@ -223,6 +243,7 @@ impl Store { Self::SQLite(store) => store.destroy().await, Self::FoundationDb(store) => store.destroy().await, Self::PostgreSQL(store) => store.destroy().await, + Self::MySQL(store) => store.destroy().await, } } @@ -289,6 +310,7 @@ impl Store { Self::SQLite(store) => store.assert_is_empty().await, Self::FoundationDb(store) => store.assert_is_empty().await, Self::PostgreSQL(store) => store.assert_is_empty().await, + Self::MySQL(store) => store.assert_is_empty().await, } } } @@ -301,6 +323,7 @@ impl BlobStore { Self::Sqlite(store) => store.get_blob(key, range).await, Self::FoundationDb(store) => store.get_blob(key, range).await, Self::PostgreSQL(store) => store.get_blob(key, range).await, + Self::MySQL(store) => store.get_blob(key, range).await, } } @@ -311,6 +334,7 @@ impl BlobStore { Self::Sqlite(store) => store.put_blob(key, data).await, Self::FoundationDb(store) => store.put_blob(key, data).await, Self::PostgreSQL(store) => store.put_blob(key, data).await, + Self::MySQL(store) => store.put_blob(key, data).await, } } @@ -321,6 +345,7 @@ impl BlobStore { Self::Sqlite(store) => store.delete_blob(key).await, Self::FoundationDb(store) => store.delete_blob(key).await, Self::PostgreSQL(store) => store.delete_blob(key).await, + Self::MySQL(store) => store.delete_blob(key).await, } } } diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 91b75575..a97816ee 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -31,7 +31,8 @@ pub mod write; pub use ahash; use backend::{ - foundationdb::FdbStore, fs::FsStore, postgres::PostgresStore, s3::S3Store, sqlite::SqliteStore, + foundationdb::FdbStore, fs::FsStore, mysql::MysqlStore, postgres::PostgresStore, s3::S3Store, + sqlite::SqliteStore, }; pub use blake3; pub use parking_lot; @@ -172,6 +173,7 @@ pub enum Store { SQLite(Arc), FoundationDb(Arc), PostgreSQL(Arc), + MySQL(Arc), } #[derive(Clone)] @@ -181,6 +183,7 @@ pub enum BlobStore { Sqlite(Arc), FoundationDb(Arc), PostgreSQL(Arc), + MySQL(Arc), } #[derive(Clone)] @@ -206,6 +209,12 @@ impl From for Store { } } +impl From for Store { + fn from(store: MysqlStore) -> Self { + Self::MySQL(Arc::new(store)) + } +} + impl From for BlobStore { fn from(store: FsStore) -> Self { Self::Fs(Arc::new(store)) @@ -230,6 +239,7 @@ impl From for BlobStore { Store::SQLite(store) => Self::Sqlite(store), Store::FoundationDb(store) => Self::FoundationDb(store), Store::PostgreSQL(store) => Self::PostgreSQL(store), + Store::MySQL(store) => Self::MySQL(store), } } } diff --git a/crates/store/src/write/bitmap.rs b/crates/store/src/write/bitmap.rs index a292df7f..25417663 100644 --- a/crates/store/src/write/bitmap.rs +++ b/crates/store/src/write/bitmap.rs @@ -107,6 +107,34 @@ pub fn next_available_index( None } +pub fn block_contains(bytes: &[u8], block_num: u32, document_id: u32) -> bool { + 'outer: for (byte_pos, byte) in bytes.iter().enumerate() { + if *byte != 0 { + let mut index = 0; + loop { + while (byte >> index) & 1 == 0 { + index += 1; + if index == 8 { + continue 'outer; + } + } + + let id = (block_num * BITS_PER_BLOCK_L) + ((byte_pos * 8) + index) as u32; + if id == document_id { + return true; + } else if index < 7 { + index += 1; + continue; + } else { + continue 'outer; + } + } + } + } + + false +} + impl DeserializeBlock for RoaringBitmap { fn deserialize_block(&mut self, bytes: &[u8], block_num: u32) { debug_assert_eq!(bytes.len(), WORD_SIZE_L * WORDS_PER_BLOCK_L as usize); diff --git a/tests/src/imap/mod.rs b/tests/src/imap/mod.rs index dc9aef7f..be06cb22 100644 --- a/tests/src/imap/mod.rs +++ b/tests/src/imap/mod.rs @@ -137,12 +137,15 @@ future-release = [ { if = "authenticated-as", ne = "", then = "99999999d"}, { else = false } ] [store.db] -#path = "{TMP}/sqlite.db" +#path = "PATH/sqlite.db" host = "localhost" -port = 5432 +#port = 5432 +port = 3307 database = "stalwart" -user = "postgres" -password = "mysecretpassword" +#user = "postgres" +#password = "mysecretpassword" +user = "root" +password = "password" [store.blob] type = "local" diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index f279b906..65cb91a2 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -135,12 +135,15 @@ future-release = [ { if = "authenticated-as", ne = "", then = "99999999d"}, { else = false } ] [store.db] -path = "{TMP}/sqlite.db" +#path = "PATH/sqlite.db" host = "localhost" -port = 5432 +#port = 5432 +port = 3307 database = "stalwart" -user = "postgres" -password = "mysecretpassword" +#user = "postgres" +#password = "mysecretpassword" +user = "root" +password = "password" [store.blob] type = "local" diff --git a/tests/src/store/blob.rs b/tests/src/store/blob.rs index a7dee4ba..0d3dd770 100644 --- a/tests/src/store/blob.rs +++ b/tests/src/store/blob.rs @@ -22,7 +22,9 @@ */ use store::{ - backend::{fs::FsStore, postgres::PostgresStore, s3::S3Store, sqlite::SqliteStore}, + backend::{ + fs::FsStore, mysql::MysqlStore, postgres::PostgresStore, s3::S3Store, sqlite::SqliteStore, + }, write::{blob::BlobQuota, now, BatchBuilder, BlobOp, F_CLEAR}, BlobClass, BlobHash, BlobStore, Store, }; @@ -46,12 +48,15 @@ path = "{TMP}" const CONFIG_DB: &str = r#" [store.db] -path = "{TMP}/db.db?mode=rwc" +#path = "PATH/sqlite.db" host = "localhost" -post = 5432 +#port = 5432 +port = 3307 database = "stalwart" -user = "postgres" -password = "mysecretpassword" +#user = "postgres" +#password = "mysecretpassword" +user = "root" +password = "password" "#; @@ -76,8 +81,9 @@ pub async fn blob_tests() { // Init store //let store: Store = SqliteStore::open( - let store: Store = PostgresStore::open( - //let store: Store = FdbStore::open( + //let store: Store = FdbStore::open( + //let store: Store = PostgresStore::open( + let store: Store = MysqlStore::open( &Config::new(&CONFIG_DB.replace("{TMP}", temp_dir.path.as_path().to_str().unwrap())) .unwrap(), ) diff --git a/tests/src/store/mod.rs b/tests/src/store/mod.rs index 429e3434..68a77df6 100644 --- a/tests/src/store/mod.rs +++ b/tests/src/store/mod.rs @@ -29,7 +29,9 @@ use std::io::Read; use ::store::Store; -use store::backend::{foundationdb::FdbStore, postgres::PostgresStore, sqlite::SqliteStore}; +use store::backend::{ + foundationdb::FdbStore, mysql::MysqlStore, postgres::PostgresStore, sqlite::SqliteStore, +}; use utils::config::Config; pub struct TempDir { @@ -44,10 +46,13 @@ local.path = "PATH" [store.db] #path = "PATH/sqlite.db" host = "localhost" -post = 5432 +#port = 5432 +port = 3307 database = "stalwart" -user = "postgres" -password = "mysecretpassword" +#user = "postgres" +#password = "mysecretpassword" +user = "root" +password = "password" "#; @@ -58,14 +63,15 @@ pub async fn store_tests() { let config_file = CONFIG.replace("PATH", &temp_dir.path.to_string_lossy()); //let db: Store = SqliteStore::open(&Config::new(&config_file).unwrap()) //let db: Store = FdbStore::open(&Config::new(&config_file).unwrap()) - let db: Store = PostgresStore::open(&Config::new(&config_file).unwrap()) + //let db: Store = PostgresStore::open(&Config::new(&config_file).unwrap()) + let db: Store = MysqlStore::open(&Config::new(&config_file).unwrap()) .await .unwrap() .into(); if insert { db.destroy().await; } - query::test(db.clone(), insert).await; + //query::test(db.clone(), insert).await; assign_id::test(db).await; temp_dir.delete(); }