From e4d5bde1ce5f4180218460078e43f6bc656af0ba Mon Sep 17 00:00:00 2001 From: mdecimus Date: Thu, 7 Dec 2023 16:27:19 +0100 Subject: [PATCH] Directory/Store backend unification + LDAP auth bind --- Cargo.lock | 498 ++----------- crates/benchy/Cargo.toml | 17 - crates/benchy/benches/bitmap.rs | 538 -------------- crates/benchy/src/main.rs | 3 - crates/directory/Cargo.toml | 7 +- crates/directory/src/cache/lookup.rs | 14 +- crates/directory/src/config.rs | 339 +-------- crates/directory/src/imap/config.rs | 11 +- crates/directory/src/imap/lookup.rs | 14 +- crates/directory/src/imap/mod.rs | 7 +- crates/directory/src/imap/pool.rs | 29 +- crates/directory/src/ldap/config.rs | 8 + crates/directory/src/ldap/lookup.rs | 129 ++-- crates/directory/src/ldap/mod.rs | 3 +- crates/directory/src/ldap/pool.rs | 30 +- crates/directory/src/lib.rs | 355 +-------- crates/directory/src/memory/config.rs | 13 +- crates/directory/src/memory/lookup.rs | 14 +- crates/directory/src/memory/mod.rs | 6 +- crates/directory/src/scheduled.rs | 60 -- crates/directory/src/smtp/config.rs | 11 +- crates/directory/src/smtp/lookup.rs | 14 +- crates/directory/src/smtp/mod.rs | 7 +- crates/directory/src/smtp/pool.rs | 40 +- crates/directory/src/sql/config.rs | 74 +- crates/directory/src/sql/lookup.rs | 257 +++---- crates/directory/src/sql/mod.rs | 6 +- crates/install/Cargo.toml | 2 +- crates/jmap/Cargo.toml | 1 - crates/jmap/src/lib.rs | 75 +- crates/main/Cargo.toml | 2 +- crates/main/src/main.rs | 17 +- crates/smtp/Cargo.toml | 2 +- crates/smtp/src/config/condition.rs | 4 +- crates/smtp/src/config/mod.rs | 10 +- crates/smtp/src/config/scripts.rs | 10 +- crates/smtp/src/core/mod.rs | 109 ++- crates/smtp/src/lib.rs | 7 +- crates/smtp/src/scripts/event_loop.rs | 9 +- crates/smtp/src/scripts/plugins/bayes.rs | 12 +- crates/smtp/src/scripts/plugins/lookup.rs | 18 +- crates/smtp/src/scripts/plugins/query.rs | 89 ++- crates/store/Cargo.toml | 9 +- crates/store/src/backend/elastic/mod.rs | 45 +- crates/store/src/backend/foundationdb/main.rs | 31 +- crates/store/src/backend/fs/mod.rs | 15 +- crates/store/src/backend/memory/glob.rs | 127 ++++ crates/store/src/backend/memory/lookup.rs | 114 +++ crates/store/src/backend/memory/main.rs | 298 ++++++++ crates/store/src/backend/memory/mod.rs | 52 ++ crates/store/src/backend/mod.rs | 1 + crates/store/src/backend/mysql/lookup.rs | 153 ++++ crates/store/src/backend/mysql/main.rs | 28 +- crates/store/src/backend/mysql/mod.rs | 1 + crates/store/src/backend/postgres/lookup.rs | 218 ++++++ crates/store/src/backend/postgres/main.rs | 23 +- crates/store/src/backend/postgres/mod.rs | 1 + crates/store/src/backend/rocksdb/main.rs | 16 +- crates/store/src/backend/s3/mod.rs | 26 +- crates/store/src/backend/sqlite/lookup.rs | 159 +++++ crates/store/src/backend/sqlite/main.rs | 14 +- crates/store/src/backend/sqlite/mod.rs | 1 + crates/store/src/config.rs | 212 ++++++ crates/store/src/dispatch.rs | 100 ++- crates/store/src/lib.rs | 328 ++++++++- crates/utils/src/config/utils.rs | 12 + tests/Cargo.toml | 3 +- .../resources/smtp/config/rules-dynvalue.toml | 31 +- tests/resources/smtp/config/rules-eval.toml | 7 +- tests/src/directory/imap.rs | 10 +- tests/src/directory/ldap.rs | 13 +- tests/src/directory/mod.rs | 172 ++++- tests/src/directory/smtp.rs | 8 +- tests/src/directory/sql.rs | 674 ++++++++++-------- tests/src/imap/mod.rs | 147 ++-- tests/src/jmap/auth_acl.rs | 57 +- tests/src/jmap/auth_limits.rs | 32 +- tests/src/jmap/auth_oauth.rs | 30 +- tests/src/jmap/blob.rs | 30 +- tests/src/jmap/crypto.rs | 29 +- tests/src/jmap/delivery.rs | 65 +- tests/src/jmap/email_changes.rs | 23 +- tests/src/jmap/email_copy.rs | 33 +- tests/src/jmap/email_get.rs | 27 +- tests/src/jmap/email_parse.rs | 34 +- tests/src/jmap/email_query.rs | 11 +- tests/src/jmap/email_query_changes.rs | 23 +- tests/src/jmap/email_search_snippet.rs | 20 +- tests/src/jmap/email_set.rs | 17 +- tests/src/jmap/email_submission.rs | 22 +- tests/src/jmap/event_source.rs | 34 +- tests/src/jmap/mailbox.rs | 9 +- tests/src/jmap/mod.rs | 148 ++-- tests/src/jmap/push_subscription.rs | 19 +- tests/src/jmap/quota.rs | 44 +- tests/src/jmap/sieve_script.rs | 30 +- tests/src/jmap/thread_get.rs | 23 +- tests/src/jmap/thread_merge.rs | 11 +- tests/src/jmap/vacation_response.rs | 36 +- tests/src/jmap/websocket.rs | 24 +- tests/src/smtp/config.rs | 39 +- tests/src/smtp/inbound/antispam.rs | 46 +- tests/src/smtp/inbound/auth.rs | 6 +- tests/src/smtp/inbound/data.rs | 8 +- tests/src/smtp/inbound/dmarc.rs | 8 +- tests/src/smtp/inbound/rcpt.rs | 8 +- tests/src/smtp/inbound/rewrite.rs | 3 +- tests/src/smtp/inbound/scripts.rs | 19 +- tests/src/smtp/inbound/sign.rs | 9 +- tests/src/smtp/inbound/vrfy.rs | 6 +- tests/src/smtp/lookup/sql.rs | 75 +- tests/src/smtp/management/queue.rs | 6 +- tests/src/smtp/management/report.rs | 6 +- tests/src/smtp/mod.rs | 1 + tests/src/store/blob.rs | 651 ++++++++--------- tests/src/store/mod.rs | 78 +- 116 files changed, 4086 insertions(+), 3634 deletions(-) delete mode 100644 crates/benchy/Cargo.toml delete mode 100644 crates/benchy/benches/bitmap.rs delete mode 100644 crates/benchy/src/main.rs delete mode 100644 crates/directory/src/scheduled.rs create mode 100644 crates/store/src/backend/memory/glob.rs create mode 100644 crates/store/src/backend/memory/lookup.rs create mode 100644 crates/store/src/backend/memory/main.rs create mode 100644 crates/store/src/backend/memory/mod.rs create mode 100644 crates/store/src/backend/mysql/lookup.rs create mode 100644 crates/store/src/backend/postgres/lookup.rs create mode 100644 crates/store/src/backend/sqlite/lookup.rs create mode 100644 crates/store/src/config.rs diff --git a/Cargo.lock b/Cargo.lock index 8e1fa843..cc678230 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -145,30 +145,30 @@ checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] name = "anstyle-parse" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +checksum = "a3a318f1f38d2418400f8209655bfd825785afd25aa30bb7ba6cc792e4596748" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.1" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" dependencies = [ "anstyle", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -306,15 +306,6 @@ dependencies = [ "syn 2.0.39", ] -[[package]] -name = "atoi" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" -dependencies = [ - "num-traits", -] - [[package]] name = "attohttpc" version = "0.22.0" @@ -453,19 +444,6 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" -[[package]] -name = "bb8" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98b4b0f25f18bcdc3ac72bdb486ed0acf7e185221fd4dc985bc15db5800b0ba2" -dependencies = [ - "async-trait", - "futures-channel", - "futures-util", - "parking_lot", - "tokio", -] - [[package]] name = "bigdecimal" version = "0.4.2" @@ -557,9 +535,6 @@ name = "bitflags" version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" -dependencies = [ - "serde", -] [[package]] name = "bitvec" @@ -899,9 +874,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.10" +version = "4.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fffed7514f420abec6d183b1d3acfd9099c79c3a10a06ade4f8203f1411272" +checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" dependencies = [ "clap_builder", "clap_derive", @@ -909,9 +884,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.9" +version = "4.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63361bae7eef3771745f02d8d892bec2fee5f6e34af316ba556e7f97a7069ff1" +checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" dependencies = [ "anstream", "anstyle", @@ -1025,21 +1000,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" - [[package]] name = "crc32fast" version = "1.3.2" @@ -1147,9 +1107,9 @@ dependencies = [ [[package]] name = "crypto-mac" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6" +checksum = "bff07008ec701e8028e2ceb8f83f0e4274ee62bd2dbdc4fefff2e9a91824081a" dependencies = [ "generic-array", "subtle", @@ -1441,8 +1401,7 @@ dependencies = [ "ahash 0.8.6", "argon2", "async-trait", - "bb8", - "flate2", + "deadpool", "futures", "ldap3", "lru-cache", @@ -1455,14 +1414,12 @@ dependencies = [ "pbkdf2 0.12.2", "pwhash", "regex", - "reqwest", "rustls 0.21.9", "scrypt", "sha1", "sha2 0.10.8", - "sieve-rs", "smtp-proto", - "sqlx", + "store", "tokio", "tokio-rustls", "tracing", @@ -1533,12 +1490,6 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" -[[package]] -name = "dotenvy" -version = "0.15.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" - [[package]] name = "dsa" version = "0.6.2" @@ -1645,9 +1596,6 @@ name = "either" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" -dependencies = [ - "serde", -] [[package]] name = "elasticsearch" @@ -1748,29 +1696,18 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "etcetera" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" -dependencies = [ - "cfg-if", - "home", - "windows-sys 0.48.0", -] - -[[package]] -name = "event-listener" -version = "2.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" - [[package]] name = "fallible-iterator" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + [[package]] name = "fallible-streaming-iterator" version = "0.1.9" @@ -1823,14 +1760,14 @@ checksum = "27573eac26f4dd11e2b1916c3fe1baa56407c83c71a773a8ba17ec0bca03b6b7" [[package]] name = "filetime" -version = "0.2.22" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" +checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.3.5", - "windows-sys 0.48.0", + "redox_syscall", + "windows-sys 0.52.0", ] [[package]] @@ -1856,17 +1793,6 @@ dependencies = [ "miniz_oxide", ] -[[package]] -name = "flume" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" -dependencies = [ - "futures-core", - "futures-sink", - "spin 0.9.8", -] - [[package]] name = "fnv" version = "1.0.7" @@ -2064,17 +1990,6 @@ dependencies = [ "futures-util", ] -[[package]] -name = "futures-intrusive" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" -dependencies = [ - "futures-core", - "lock_api", - "parking_lot", -] - [[package]] name = "futures-io" version = "0.3.29" @@ -2269,9 +2184,6 @@ name = "heck" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -dependencies = [ - "unicode-segmentation", -] [[package]] name = "hermit-abi" @@ -2747,15 +2659,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.9" @@ -2842,7 +2745,6 @@ dependencies = [ "sieve-rs", "smtp", "smtp-proto", - "sqlx", "store", "tokio", "tokio-tungstenite", @@ -2915,9 +2817,9 @@ dependencies = [ [[package]] name = "konst" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29a6ee015a18f4a121ba670f947f801b207139f0f810b3cd266e319065129782" +checksum = "8d712a8c49d4274f8d8a5cf61368cb5f3c143d149882b1a2918129e53395fdb0" dependencies = [ "const_panic", "konst_kernel", @@ -2926,9 +2828,9 @@ dependencies = [ [[package]] name = "konst_kernel" -version = "0.3.6" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3376133edc39f027d551eb77b077c2865a0ef252b2e7d0dd6b6dc303db95d8b5" +checksum = "dac6ea8c376b6e208a81cf39b8e82bebf49652454d98a4829e907dac16ef1790" dependencies = [ "typewit", ] @@ -2944,7 +2846,7 @@ dependencies = [ "diff", "ena", "is-terminal", - "itertools 0.10.5", + "itertools", "lalrpop-util", "petgraph", "regex", @@ -3043,7 +2945,7 @@ checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" dependencies = [ "bitflags 2.4.1", "libc", - "redox_syscall 0.4.1", + "redox_syscall", ] [[package]] @@ -3064,9 +2966,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326" +checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" dependencies = [ "cc", "pkg-config", @@ -3162,7 +3064,7 @@ dependencies = [ "mail-parser", "parking_lot", "quick-xml 0.30.0", - "ring 0.17.6", + "ring 0.17.7", "rustls-pemfile 1.0.4", "serde", "serde_json", @@ -3371,9 +3273,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" dependencies = [ "libc", "log", @@ -3650,9 +3552,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.60" +version = "0.10.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800" +checksum = "6b8419dc8cc6d866deb801274bba2e6f8f6108c1bb7fcc10ee5ab864931dbb45" dependencies = [ "bitflags 2.4.1", "cfg-if", @@ -3691,9 +3593,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.96" +version = "0.9.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f" +checksum = "c3eaad34cdd97d81de97964fc7f29e2d104f483840d906ef56daa1912338460b" dependencies = [ "cc", "libc", @@ -3797,9 +3699,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "4.1.1" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "536900a8093134cf9ccf00a27deb3532421099e958d9dd431135d0c7543ca1e8" +checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e" dependencies = [ "num-traits", ] @@ -3850,7 +3752,7 @@ checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.4.1", + "redox_syscall", "smallvec", "windows-targets 0.48.5", ] @@ -3877,12 +3779,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "paste" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" - [[package]] name = "pbkdf2" version = "0.11.0" @@ -4088,9 +3984,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" [[package]] name = "postgres-protocol" @@ -4101,7 +3997,7 @@ dependencies = [ "base64 0.21.5", "byteorder", "bytes", - "fallible-iterator", + "fallible-iterator 0.2.0", "hmac 0.12.1", "md-5 0.10.6", "memchr", @@ -4117,7 +4013,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d2234cdee9408b523530a9b6d2d6b373d1db34f6a8e51dc03ded1828d7fb67c" dependencies = [ "bytes", - "fallible-iterator", + "fallible-iterator 0.2.0", "postgres-protocol", ] @@ -4251,7 +4147,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools", "proc-macro2", "quote", "syn 1.0.109", @@ -4412,7 +4308,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48d1fd02e16232e942b5e7ce305b447c550d09a9146255a3e8a2cf62a0e2ac2d" dependencies = [ "either", - "itertools 0.10.5", + "itertools", "proc-macro2", "quote", "rayon", @@ -4449,15 +4345,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "redox_syscall" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "redox_syscall" version = "0.4.1" @@ -4624,9 +4511,9 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.6" +version = "0.17.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "684d5e6e18f669ccebf64a92236bb7db9a34f07be010e3627368182027180866" +checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" dependencies = [ "cc", "getrandom", @@ -4737,12 +4624,12 @@ dependencies = [ [[package]] name = "rusqlite" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2" +checksum = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d" dependencies = [ "bitflags 2.4.1", - "fallible-iterator", + "fallible-iterator 0.3.0", "fallible-streaming-iterator", "hashlink", "libsqlite3-sys", @@ -4888,7 +4775,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" dependencies = [ "log", - "ring 0.17.6", + "ring 0.17.7", "rustls-webpki", "sct", ] @@ -4926,9 +4813,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb0a1f9b9efec70d32e6d6aa3e58ebd88c3754ec98dfe9145c63cf54cc829b83" +checksum = "e7673e0aa20ee4937c6aacfc12bb8341cfbf054cdd21df6bec5fd0629fe9339b" [[package]] name = "rustls-webpki" @@ -4936,7 +4823,7 @@ version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.17.6", + "ring 0.17.7", "untrusted 0.9.0", ] @@ -5009,7 +4896,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.17.6", + "ring 0.17.7", "untrusted 0.9.0", ] @@ -5414,7 +5301,7 @@ dependencies = [ "sha2 0.10.8", "sieve-rs", "smtp-proto", - "sqlx", + "store", "tokio", "tokio-rustls", "tracing", @@ -5484,9 +5371,6 @@ name = "spin" version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -dependencies = [ - "lock_api", -] [[package]] name = "spki" @@ -5498,215 +5382,6 @@ dependencies = [ "der", ] -[[package]] -name = "sqlformat" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7b278788e7be4d0d29c0f39497a0eef3fba6bbc8e70d8bf7fde46edeaa9e85" -dependencies = [ - "itertools 0.11.0", - "nom", - "unicode_categories", -] - -[[package]] -name = "sqlx" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e50c216e3624ec8e7ecd14c6a6a6370aad6ee5d8cfc3ab30b5162eeeef2ed33" -dependencies = [ - "sqlx-core", - "sqlx-macros", - "sqlx-mysql", - "sqlx-postgres", - "sqlx-sqlite", -] - -[[package]] -name = "sqlx-core" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d6753e460c998bbd4cd8c6f0ed9a64346fcca0723d6e75e52fdc351c5d2169d" -dependencies = [ - "ahash 0.8.6", - "atoi", - "byteorder", - "bytes", - "crc", - "crossbeam-queue", - "dotenvy", - "either", - "event-listener", - "futures-channel", - "futures-core", - "futures-intrusive", - "futures-io", - "futures-util", - "hashlink", - "hex", - "indexmap 2.1.0", - "log", - "memchr", - "once_cell", - "paste", - "percent-encoding", - "rustls 0.21.9", - "rustls-pemfile 1.0.4", - "serde", - "serde_json", - "sha2 0.10.8", - "smallvec", - "sqlformat", - "thiserror", - "tokio", - "tokio-stream", - "tracing", - "url", - "webpki-roots 0.24.0", -] - -[[package]] -name = "sqlx-macros" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a793bb3ba331ec8359c1853bd39eed32cdd7baaf22c35ccf5c92a7e8d1189ec" -dependencies = [ - "proc-macro2", - "quote", - "sqlx-core", - "sqlx-macros-core", - "syn 1.0.109", -] - -[[package]] -name = "sqlx-macros-core" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a4ee1e104e00dedb6aa5ffdd1343107b0a4702e862a84320ee7cc74782d96fc" -dependencies = [ - "dotenvy", - "either", - "heck", - "hex", - "once_cell", - "proc-macro2", - "quote", - "serde", - "serde_json", - "sha2 0.10.8", - "sqlx-core", - "sqlx-mysql", - "sqlx-postgres", - "sqlx-sqlite", - "syn 1.0.109", - "tempfile", - "tokio", - "url", -] - -[[package]] -name = "sqlx-mysql" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "864b869fdf56263f4c95c45483191ea0af340f9f3e3e7b4d57a61c7c87a970db" -dependencies = [ - "atoi", - "base64 0.21.5", - "bitflags 2.4.1", - "byteorder", - "bytes", - "crc", - "digest 0.10.7", - "dotenvy", - "either", - "futures-channel", - "futures-core", - "futures-io", - "futures-util", - "generic-array", - "hex", - "hkdf", - "hmac 0.12.1", - "itoa", - "log", - "md-5 0.10.6", - "memchr", - "once_cell", - "percent-encoding", - "rand", - "rsa", - "serde", - "sha1", - "sha2 0.10.8", - "smallvec", - "sqlx-core", - "stringprep", - "thiserror", - "tracing", - "whoami", -] - -[[package]] -name = "sqlx-postgres" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb7ae0e6a97fb3ba33b23ac2671a5ce6e3cabe003f451abd5a56e7951d975624" -dependencies = [ - "atoi", - "base64 0.21.5", - "bitflags 2.4.1", - "byteorder", - "crc", - "dotenvy", - "etcetera", - "futures-channel", - "futures-core", - "futures-io", - "futures-util", - "hex", - "hkdf", - "hmac 0.12.1", - "home", - "itoa", - "log", - "md-5 0.10.6", - "memchr", - "once_cell", - "rand", - "serde", - "serde_json", - "sha1", - "sha2 0.10.8", - "smallvec", - "sqlx-core", - "stringprep", - "thiserror", - "tracing", - "whoami", -] - -[[package]] -name = "sqlx-sqlite" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59dc83cf45d89c555a577694534fcd1b55c545a816c816ce51f20bbe56a4f3f" -dependencies = [ - "atoi", - "flume", - "futures-channel", - "futures-core", - "futures-executor", - "futures-intrusive", - "futures-util", - "libsqlite3-sys", - "log", - "percent-encoding", - "serde", - "sqlx-core", - "tracing", - "url", -] - [[package]] name = "stalwart-cli" version = "0.4.2" @@ -5760,10 +5435,13 @@ name = "store" version = "0.1.0" dependencies = [ "ahash 0.8.6", + "async-trait", "blake3", + "bytes", "deadpool-postgres", "elasticsearch", "farmhash", + "flate2", "foundationdb", "futures", "lazy_static", @@ -5776,7 +5454,9 @@ dependencies = [ "r2d2", "rand", "rayon", - "ring 0.17.6", + "regex", + "reqwest", + "ring 0.17.7", "roaring", "rocksdb", "rusqlite", @@ -5834,9 +5514,9 @@ dependencies = [ [[package]] name = "subtle" -version = "2.5.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" @@ -5936,7 +5616,7 @@ checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" dependencies = [ "cfg-if", "fastrand", - "redox_syscall 0.4.1", + "redox_syscall", "rustix", "windows-sys 0.48.0", ] @@ -6001,7 +5681,6 @@ dependencies = [ "sieve-rs", "smtp", "smtp-proto", - "sqlx", "store", "tokio", "tokio-rustls", @@ -6152,7 +5831,7 @@ dependencies = [ "async-trait", "byteorder", "bytes", - "fallible-iterator", + "fallible-iterator 0.2.0", "futures-channel", "futures-util", "log", @@ -6529,12 +6208,6 @@ dependencies = [ "unicode-script", ] -[[package]] -name = "unicode-segmentation" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" - [[package]] name = "unicode-width" version = "0.1.11" @@ -6547,12 +6220,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" -[[package]] -name = "unicode_categories" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" - [[package]] name = "universal-hash" version = "0.5.1" @@ -6786,7 +6453,7 @@ version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" dependencies = [ - "ring 0.17.6", + "ring 0.17.7", "untrusted 0.9.0", ] @@ -6799,15 +6466,6 @@ dependencies = [ "webpki", ] -[[package]] -name = "webpki-roots" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b291546d5d9d1eab74f069c77749f2cb8504a12caa20f0f2de93ddbf6f411888" -dependencies = [ - "rustls-webpki", -] - [[package]] name = "webpki-roots" version = "0.25.3" @@ -7101,9 +6759,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.19" +version = "0.5.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b" +checksum = "b7e87b8dfbe3baffbe687eef2e164e32286eff31a5ee16463ce03d991643ec94" dependencies = [ "memchr", ] @@ -7178,18 +6836,18 @@ checksum = "9828b178da53440fa9c766a3d2f73f7cf5d0ac1fe3980c1e5018d899fd19e07b" [[package]] name = "zerocopy" -version = "0.7.28" +version = "0.7.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d6f15f7ade05d2a4935e34a457b936c23dc70a05cc1d97133dc99e7a3fe0f0e" +checksum = "5d075cf85bbb114e933343e087b92f2146bac0d55b534cbb8188becf0039948e" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.28" +version = "0.7.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbbad221e3f78500350ecbd7dfa4e63ef945c05f4c61cb7f4d3f84cd0bba649b" +checksum = "86cd5ca076997b97ef09d3ad65efe811fa68c9e874cb636ccb211223a813b0c2" dependencies = [ "proc-macro2", "quote", diff --git a/crates/benchy/Cargo.toml b/crates/benchy/Cargo.toml deleted file mode 100644 index c616901d..00000000 --- a/crates/benchy/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "benchy" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -rusqlite = { version = "0.29.0", features = ["bundled"] } -roaring = "0.10.1" - -[dev-dependencies] -criterion = "0.5.1" - -[[bench]] -name = "bitmap" -harness = false diff --git a/crates/benchy/benches/bitmap.rs b/crates/benchy/benches/bitmap.rs deleted file mode 100644 index ad1f1b2e..00000000 --- a/crates/benchy/benches/bitmap.rs +++ /dev/null @@ -1,538 +0,0 @@ -use criterion::{criterion_group, criterion_main, Criterion}; -use roaring::RoaringBitmap; -use rusqlite::{params, Connection, OpenFlags, OptionalExtension, TransactionBehavior}; -use std::path::PathBuf; - -// Functions to setup the database with the different layouts -// ... - -// Functions to insert data into each layout -#[inline(always)] -fn insert_into_layout1(conn: &mut Connection) { - conn.prepare_cached("DELETE FROM l1") - .unwrap() - .execute([]) - .unwrap(); - - let mut bitmap_block_num; - let mut bitmap_col_num; - let mut bitmap_value_set; - let trx = conn - .transaction_with_behavior(TransactionBehavior::Immediate) - .unwrap(); - - for document_id in 0u32..100_000u32 { - bitmap_block_num = document_id / BITS_PER_BLOCK; - let index = document_id & BITS_MASK; - bitmap_col_num = (index / 64) as usize; - bitmap_value_set = (1u64 << (index as u64 & 63)) as i64; - - for key in [b"key1", b"key2"] { - if key == b"key2" && document_id % 2 == 0 { - continue; - } - let mut key = key.to_vec(); - key.extend_from_slice(bitmap_block_num.to_be_bytes().as_ref()); - - trx.prepare_cached(SET_QUERIES[bitmap_col_num]) - .unwrap() - .execute(params![bitmap_value_set, &key]) - .unwrap(); - if trx.changes() == 0 { - trx.prepare_cached(INSERT_QUERIES[bitmap_col_num]) - .unwrap() - .execute(params![&key, bitmap_value_set]) - .unwrap(); - } - } - } - - trx.commit().unwrap(); -} - -#[inline(always)] -fn insert_into_layout1a(conn: &mut Connection) { - conn.prepare_cached("DELETE FROM l1a") - .unwrap() - .execute([]) - .unwrap(); - - let mut bitmap_block_num; - let mut bitmap_col_num; - let mut bitmap_value_set; - let trx = conn - .transaction_with_behavior(TransactionBehavior::Immediate) - .unwrap(); - - for document_id in 0u32..100_000u32 { - bitmap_block_num = document_id / BITS_PER_BLOCK; - let index = document_id & BITS_MASK; - bitmap_col_num = (index / 64) as usize; - bitmap_value_set = (1u64 << (index as u64 & 63)) as i64; - - for key in [b"key1", b"key2"] { - if key == b"key2" && document_id % 2 == 0 { - continue; - } - let mut block = Vec::new(); - block.extend_from_slice(bitmap_block_num.to_be_bytes().as_ref()); - - trx.prepare_cached(SET_QUERIES2[bitmap_col_num]) - .unwrap() - .execute(params![bitmap_value_set, &key, &block]) - .unwrap(); - if trx.changes() == 0 { - trx.prepare_cached(INSERT_QUERIES2[bitmap_col_num]) - .unwrap() - .execute(params![&key, &block, bitmap_value_set]) - .unwrap(); - } - } - } - - trx.commit().unwrap(); -} - -#[inline(always)] -fn insert_into_layout2(conn: &mut Connection) { - conn.prepare_cached("DELETE FROM l2") - .unwrap() - .execute([]) - .unwrap(); - - let trx = conn - .transaction_with_behavior(TransactionBehavior::Immediate) - .unwrap(); - - for document_id in 0u32..100_000u32 { - for key in [b"key1", b"key2"] { - if key == b"key2" && document_id % 2 == 0 { - continue; - } - - let bm = trx - .prepare_cached("SELECT v FROM l2 WHERE k = ?") - .unwrap() - .query_row([&key], |row| { - Ok( - RoaringBitmap::deserialize_unchecked_from(row.get_ref(0)?.as_bytes()?) - .unwrap(), - ) - }) - .optional() - .unwrap(); - - if let Some(mut bm) = bm { - bm.insert(document_id); - let mut buf = Vec::with_capacity(bm.serialized_size()); - bm.serialize_into(&mut buf).unwrap(); - - trx.prepare_cached("UPDATE l2 SET v = ? WHERE k = ?") - .unwrap() - .execute(params![&buf, key]) - .unwrap(); - } else { - let mut bm = RoaringBitmap::new(); - bm.insert(document_id); - let mut buf = Vec::with_capacity(bm.serialized_size()); - bm.serialize_into(&mut buf).unwrap(); - trx.prepare_cached("INSERT INTO l2 (k, v) VALUES (?, ?)") - .unwrap() - .execute(params![&key, buf]) - .unwrap(); - } - } - } - - trx.commit().unwrap(); -} - -#[inline(always)] -fn insert_into_layout3(conn: &mut Connection) { - conn.prepare_cached("DELETE FROM l3") - .unwrap() - .execute([]) - .unwrap(); - let trx = conn - .transaction_with_behavior(TransactionBehavior::Immediate) - .unwrap(); - - for document_id in 0u32..100_000u32 { - for key in [b"key1", b"key2"] { - if key == b"key2" && document_id % 2 == 0 { - continue; - } - let mut key = key.to_vec(); - key.extend_from_slice(document_id.to_be_bytes().as_ref()); - - trx.prepare_cached("INSERT INTO l3 (k) VALUES (?)") - .unwrap() - .execute(params![key]) - .unwrap(); - } - } - - trx.commit().unwrap(); -} - -// Functions to query each layout -#[inline(always)] -fn query_layout1(conn: &Connection) { - for (pos, key) in [b"key1", b"key2"].into_iter().enumerate() { - let mut begin = key.to_vec(); - begin.extend_from_slice(0u32.to_be_bytes().as_ref()); - let key_len = begin.len(); - let mut end = key.to_vec(); - end.extend_from_slice(u32::MAX.to_be_bytes().as_ref()); - let mut query = conn - .prepare_cached("SELECT z, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p FROM l1 WHERE z >= ? AND z <= ?").unwrap(); - let mut rows = query.query([&begin, &end]).unwrap(); - - let mut bm = roaring::RoaringBitmap::new(); - while let Some(row) = rows.next().unwrap() { - let key = row.get_ref(0).unwrap().as_bytes().unwrap(); - if key.len() == key_len { - let block_num = deserialize_be_u32(key, key.len() - std::mem::size_of::()); - - for word_num in 0..WORDS_PER_BLOCK { - match row.get::<_, i64>((word_num + 1) as usize).unwrap() as u64 { - 0 => (), - u64::MAX => { - bm.insert_range( - block_num * BITS_PER_BLOCK + word_num * WORD_SIZE_BITS - ..(block_num * BITS_PER_BLOCK + word_num * WORD_SIZE_BITS) - + WORD_SIZE_BITS, - ); - } - mut word => { - while word != 0 { - let trailing_zeros = word.trailing_zeros(); - bm.insert( - block_num * BITS_PER_BLOCK - + word_num * WORD_SIZE_BITS - + trailing_zeros, - ); - word ^= 1 << trailing_zeros; - } - } - } - } - } - } - - assert_eq!(bm.len(), 100_000u64 / std::cmp::max(1, pos as u64 * 2)); - } -} - -#[inline(always)] -fn query_layout1a(conn: &Connection) { - for (pos, key) in [b"key1", b"key2"].into_iter().enumerate() { - let mut query = conn - .prepare_cached( - "SELECT y, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p FROM l1 WHERE z = ?", - ) - .unwrap(); - let mut rows = query.query([&key]).unwrap(); - - let mut bm = roaring::RoaringBitmap::new(); - while let Some(row) = rows.next().unwrap() { - let block_num = deserialize_be_u32(row.get_ref(0).unwrap().as_bytes().unwrap(), 0); - - for word_num in 0..WORDS_PER_BLOCK { - match row.get::<_, i64>((word_num + 1) as usize).unwrap() as u64 { - 0 => (), - u64::MAX => { - bm.insert_range( - block_num * BITS_PER_BLOCK + word_num * WORD_SIZE_BITS - ..(block_num * BITS_PER_BLOCK + word_num * WORD_SIZE_BITS) - + WORD_SIZE_BITS, - ); - } - mut word => { - while word != 0 { - let trailing_zeros = word.trailing_zeros(); - bm.insert( - block_num * BITS_PER_BLOCK - + word_num * WORD_SIZE_BITS - + trailing_zeros, - ); - word ^= 1 << trailing_zeros; - } - } - } - } - } - - assert_eq!(bm.len(), 100_000u64 / std::cmp::max(1, pos as u64 * 2)); - } -} - -#[inline(always)] -fn query_layout2(conn: &Connection) { - for (pos, key) in [b"key1", b"key2"].into_iter().enumerate() { - let bm = conn - .prepare_cached("SELECT v FROM l2 WHERE k = ?") - .unwrap() - .query_row([key], |row| { - Ok(RoaringBitmap::deserialize_unchecked_from(row.get_ref(0)?.as_bytes()?).unwrap()) - }) - .optional() - .unwrap() - .unwrap(); - - assert_eq!(bm.len(), 100_000u64 / std::cmp::max(1, pos as u64 * 2)); - } -} - -#[inline(always)] -fn query_layout3(conn: &Connection) { - for (pos, key) in [b"key1", b"key2"].into_iter().enumerate() { - let mut begin = key.to_vec(); - begin.extend_from_slice(0u32.to_be_bytes().as_ref()); - let key_len = begin.len(); - let mut end = key.to_vec(); - end.extend_from_slice(u32::MAX.to_be_bytes().as_ref()); - let mut query = conn - .prepare_cached("SELECT k FROM l3 WHERE k >= ? AND k <= ?") - .unwrap(); - let mut rows = query.query([&begin, &end]).unwrap(); - - let mut bm = roaring::RoaringBitmap::new(); - while let Some(row) = rows.next().unwrap() { - let key = row.get_ref(0).unwrap().as_bytes().unwrap(); - if key.len() == key_len { - bm.insert(deserialize_be_u32( - key, - key.len() - std::mem::size_of::(), - )); - } - } - - assert_eq!(bm.len(), 100_000u64 / std::cmp::max(1, pos as u64 * 2)); - } -} - -// Criterion benchmarks -pub fn insertion_benchmark(c: &mut Criterion) { - let path = PathBuf::from("/tmp/benchy.sqlite3"); - if path.exists() { - std::fs::remove_file(&path).unwrap(); - } - - let mut conn = Connection::open_with_flags(path, OpenFlags::default()).unwrap(); - let mut group = c.benchmark_group("SQLite Layouts Insertion"); - group.measurement_time(std::time::Duration::new(15, 0)); - group.sample_size(10); - - conn.execute_batch(concat!( - "PRAGMA journal_mode = WAL; ", - "PRAGMA synchronous = NORMAL; ", - "PRAGMA temp_store = memory;", - "PRAGMA busy_timeout = 30000;" - )) - .unwrap(); - - // Setup each layout and benchmark insertion - conn.execute( - "CREATE TABLE IF NOT EXISTS l1 ( - z BLOB PRIMARY KEY, - a INTEGER NOT NULL DEFAULT 0, - b INTEGER NOT NULL DEFAULT 0, - c INTEGER NOT NULL DEFAULT 0, - d INTEGER NOT NULL DEFAULT 0, - e INTEGER NOT NULL DEFAULT 0, - f INTEGER NOT NULL DEFAULT 0, - g INTEGER NOT NULL DEFAULT 0, - h INTEGER NOT NULL DEFAULT 0, - i INTEGER NOT NULL DEFAULT 0, - j INTEGER NOT NULL DEFAULT 0, - k INTEGER NOT NULL DEFAULT 0, - l INTEGER NOT NULL DEFAULT 0, - m INTEGER NOT NULL DEFAULT 0, - n INTEGER NOT NULL DEFAULT 0, - o INTEGER NOT NULL DEFAULT 0, - p INTEGER NOT NULL DEFAULT 0 - )", - [], - ) - .unwrap(); - - conn.execute( - "CREATE TABLE IF NOT EXISTS l1a ( - z BLOB NOT NULL, - y BLOB NOT NULL, - a INTEGER NOT NULL DEFAULT 0, - b INTEGER NOT NULL DEFAULT 0, - c INTEGER NOT NULL DEFAULT 0, - d INTEGER NOT NULL DEFAULT 0, - e INTEGER NOT NULL DEFAULT 0, - f INTEGER NOT NULL DEFAULT 0, - g INTEGER NOT NULL DEFAULT 0, - h INTEGER NOT NULL DEFAULT 0, - i INTEGER NOT NULL DEFAULT 0, - j INTEGER NOT NULL DEFAULT 0, - k INTEGER NOT NULL DEFAULT 0, - l INTEGER NOT NULL DEFAULT 0, - m INTEGER NOT NULL DEFAULT 0, - n INTEGER NOT NULL DEFAULT 0, - o INTEGER NOT NULL DEFAULT 0, - p INTEGER NOT NULL DEFAULT 0, - PRIMARY KEY (z, y) - )", - [], - ) - .unwrap(); - - conn.execute( - "CREATE TABLE IF NOT EXISTS l2 ( - k BLOB PRIMARY KEY, - v BLOB NOT NULL)", - [], - ) - .unwrap(); - conn.execute( - "CREATE TABLE IF NOT EXISTS l3 ( - k BLOB PRIMARY KEY)", - [], - ) - .unwrap(); - - group.bench_function("Insertion Layout 1", |b| { - b.iter(|| insert_into_layout1(&mut conn)) - }); - - group.bench_function("Insertion Layout 1a", |b| { - b.iter(|| insert_into_layout1a(&mut conn)) - }); - - /*group.bench_function("Insertion Layout 2", |b| { - b.iter(|| insert_into_layout2(&mut conn)) - }); - - group.bench_function("Insertion Layout 3", |b| { - b.iter(|| insert_into_layout3(&mut conn)) - });*/ - - group.finish(); -} - -pub fn query_benchmark(c: &mut Criterion) { - let conn = Connection::open_with_flags("/tmp/benchy.sqlite3", OpenFlags::default()).unwrap(); - conn.execute_batch(concat!( - "PRAGMA journal_mode = WAL; ", - "PRAGMA synchronous = NORMAL; ", - "PRAGMA temp_store = memory;", - "PRAGMA busy_timeout = 30000;" - )) - .unwrap(); - - let mut group = c.benchmark_group("SQLite Layouts Query"); - //group.measurement_time(Duration::new(5, 0)); - //group.sample_size(10); - - // Assume the layouts are already populated with data - // Benchmark querying for each layout - group.bench_function("Query Layout 1", |b| b.iter(|| query_layout1(&conn))); - group.bench_function("Query Layout 1a", |b| b.iter(|| query_layout1(&conn))); - - //group.bench_function("Query Layout 2", |b| b.iter(|| query_layout2(&conn))); - //group.bench_function("Query Layout 3", |b| b.iter(|| query_layout3(&conn))); - - group.finish(); -} - -// Criterion groups -//criterion_group!(insertion_benches, insertion_benchmark); -criterion_group!(query_benches, query_benchmark); -//criterion_main!(insertion_benches, query_benches); -criterion_main!(query_benches); - -const INSERT_QUERIES: &[&str] = &[ - "INSERT INTO l1 (z, a) VALUES (?, ?)", - "INSERT INTO l1 (z, b) VALUES (?, ?)", - "INSERT INTO l1 (z, c) VALUES (?, ?)", - "INSERT INTO l1 (z, d) VALUES (?, ?)", - "INSERT INTO l1 (z, e) VALUES (?, ?)", - "INSERT INTO l1 (z, f) VALUES (?, ?)", - "INSERT INTO l1 (z, g) VALUES (?, ?)", - "INSERT INTO l1 (z, h) VALUES (?, ?)", - "INSERT INTO l1 (z, i) VALUES (?, ?)", - "INSERT INTO l1 (z, j) VALUES (?, ?)", - "INSERT INTO l1 (z, k) VALUES (?, ?)", - "INSERT INTO l1 (z, l) VALUES (?, ?)", - "INSERT INTO l1 (z, m) VALUES (?, ?)", - "INSERT INTO l1 (z, n) VALUES (?, ?)", - "INSERT INTO l1 (z, o) VALUES (?, ?)", - "INSERT INTO l1 (z, p) VALUES (?, ?)", -]; -const SET_QUERIES: &[&str] = &[ - "UPDATE l1 SET a = a | ? WHERE z = ?", - "UPDATE l1 SET b = b | ? WHERE z = ?", - "UPDATE l1 SET c = c | ? WHERE z = ?", - "UPDATE l1 SET d = d | ? WHERE z = ?", - "UPDATE l1 SET e = e | ? WHERE z = ?", - "UPDATE l1 SET f = f | ? WHERE z = ?", - "UPDATE l1 SET g = g | ? WHERE z = ?", - "UPDATE l1 SET h = h | ? WHERE z = ?", - "UPDATE l1 SET i = i | ? WHERE z = ?", - "UPDATE l1 SET j = j | ? WHERE z = ?", - "UPDATE l1 SET k = k | ? WHERE z = ?", - "UPDATE l1 SET l = l | ? WHERE z = ?", - "UPDATE l1 SET m = m | ? WHERE z = ?", - "UPDATE l1 SET n = n | ? WHERE z = ?", - "UPDATE l1 SET o = o | ? WHERE z = ?", - "UPDATE l1 SET p = p | ? WHERE z = ?", -]; - -const INSERT_QUERIES2: &[&str] = &[ - "INSERT INTO l1a (z, y, a) VALUES (?, ?, ?)", - "INSERT INTO l1a (z, y, b) VALUES (?, ?, ?)", - "INSERT INTO l1a (z, y, c) VALUES (?, ?, ?)", - "INSERT INTO l1a (z, y, d) VALUES (?, ?, ?)", - "INSERT INTO l1a (z, y, e) VALUES (?, ?, ?)", - "INSERT INTO l1a (z, y, f) VALUES (?, ?, ?)", - "INSERT INTO l1a (z, y, g) VALUES (?, ?, ?)", - "INSERT INTO l1a (z, y, h) VALUES (?, ?, ?)", - "INSERT INTO l1a (z, y, i) VALUES (?, ?, ?)", - "INSERT INTO l1a (z, y, j) VALUES (?, ?, ?)", - "INSERT INTO l1a (z, y, k) VALUES (?, ?, ?)", - "INSERT INTO l1a (z, y, l) VALUES (?, ?, ?)", - "INSERT INTO l1a (z, y, m) VALUES (?, ?, ?)", - "INSERT INTO l1a (z, y, n) VALUES (?, ?, ?)", - "INSERT INTO l1a (z, y, o) VALUES (?, ?, ?)", - "INSERT INTO l1a (z, y, p) VALUES (?, ?, ?)", -]; -const SET_QUERIES2: &[&str] = &[ - "UPDATE l1a SET a = a | ? WHERE z = ? AND y = ?", - "UPDATE l1a SET b = b | ? WHERE z = ? AND y = ?", - "UPDATE l1a SET c = c | ? WHERE z = ? AND y = ?", - "UPDATE l1a SET d = d | ? WHERE z = ? AND y = ?", - "UPDATE l1a SET e = e | ? WHERE z = ? AND y = ?", - "UPDATE l1a SET f = f | ? WHERE z = ? AND y = ?", - "UPDATE l1a SET g = g | ? WHERE z = ? AND y = ?", - "UPDATE l1a SET h = h | ? WHERE z = ? AND y = ?", - "UPDATE l1a SET i = i | ? WHERE z = ? AND y = ?", - "UPDATE l1a SET j = j | ? WHERE z = ? AND y = ?", - "UPDATE l1a SET k = k | ? WHERE z = ? AND y = ?", - "UPDATE l1a SET l = l | ? WHERE z = ? AND y = ?", - "UPDATE l1a SET m = m | ? WHERE z = ? AND y = ?", - "UPDATE l1a SET n = n | ? WHERE z = ? AND y = ?", - "UPDATE l1a SET o = o | ? WHERE z = ? AND y = ?", - "UPDATE l1a SET p = p | ? WHERE z = ? AND y = ?", -]; - -const WORD_SIZE_BITS: u32 = (WORD_SIZE * 8) as u32; -const WORD_SIZE: usize = std::mem::size_of::(); -const WORDS_PER_BLOCK: u32 = 16; -pub const BITS_PER_BLOCK: u32 = WORD_SIZE_BITS * WORDS_PER_BLOCK; -const BITS_MASK: u32 = BITS_PER_BLOCK - 1; - -fn deserialize_be_u32(bytes: &[u8], index: usize) -> u32 { - u32::from_be_bytes( - bytes - .get(index..index + std::mem::size_of::()) - .unwrap() - .try_into() - .unwrap(), - ) -} diff --git a/crates/benchy/src/main.rs b/crates/benchy/src/main.rs deleted file mode 100644 index e7a11a96..00000000 --- a/crates/benchy/src/main.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - println!("Hello, world!"); -} diff --git a/crates/directory/Cargo.toml b/crates/directory/Cargo.toml index 40bbb17e..11af5e9f 100644 --- a/crates/directory/Cargo.toml +++ b/crates/directory/Cargo.toml @@ -6,17 +6,16 @@ resolver = "2" [dependencies] utils = { path = "../utils" } +store = { path = "../store" } smtp-proto = { git = "https://github.com/stalwartlabs/smtp-proto" } mail-parser = { git = "https://github.com/stalwartlabs/mail-parser", features = ["full_encoding", "serde_support", "ludicrous_mode"] } mail-send = { git = "https://github.com/stalwartlabs/mail-send", default-features = false, features = ["cram-md5", "skip-ehlo"] } mail-builder = { git = "https://github.com/stalwartlabs/mail-builder", features = ["ludicrous_mode"] } -sieve-rs = { git = "https://github.com/stalwartlabs/sieve" } tokio = { version = "1.23", features = ["net"] } tokio-rustls = { version = "0.24.0"} rustls = "0.21.0" -sqlx = { version = "0.7", features = [ "runtime-tokio-rustls", "postgres", "mysql", "sqlite" ] } ldap3 = { version = "0.11.1", default-features = false, features = ["tls-rustls"] } -bb8 = "0.8.1" +deadpool = { version = "0.10.0", features = ["managed"] } parking_lot = "0.12" async-trait = "0.1.68" ahash = { version = "0.8" } @@ -32,8 +31,6 @@ sha2 = "0.10.6" md5 = "0.7.0" futures = "0.3" regex = "1.7.0" -reqwest = { version = "0.11", default-features = false, features = ["rustls-tls-webpki-roots", "blocking"] } -flate2 = "1.0" [dev-dependencies] tokio = { version = "1.23", features = ["full"] } diff --git a/crates/directory/src/cache/lookup.rs b/crates/directory/src/cache/lookup.rs index b4a682a8..3a2c6088 100644 --- a/crates/directory/src/cache/lookup.rs +++ b/crates/directory/src/cache/lookup.rs @@ -23,7 +23,7 @@ use mail_send::Credentials; -use crate::{DatabaseColumn, Directory, Principal}; +use crate::{Directory, Principal}; use super::CachedDirectory; @@ -71,18 +71,6 @@ impl Directory for CachedDirectory { self.inner.expn(address).await } - async fn lookup(&self, query: &str, params: &[DatabaseColumn<'_>]) -> crate::Result { - self.inner.lookup(query, params).await - } - - async fn query( - &self, - query: &str, - params: &[DatabaseColumn<'_>], - ) -> crate::Result>> { - self.inner.query(query, params).await - } - async fn is_local_domain(&self, domain: &str) -> crate::Result { if let Some(result) = { let result = self.cached_domains.lock().get(domain); diff --git a/crates/directory/src/config.rs b/crates/directory/src/config.rs index 4fd581f4..20d3ee59 100644 --- a/crates/directory/src/config.rs +++ b/crates/directory/src/config.rs @@ -21,17 +21,14 @@ * for more details. */ -use bb8::{ManageConnection, Pool}; -use regex::Regex; -use sieve::runtime::{tests::glob::GlobPattern, Variable}; -use std::{ - fs::File, - io::{BufRead, BufReader}, - sync::Arc, - time::Duration, +use deadpool::{ + managed::{Manager, Pool}, + Runtime, }; +use regex::Regex; +use std::time::Duration; +use store::Stores; use utils::config::{ - cron::SimpleCron, utils::{AsKey, ParseValue}, Config, }; @@ -40,25 +37,17 @@ use ahash::AHashMap; use crate::{ imap::ImapDirectory, ldap::LdapDirectory, memory::MemoryDirectory, smtp::SmtpDirectory, - sql::SqlDirectory, AddressMapping, DirectoryConfig, DirectoryOptions, DirectorySchedule, - Lookup, LookupList, MatchType, + sql::SqlDirectory, AddressMapping, Directories, DirectoryOptions, }; pub trait ConfigDirectory { - fn parse_directory(&self) -> utils::config::Result; - fn parse_lookup_list( - &self, - key: K, - format: LookupFormat, - ) -> utils::config::Result; + fn parse_directory(&self, stores: &Stores) -> utils::config::Result; } impl ConfigDirectory for Config { - fn parse_directory(&self) -> utils::config::Result { - let mut config = DirectoryConfig { + fn parse_directory(&self, stores: &Stores) -> utils::config::Result { + let mut config = Directories { directories: AHashMap::new(), - lookups: AHashMap::new(), - schedules: Vec::new(), }; for id in self.sub_keys("directory") { // Parse directory @@ -66,7 +55,7 @@ impl ConfigDirectory for Config { let prefix = ("directory", id); let directory = match protocol { "ldap" => LdapDirectory::from_config(self, prefix)?, - "sql" => SqlDirectory::from_config(self, prefix)?, + "sql" => SqlDirectory::from_config(self, prefix, stores)?, "imap" => ImapDirectory::from_config(self, prefix)?, "smtp" => SmtpDirectory::from_config(self, prefix, false)?, "lmtp" => SmtpDirectory::from_config(self, prefix, true)?, @@ -76,285 +65,11 @@ impl ConfigDirectory for Config { } }; - // Add queries/filters as lookups - let is_directory = ["sql", "ldap"].contains(&protocol); - if is_directory { - let name = if protocol == "sql" { "query" } else { "filter" }; - for lookup_id in self.sub_keys(("directory", id, name)) { - config.lookups.insert( - format!("{id}/{lookup_id}"), - Arc::new(Lookup::Directory { - directory: directory.clone(), - query: self - .value_require(("directory", id, name, lookup_id))? - .to_string(), - }), - ); - } - - // Parse schedules - if let Some(cron) = - self.property::(("directory", id, "schedule.frequency"))? - { - let mut query = Vec::new(); - for (_, value) in self.values(("directory", id, "schedule.query")) { - query.push(value.to_string()); - } - - if !query.is_empty() { - config.schedules.push(DirectorySchedule { - cron, - query, - directory: directory.clone(), - }) - } else { - tracing::warn!("No scheduled query specified for directory {id:?}"); - } - } - } - - // Parse lookups - for lookup_id in self.sub_keys(("directory", id, "lookup")) { - let key = ("directory", id, "lookup", lookup_id).as_key(); - let lookup = if is_directory { - Lookup::Directory { - directory: directory.clone(), - query: self - .value_require(("directory", id, "lookup", lookup_id))? - .to_string(), - } - } else { - let lookup_type = self.property::((&key, "type"))?; - let format = LookupFormat { - lookup_type: lookup_type.unwrap_or(LookupType::List), - comment: self.value((&key, "comment")).map(|s| s.to_string()), - separator: self.value((&key, "separator")).map(|s| s.to_string()), - }; - - match lookup_type { - Some(LookupType::Map) => Lookup::Map { - map: self.parse_lookup_list((&key, "values"), format)?, - }, - Some(_) => Lookup::List { - list: self.parse_lookup_list((&key, "values"), format)?, - }, - None => Lookup::List { - list: self.parse_lookup_list(key.as_str(), format)?, - }, - } - }; - - config - .lookups - .insert(format!("{id}/{lookup_id}"), Arc::new(lookup)); - } - config.directories.insert(id.to_string(), directory); } Ok(config) } - - fn parse_lookup_list( - &self, - key: K, - format: LookupFormat, - ) -> utils::config::Result { - let mut list = T::default(); - let mut last_failed = false; - for (_, mut value) in self.values(key.clone()) { - if let Some(new_value) = value.strip_prefix("fallback+") { - if last_failed { - value = new_value; - } else { - continue; - } - } - last_failed = false; - - if value.starts_with("https://") || value.starts_with("http://") { - match tokio::task::block_in_place(|| { - reqwest::blocking::get(value).and_then(|r| { - if r.status().is_success() { - r.bytes().map(Ok) - } else { - Ok(Err(r)) - } - }) - }) { - Ok(Ok(bytes)) => { - match list.insert_lines(&*bytes, &format, value.ends_with(".gz")) { - Ok(_) => continue, - Err(err) => { - tracing::warn!( - "Failed to read list {key:?} from {value:?}: {err}", - key = key.as_key(), - value = value, - err = err - ); - } - } - } - Ok(Err(response)) => { - tracing::warn!( - "Failed to fetch list {key:?} from {value:?}: Status {status}", - key = key.as_key(), - value = value, - status = response.status() - ); - } - Err(err) => { - tracing::warn!( - "Failed to fetch list {key:?} from {value:?}: {err}", - key = key.as_key(), - value = value, - err = err - ); - } - } - last_failed = true; - } else if let Some(path) = value.strip_prefix("file://") { - list.insert_lines( - File::open(path).map_err(|err| { - format!( - "Failed to read file {path:?} for list {}: {err}", - key.as_key() - ) - })?, - &format, - value.ends_with(".gz"), - ) - .map_err(|err| { - format!( - "Failed to read file {path:?} for list {}: {err}", - key.as_key() - ) - })?; - } else { - list.insert(value.to_string(), &format); - } - } - Ok(list) - } -} - -pub trait InsertLine: Default { - fn insert(&mut self, entry: String, format: &LookupFormat); - fn insert_lines( - &mut self, - reader: R, - format: &LookupFormat, - decompress: bool, - ) -> Result<(), std::io::Error> { - let reader: Box = if decompress { - Box::new(flate2::read::GzDecoder::new(reader)) - } else { - Box::new(reader) - }; - - for line in BufReader::new(reader).lines() { - let line_ = line?; - let line = line_.trim(); - if !line.is_empty() - && format - .comment - .as_ref() - .map_or(true, |c| !line.starts_with(c)) - { - self.insert(line.to_string(), format); - } - } - Ok(()) - } -} - -impl InsertLine for LookupList { - fn insert(&mut self, entry: String, format: &LookupFormat) { - match format.lookup_type { - LookupType::List => { - self.set.insert(entry); - } - LookupType::Glob => { - let n_wildcards = entry - .as_bytes() - .iter() - .filter(|&&ch| ch == b'*' || ch == b'?') - .count(); - if n_wildcards > 0 { - if n_wildcards == 1 { - if let Some(s) = entry.strip_prefix('*') { - if !s.is_empty() { - self.matches.push(MatchType::EndsWith(s.to_string())); - } - return; - } else if let Some(s) = entry.strip_suffix('*') { - if !s.is_empty() { - self.matches.push(MatchType::StartsWith(s.to_string())); - } - return; - } - } - self.matches - .push(MatchType::Glob(GlobPattern::compile(&entry, false))); - } else { - self.set.insert(entry); - } - } - LookupType::Regex => match regex::Regex::new(&entry) { - Ok(regex) => { - self.matches.push(MatchType::Regex(regex)); - } - Err(err) => { - tracing::warn!("Invalid regular expression {:?}: {}", entry, err); - } - }, - LookupType::Map => unreachable!(), - } - } -} - -impl InsertLine for AHashMap { - fn insert(&mut self, entry: String, format: &LookupFormat) { - let (key, value) = entry - .split_once(format.separator.as_deref().unwrap_or(" ")) - .unwrap_or((entry.as_str(), "")); - let key = key.trim(); - if key.is_empty() { - return; - } else if value.is_empty() { - self.insert(key.to_string(), Variable::default()); - return; - } - let mut has_digit = false; - let mut has_dots = false; - let mut has_other = false; - - for (pos, ch) in value.bytes().enumerate() { - if ch.is_ascii_digit() { - has_digit = true; - } else if ch == b'.' { - has_dots = true; - } else if pos > 0 || ch != b'-' { - has_other = true; - } - } - - let value = if has_other || !has_digit { - Variable::String(value.to_string().into()) - } else if has_dots { - value - .parse() - .map(Variable::Float) - .unwrap_or_else(|_| Variable::String(value.to_string().into())) - } else { - value - .parse() - .map(Variable::Integer) - .unwrap_or_else(|_| Variable::String(value.to_string().into())) - }; - - self.insert(key.to_string(), value); - } } impl DirectoryOptions { @@ -400,31 +115,29 @@ impl AddressMapping { } } -pub(crate) fn build_pool( +pub(crate) fn build_pool( config: &Config, prefix: &str, manager: M, ) -> utils::config::Result> { - Ok(Pool::builder() - .min_idle( - config - .property((prefix, "pool.min-connections"))? - .and_then(|v| if v > 0 { Some(v) } else { None }), - ) + Pool::builder(manager) + .runtime(Runtime::Tokio1) .max_size(config.property_or_static((prefix, "pool.max-connections"), "10")?) - .max_lifetime( + .create_timeout( config - .property_or_static::((prefix, "pool.max-lifetime"), "30m")? + .property_or_static::((prefix, "pool.create-timeout"), "30s")? .into(), ) - .idle_timeout( - config - .property_or_static::((prefix, "pool.idle-timeout"), "10m")? - .into(), - ) - .connection_timeout(config.property_or_static((prefix, "pool.connect-timeout"), "30s")?) - .test_on_check_out(true) - .build_unchecked(manager)) + .wait_timeout(config.property_or_static((prefix, "pool.wait-timeout"), "30s")?) + .recycle_timeout(config.property_or_static((prefix, "pool.recycle-timeout"), "30s")?) + .build() + .map_err(|err| { + format!( + "Failed to build pool for {prefix:?}: {err}", + prefix = prefix, + err = err + ) + }) } #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/crates/directory/src/imap/config.rs b/crates/directory/src/imap/config.rs index 049a588c..5f4f20f1 100644 --- a/crates/directory/src/imap/config.rs +++ b/crates/directory/src/imap/config.rs @@ -26,12 +26,7 @@ use std::sync::Arc; use mail_send::smtp::tls::build_tls_connector; use utils::config::{utils::AsKey, Config}; -use crate::{ - cache::CachedDirectory, - config::{build_pool, ConfigDirectory, LookupFormat}, - imap::ImapConnectionManager, - Directory, -}; +use crate::{cache::CachedDirectory, config::build_pool, imap::ImapConnectionManager, Directory}; use super::ImapDirectory; @@ -63,7 +58,9 @@ impl ImapDirectory { ImapDirectory { pool: build_pool(config, &prefix, manager)?, domains: config - .parse_lookup_list((&prefix, "lookup.domains"), LookupFormat::default())?, + .values((&prefix, "local-domains")) + .map(|(_, v)| v.to_lowercase()) + .collect(), }, ) } diff --git a/crates/directory/src/imap/lookup.rs b/crates/directory/src/imap/lookup.rs index 94c6f0d4..49be21ab 100644 --- a/crates/directory/src/imap/lookup.rs +++ b/crates/directory/src/imap/lookup.rs @@ -24,7 +24,7 @@ use mail_send::Credentials; use smtp_proto::{AUTH_CRAM_MD5, AUTH_LOGIN, AUTH_OAUTHBEARER, AUTH_PLAIN, AUTH_XOAUTH2}; -use crate::{DatabaseColumn, Directory, DirectoryError, Principal}; +use crate::{Directory, DirectoryError, Principal}; use super::{ImapDirectory, ImapError}; @@ -98,18 +98,6 @@ impl Directory for ImapDirectory { Err(DirectoryError::unsupported("imap", "expn")) } - async fn lookup(&self, _: &str, _: &[DatabaseColumn<'_>]) -> crate::Result { - Err(DirectoryError::unsupported("imap", "lookup")) - } - - async fn query( - &self, - _: &str, - _: &[DatabaseColumn<'_>], - ) -> crate::Result>> { - Err(DirectoryError::unsupported("imap", "query")) - } - async fn is_local_domain(&self, domain: &str) -> crate::Result { Ok(self.domains.contains(domain)) } diff --git a/crates/directory/src/imap/mod.rs b/crates/directory/src/imap/mod.rs index a053ec3c..7556bab7 100644 --- a/crates/directory/src/imap/mod.rs +++ b/crates/directory/src/imap/mod.rs @@ -29,15 +29,14 @@ pub mod tls; use std::{fmt::Display, sync::atomic::AtomicU64, time::Duration}; -use bb8::Pool; +use ahash::AHashSet; +use deadpool::managed::Pool; use tokio::io::{AsyncRead, AsyncWrite}; use tokio_rustls::TlsConnector; -use crate::LookupList; - pub struct ImapDirectory { pool: Pool, - domains: LookupList, + domains: AHashSet, } pub struct ImapConnectionManager { diff --git a/crates/directory/src/imap/pool.rs b/crates/directory/src/imap/pool.rs index 30bdf7a0..deb60fc2 100644 --- a/crates/directory/src/imap/pool.rs +++ b/crates/directory/src/imap/pool.rs @@ -23,19 +23,19 @@ use std::sync::atomic::Ordering; -use bb8::ManageConnection; +use async_trait::async_trait; +use deadpool::managed; use tokio::net::TcpStream; use tokio_rustls::client::TlsStream; use super::{ImapClient, ImapConnectionManager, ImapError}; -#[async_trait::async_trait] -impl ManageConnection for ImapConnectionManager { - type Connection = ImapClient>; +#[async_trait] +impl managed::Manager for ImapConnectionManager { + type Type = ImapClient>; type Error = ImapError; - /// Attempts to create a new connection. - async fn connect(&self) -> Result { + async fn create(&self) -> Result>, ImapError> { let mut conn = ImapClient::connect( &self.addr, self.timeout, @@ -55,13 +55,14 @@ impl ManageConnection for ImapConnectionManager { Ok(conn) } - /// Determines if the connection is still connected to the database. - async fn is_valid(&self, conn: &mut Self::Connection) -> Result<(), Self::Error> { - conn.noop().await - } - - /// Synchronously determine if the connection is no longer usable, if possible. - fn has_broken(&self, conn: &mut Self::Connection) -> bool { - !conn.is_valid + async fn recycle( + &self, + conn: &mut ImapClient>, + _: &managed::Metrics, + ) -> managed::RecycleResult { + conn.noop() + .await + .map(|_| ()) + .map_err(managed::RecycleError::Backend) } } diff --git a/crates/directory/src/ldap/config.rs b/crates/directory/src/ldap/config.rs index 2786476c..821e18c9 100644 --- a/crates/directory/src/ldap/config.rs +++ b/crates/directory/src/ldap/config.rs @@ -115,6 +115,13 @@ impl LdapDirectory { .attrs_email .extend(mappings.attr_email_address.iter().cloned()); + let auth_bind = + if config.property_or_static::((&prefix, "auth-bind.enable"), "false")? { + LdapFilter::from_config(config, (&prefix, "auth-bind.dn"))?.into() + } else { + None + }; + CachedDirectory::try_from_config( config, &prefix, @@ -122,6 +129,7 @@ impl LdapDirectory { mappings, pool: build_pool(config, &prefix, manager)?, opt: DirectoryOptions::from_config(config, prefix.as_str())?, + auth_bind, }, ) } diff --git a/crates/directory/src/ldap/lookup.rs b/crates/directory/src/ldap/lookup.rs index db7c8112..0cc636f0 100644 --- a/crates/directory/src/ldap/lookup.rs +++ b/crates/directory/src/ldap/lookup.rs @@ -21,10 +21,10 @@ * for more details. */ -use ldap3::{ResultEntry, Scope, SearchEntry}; +use ldap3::{Ldap, LdapConnAsync, LdapError, ResultEntry, Scope, SearchEntry}; use mail_send::Credentials; -use crate::{DatabaseColumn, Directory, Principal, Type}; +use crate::{Directory, DirectoryError, Principal, Type}; use super::{LdapDirectory, LdapMappings}; @@ -39,23 +39,50 @@ impl Directory for LdapDirectory { Credentials::OAuthBearer { token } => (token, token), Credentials::XOauth2 { username, secret } => (username, secret), }; - match self - .find_principal(&self.mappings.filter_name.build(username)) - .await - { - Ok(Some(principal)) => { - if principal.verify_secret(secret).await { - Ok(Some(principal)) - } else { + + if let Some(auth_bind) = &self.auth_bind { + let (conn, mut ldap) = LdapConnAsync::with_settings( + self.pool.manager().settings.clone(), + &self.pool.manager().address, + ) + .await?; + + ldap3::drive!(conn); + + ldap.simple_bind(&auth_bind.build(username), secret).await?; + + match self + .find_principal(&mut ldap, &self.mappings.filter_name.build(username)) + .await + { + Err(DirectoryError::Ldap(LdapError::LdapResult { result })) + if [49, 50].contains(&result.rc) => + { Ok(None) } + result => result, + } + } else { + let mut conn = self.pool.get().await?; + match self + .find_principal(&mut conn, &self.mappings.filter_name.build(username)) + .await + { + Ok(Some(principal)) => { + if principal.verify_secret(secret).await { + Ok(Some(principal)) + } else { + Ok(None) + } + } + result => result, } - result => result, } } async fn principal(&self, name: &str) -> crate::Result> { - self.find_principal(&self.mappings.filter_name.build(name)) + let mut conn = self.pool.get().await?; + self.find_principal(&mut conn, &self.mappings.filter_name.build(name)) .await } @@ -239,35 +266,6 @@ impl Directory for LdapDirectory { Ok(emails) } - async fn lookup(&self, query: &str, params: &[DatabaseColumn<'_>]) -> crate::Result { - self.query_(query, params) - .await - .map(|entry| entry.is_some()) - } - - async fn query( - &self, - query: &str, - params: &[DatabaseColumn<'_>], - ) -> crate::Result>> { - self.query_(query, params).await.map(|entry| { - if let Some(entry) = entry { - let mut object = String::new(); - for (attr, values) in SearchEntry::construct(entry).attrs { - for value in values { - object.push_str(&attr); - object.push(':'); - object.push_str(&value); - object.push('\n'); - } - } - vec![DatabaseColumn::Text(object.into())] - } else { - vec![] - } - }) - } - async fn is_local_domain(&self, domain: &str) -> crate::Result { self.pool .get() @@ -287,50 +285,12 @@ impl Directory for LdapDirectory { } impl LdapDirectory { - async fn query_( + async fn find_principal( &self, - query: &str, - params: &[DatabaseColumn<'_>], - ) -> crate::Result> { - let mut conn = self.pool.get().await?; - tracing::trace!(context = "directory", event = "query", query = query, params = ?params); - - if !params.is_empty() { - let mut expanded_query = String::with_capacity(query.len() + params.len() * 2); - for (pos, item) in query.split('?').enumerate() { - if pos > 0 { - if let Some(param) = params.get(pos - 1) { - expanded_query.push_str(param.as_str()); - } - } - expanded_query.push_str(item); - } - conn.streaming_search( - &self.mappings.base_dn, - Scope::Subtree, - &expanded_query, - Vec::::new(), - ) - .await - } else { - conn.streaming_search( - &self.mappings.base_dn, - Scope::Subtree, - query, - Vec::::new(), - ) - .await - }? - .next() - .await - .map_err(|e| e.into()) - } - - async fn find_principal(&self, filter: &str) -> crate::Result> { - let (rs, _res) = self - .pool - .get() - .await? + conn: &mut Ldap, + filter: &str, + ) -> crate::Result> { + let (rs, _res) = conn .search( &self.mappings.base_dn, Scope::Subtree, @@ -346,7 +306,6 @@ impl LdapDirectory { }) { // Map groups if !principal.member_of.is_empty() { - let mut conn = self.pool.get().await?; let mut names = Vec::with_capacity(principal.member_of.len()); for group in principal.member_of { if group.contains('=') { diff --git a/crates/directory/src/ldap/mod.rs b/crates/directory/src/ldap/mod.rs index 8cd12d91..8f0fe882 100644 --- a/crates/directory/src/ldap/mod.rs +++ b/crates/directory/src/ldap/mod.rs @@ -21,7 +21,7 @@ * for more details. */ -use bb8::Pool; +use deadpool::managed::Pool; use ldap3::{ldap_escape, LdapConnSettings}; use crate::DirectoryOptions; @@ -34,6 +34,7 @@ pub struct LdapDirectory { pool: Pool, mappings: LdapMappings, opt: DirectoryOptions, + auth_bind: Option, } #[derive(Debug, Default)] diff --git a/crates/directory/src/ldap/pool.rs b/crates/directory/src/ldap/pool.rs index c62fc6b9..4cc52978 100644 --- a/crates/directory/src/ldap/pool.rs +++ b/crates/directory/src/ldap/pool.rs @@ -21,20 +21,21 @@ * for more details. */ -use bb8::ManageConnection; +use async_trait::async_trait; +use deadpool::managed; use ldap3::{exop::WhoAmI, Ldap, LdapConnAsync, LdapError}; use super::LdapConnectionManager; -#[async_trait::async_trait] -impl ManageConnection for LdapConnectionManager { - type Connection = Ldap; +#[async_trait] +impl managed::Manager for LdapConnectionManager { + type Type = Ldap; type Error = LdapError; - /// Attempts to create a new connection. - async fn connect(&self) -> Result { + async fn create(&self) -> Result { let (conn, mut ldap) = LdapConnAsync::with_settings(self.settings.clone(), &self.address).await?; + ldap3::drive!(conn); if let Some(bind) = &self.bind_dn { @@ -44,13 +45,14 @@ impl ManageConnection for LdapConnectionManager { Ok(ldap) } - /// Determines if the connection is still connected to the database. - async fn is_valid(&self, conn: &mut Self::Connection) -> Result<(), Self::Error> { - conn.extended(WhoAmI).await.map(|_| ()) - } - - /// Synchronously determine if the connection is no longer usable, if possible. - fn has_broken(&self, conn: &mut Self::Connection) -> bool { - conn.is_closed() + async fn recycle( + &self, + conn: &mut Ldap, + _: &managed::Metrics, + ) -> managed::RecycleResult { + conn.extended(WhoAmI) + .await + .map(|_| ()) + .map_err(managed::RecycleError::Backend) } } diff --git a/crates/directory/src/lib.rs b/crates/directory/src/lib.rs index 4917ad02..cfbe97f1 100644 --- a/crates/directory/src/lib.rs +++ b/crates/directory/src/lib.rs @@ -21,27 +21,20 @@ * for more details. */ -use std::{ - borrow::Cow, - fmt::{Debug, Display}, - sync::Arc, -}; +use std::{borrow::Cow, fmt::Debug, sync::Arc}; -use ahash::{AHashMap, AHashSet}; -use bb8::RunError; +use ahash::AHashMap; +use deadpool::managed::PoolError; use imap::ImapError; use ldap3::LdapError; use mail_send::Credentials; -use sieve::runtime::{tests::glob::GlobPattern, Variable}; -use smtp_proto::IntoString; -use utils::config::{cron::SimpleCron, DynValue}; +use utils::config::DynValue; pub mod cache; pub mod config; pub mod imap; pub mod ldap; pub mod memory; -pub mod scheduled; pub mod secret; pub mod smtp; pub mod sql; @@ -70,9 +63,10 @@ pub enum Type { #[derive(Debug)] pub enum DirectoryError { Ldap(LdapError), - Sql(sqlx::Error), + Sql(store::Error), Imap(ImapError), Smtp(mail_send::Error), + Pool(String), TimedOut, Unsupported, } @@ -87,177 +81,8 @@ pub trait Directory: Sync + Send { async fn rcpt(&self, address: &str) -> crate::Result; async fn vrfy(&self, address: &str) -> Result>; async fn expn(&self, address: &str) -> Result>; - async fn lookup(&self, query: &str, params: &[DatabaseColumn<'_>]) -> Result; - async fn query( - &self, - query: &str, - params: &[DatabaseColumn<'_>], - ) -> Result>>; - - fn type_name(&self) -> &'static str { - std::any::type_name::() - } } -#[derive(Clone, Debug)] -pub enum DatabaseColumn<'x> { - Integer(i64), - Bool(bool), - Float(f64), - Text(Cow<'x, str>), - Blob(Cow<'x, [u8]>), - Null, -} - -#[derive(Clone)] -pub enum Lookup { - Directory { - directory: Arc, - query: String, - }, - List { - list: LookupList, - }, - Map { - map: AHashMap, - }, -} - -#[derive(Debug, Default, Clone, PartialEq, Eq)] -pub struct LookupList { - pub set: AHashSet, - pub matches: Vec, -} - -#[derive(Debug, Clone)] -pub enum MatchType { - StartsWith(String), - EndsWith(String), - Glob(GlobPattern), - Regex(regex::Regex), -} - -impl LookupList { - pub fn contains(&self, value: &str) -> bool { - if self.set.contains(value) { - true - } else { - for match_type in &self.matches { - let result = match match_type { - MatchType::StartsWith(s) => value.starts_with(s), - MatchType::EndsWith(s) => value.ends_with(s), - MatchType::Glob(g) => g.matches(value), - MatchType::Regex(r) => r.is_match(value), - }; - if result { - return true; - } - } - false - } - } - - pub fn extend(&mut self, other: Self) { - self.set.extend(other.set); - self.matches.extend(other.matches); - } -} - -impl PartialEq for MatchType { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - (Self::StartsWith(l0), Self::StartsWith(r0)) => l0 == r0, - (Self::EndsWith(l0), Self::EndsWith(r0)) => l0 == r0, - (Self::Glob(l0), Self::Glob(r0)) => l0 == r0, - (Self::Regex(_), Self::Regex(_)) => true, - _ => false, - } - } -} - -impl Eq for MatchType {} - -impl Lookup { - pub async fn contains(&self, item: impl Into>) -> Option { - match self { - Lookup::Directory { directory, query } => { - match directory.lookup(query, &[item.into()]).await { - Ok(result) => result.into(), - Err(_) => None, - } - } - Lookup::List { list } => list.contains(item.into().as_str()).into(), - Lookup::Map { map } => map.contains_key(item.into().as_str()).into(), - } - } - - pub async fn lookup(&self, items: &[DatabaseColumn<'_>]) -> Option { - match self { - Lookup::Directory { directory, query } => match directory.query(query, items).await { - Ok(mut result) => { - match result.len() { - 1 if !matches!(result.first(), Some(DatabaseColumn::Null)) => { - result.pop().map(Variable::from).unwrap() - } - 0 => Variable::default(), - _ => Variable::Array( - result - .into_iter() - .map(Variable::from) - .collect::>() - .into(), - ), - } - } - .into(), - Err(_) => None, - }, - Lookup::List { list } => Some(list.contains(items[0].as_str()).into()), - Lookup::Map { map } => map.get(items[0].as_str()).cloned(), - } - } - - pub async fn query( - &self, - items: &[DatabaseColumn<'_>], - ) -> Option>> { - match self { - Lookup::Directory { directory, query } => match directory.query(query, items).await { - Ok(result) => Some(result), - Err(_) => None, - }, - _ => None, - } - } -} - -impl<'x> From> for Variable { - fn from(value: DatabaseColumn) -> Self { - match value { - DatabaseColumn::Integer(v) => Variable::Integer(v), - DatabaseColumn::Bool(v) => Variable::Integer(i64::from(v)), - DatabaseColumn::Float(v) => Variable::Float(v), - DatabaseColumn::Text(v) => Variable::String(v.into_owned().into()), - DatabaseColumn::Blob(v) => Variable::String(v.into_owned().into_string().into()), - DatabaseColumn::Null => Variable::default(), - } - } -} - -impl PartialEq for Lookup { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - (Lookup::Directory { query, .. }, Lookup::Directory { query: other, .. }) => { - query == other - } - (Lookup::List { list }, Lookup::List { list: other }) => list == other, - _ => false, - } - } -} - -impl Eq for Lookup {} - impl Principal { pub fn name(&self) -> &str { &self.name @@ -274,21 +99,7 @@ impl Principal { impl Debug for dyn Directory { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Directory") - .field("type", &self.type_name()) - .finish() - } -} - -impl Debug for Lookup { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Directory { query, .. } => { - f.debug_struct("Directory").field("query", query).finish() - } - Self::List { list } => f.debug_struct("List").field("list", list).finish(), - Self::Map { map } => f.debug_struct("Map").field("map", &map.keys()).finish(), - } + f.debug_struct("Directory").finish() } } @@ -323,44 +134,38 @@ pub enum AddressMapping { } #[derive(Default, Clone, Debug)] -pub struct DirectoryConfig { +pub struct Directories { pub directories: AHashMap>, - pub lookups: AHashMap>, - pub schedules: Vec, -} - -#[derive(Debug, Clone)] -pub struct DirectorySchedule { - pub cron: SimpleCron, - pub query: Vec, - pub directory: Arc, } pub type Result = std::result::Result; -impl From> for DirectoryError { - fn from(error: RunError) -> Self { +impl From> for DirectoryError { + fn from(error: PoolError) -> Self { match error { - RunError::User(error) => error.into(), - RunError::TimedOut => DirectoryError::timeout("ldap"), + PoolError::Backend(error) => error.into(), + PoolError::Timeout(_) => DirectoryError::timeout("ldap"), + error => DirectoryError::Pool(error.to_string()), } } } -impl From> for DirectoryError { - fn from(error: RunError) -> Self { +impl From> for DirectoryError { + fn from(error: PoolError) -> Self { match error { - RunError::User(error) => error.into(), - RunError::TimedOut => DirectoryError::timeout("imap"), + PoolError::Backend(error) => error.into(), + PoolError::Timeout(_) => DirectoryError::timeout("imap"), + error => DirectoryError::Pool(error.to_string()), } } } -impl From> for DirectoryError { - fn from(error: RunError) -> Self { +impl From> for DirectoryError { + fn from(error: PoolError) -> Self { match error { - RunError::User(error) => error.into(), - RunError::TimedOut => DirectoryError::timeout("smtp"), + PoolError::Backend(error) => error.into(), + PoolError::Timeout(_) => DirectoryError::timeout("smtp"), + error => DirectoryError::Pool(error.to_string()), } } } @@ -379,8 +184,8 @@ impl From for DirectoryError { } } -impl From for DirectoryError { - fn from(error: sqlx::Error) -> Self { +impl From for DirectoryError { + fn from(error: store::Error) -> Self { tracing::warn!( context = "directory", event = "error", @@ -495,113 +300,3 @@ impl AddressMapping { } } } - -impl<'x> DatabaseColumn<'x> { - pub fn as_str(&self) -> &str { - match self { - Self::Text(v) => v.as_ref(), - _ => "", - } - } -} - -impl<'x> From<&'x str> for DatabaseColumn<'x> { - fn from(value: &'x str) -> Self { - Self::Text(value.into()) - } -} - -impl<'x> From for DatabaseColumn<'x> { - fn from(value: String) -> Self { - Self::Text(value.into()) - } -} - -impl<'x> From<&'x String> for DatabaseColumn<'x> { - fn from(value: &'x String) -> Self { - Self::Text(value.into()) - } -} - -impl<'x> From> for DatabaseColumn<'x> { - fn from(value: Cow<'x, str>) -> Self { - Self::Text(value) - } -} - -impl<'x> From for DatabaseColumn<'x> { - fn from(value: bool) -> Self { - Self::Bool(value) - } -} - -impl<'x> From for DatabaseColumn<'x> { - fn from(value: i64) -> Self { - Self::Integer(value) - } -} - -impl<'x> From for DatabaseColumn<'x> { - fn from(value: u64) -> Self { - Self::Integer(value as i64) - } -} - -impl<'x> From for DatabaseColumn<'x> { - fn from(value: u32) -> Self { - Self::Integer(value as i64) - } -} - -impl<'x> From for DatabaseColumn<'x> { - fn from(value: f64) -> Self { - Self::Float(value) - } -} - -impl<'x> From<&'x [u8]> for DatabaseColumn<'x> { - fn from(value: &'x [u8]) -> Self { - Self::Blob(value.into()) - } -} - -impl<'x> From> for DatabaseColumn<'x> { - fn from(value: Vec) -> Self { - Self::Blob(value.into()) - } -} - -impl<'x> From for DatabaseColumn<'x> { - fn from(value: Variable) -> Self { - match value { - Variable::String(v) => Self::Text(v.to_string().into()), - Variable::Integer(v) => Self::Integer(v), - Variable::Float(v) => Self::Float(v), - v => Self::Text(v.to_string().into_owned().into()), - } - } -} - -impl<'x> From<&'x Variable> for DatabaseColumn<'x> { - fn from(value: &'x Variable) -> Self { - match value { - Variable::String(v) => Self::Text(v.to_string().into()), - Variable::Integer(v) => Self::Integer(*v), - Variable::Float(v) => Self::Float(*v), - v => Self::Text(v.to_string().into_owned().into()), - } - } -} - -impl<'x> Display for DatabaseColumn<'x> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - DatabaseColumn::Text(v) => f.write_str(v.as_ref()), - DatabaseColumn::Integer(v) => write!(f, "{}", v), - DatabaseColumn::Bool(v) => write!(f, "{}", v), - DatabaseColumn::Float(v) => write!(f, "{}", v), - DatabaseColumn::Blob(v) => write!(f, "{}", String::from_utf8_lossy(v.as_ref())), - DatabaseColumn::Null => write!(f, "NULL"), - } - } -} diff --git a/crates/directory/src/memory/config.rs b/crates/directory/src/memory/config.rs index 6798cd65..9f1d7617 100644 --- a/crates/directory/src/memory/config.rs +++ b/crates/directory/src/memory/config.rs @@ -25,10 +25,7 @@ use std::sync::Arc; use utils::config::{utils::AsKey, Config}; -use crate::{ - config::{ConfigDirectory, LookupFormat}, - Directory, DirectoryOptions, Principal, Type, -}; +use crate::{Directory, DirectoryOptions, Principal, Type}; use super::{EmailType, MemoryDirectory}; @@ -106,10 +103,6 @@ impl MemoryDirectory { directory.parse_emails(config, (prefix.as_str(), "groups", lookup_id), name)?; } - directory.domains.extend( - config.parse_lookup_list((&prefix, "lookup.domains"), LookupFormat::default())?, - ); - Ok(Arc::new(directory)) } } @@ -135,7 +128,7 @@ impl MemoryDirectory { }); if let Some((_, domain)) = email.rsplit_once('@') { - self.domains.set.insert(domain.to_lowercase()); + self.domains.insert(domain.to_lowercase()); } emails.push(if pos > 0 { @@ -150,7 +143,7 @@ impl MemoryDirectory { .or_default() .push(EmailType::List(name.clone())); if let Some((_, domain)) = email.rsplit_once('@') { - self.domains.set.insert(domain.to_lowercase()); + self.domains.insert(domain.to_lowercase()); } emails.push(EmailType::List(email.to_lowercase())); } diff --git a/crates/directory/src/memory/lookup.rs b/crates/directory/src/memory/lookup.rs index 0cc0aa41..54c230f1 100644 --- a/crates/directory/src/memory/lookup.rs +++ b/crates/directory/src/memory/lookup.rs @@ -23,7 +23,7 @@ use mail_send::Credentials; -use crate::{DatabaseColumn, Directory, DirectoryError, Principal}; +use crate::{Directory, Principal}; use super::{EmailType, MemoryDirectory}; @@ -132,18 +132,6 @@ impl Directory for MemoryDirectory { Ok(result) } - async fn lookup(&self, _: &str, _: &[DatabaseColumn<'_>]) -> crate::Result { - Err(DirectoryError::unsupported("memory", "lookp")) - } - - async fn query( - &self, - _: &str, - _: &[DatabaseColumn<'_>], - ) -> crate::Result>> { - Err(DirectoryError::unsupported("memory", "query")) - } - async fn is_local_domain(&self, domain: &str) -> crate::Result { Ok(self.domains.contains(domain)) } diff --git a/crates/directory/src/memory/mod.rs b/crates/directory/src/memory/mod.rs index 62a1bb35..916333a8 100644 --- a/crates/directory/src/memory/mod.rs +++ b/crates/directory/src/memory/mod.rs @@ -21,9 +21,9 @@ * for more details. */ -use ahash::AHashMap; +use ahash::{AHashMap, AHashSet}; -use crate::{DirectoryOptions, LookupList, Principal}; +use crate::{DirectoryOptions, Principal}; pub mod config; pub mod lookup; @@ -33,7 +33,7 @@ pub struct MemoryDirectory { principals: AHashMap, emails_to_names: AHashMap>, names_to_email: AHashMap>, - domains: LookupList, + domains: AHashSet, opt: DirectoryOptions, } diff --git a/crates/directory/src/scheduled.rs b/crates/directory/src/scheduled.rs deleted file mode 100644 index c70ebfad..00000000 --- a/crates/directory/src/scheduled.rs +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use tokio::sync::watch; - -use crate::DirectorySchedule; - -impl DirectorySchedule { - pub fn spawn(self, mut shutdown_rx: watch::Receiver) { - tracing::debug!("Directory query scheduler task starting."); - tokio::spawn(async move { - loop { - if tokio::time::timeout(self.cron.time_to_next(), shutdown_rx.changed()) - .await - .is_ok() - { - tracing::debug!("Directory query scheduler task exiting."); - return; - } - - for query in &self.query { - if let Err(err) = self.directory.query(query, &[]).await { - tracing::warn!( - context = "directory-scheduler", - event = "error", - query = query, - reason = ?err, - ); - } else { - tracing::debug!( - context = "directory-scheduler", - event = "success", - query = query, - ); - } - } - } - }); - } -} diff --git a/crates/directory/src/smtp/config.rs b/crates/directory/src/smtp/config.rs index 60262f0b..c0fd3835 100644 --- a/crates/directory/src/smtp/config.rs +++ b/crates/directory/src/smtp/config.rs @@ -26,12 +26,7 @@ use std::sync::Arc; use mail_send::{smtp::tls::build_tls_connector, SmtpClientBuilder}; use utils::config::{utils::AsKey, Config}; -use crate::{ - cache::CachedDirectory, - config::{build_pool, ConfigDirectory, LookupFormat}, - smtp::SmtpConnectionManager, - Directory, -}; +use crate::{cache::CachedDirectory, config::build_pool, smtp::SmtpConnectionManager, Directory}; use super::SmtpDirectory; @@ -73,7 +68,9 @@ impl SmtpDirectory { SmtpDirectory { pool: build_pool(config, &prefix, manager)?, domains: config - .parse_lookup_list((&prefix, "lookup.domains"), LookupFormat::default())?, + .values((&prefix, "local-domains")) + .map(|(_, v)| v.to_lowercase()) + .collect(), }, ) } diff --git a/crates/directory/src/smtp/lookup.rs b/crates/directory/src/smtp/lookup.rs index a0ef2fda..8ff24380 100644 --- a/crates/directory/src/smtp/lookup.rs +++ b/crates/directory/src/smtp/lookup.rs @@ -24,7 +24,7 @@ use mail_send::{smtp::AssertReply, Credentials}; use smtp_proto::Severity; -use crate::{DatabaseColumn, Directory, DirectoryError, Principal}; +use crate::{Directory, DirectoryError, Principal}; use super::{SmtpClient, SmtpDirectory}; @@ -93,18 +93,6 @@ impl Directory for SmtpDirectory { .await } - async fn lookup(&self, _: &str, _: &[DatabaseColumn<'_>]) -> crate::Result { - Err(DirectoryError::unsupported("smtp", "lookup")) - } - - async fn query( - &self, - _: &str, - _: &[DatabaseColumn<'_>], - ) -> crate::Result>> { - Err(DirectoryError::unsupported("smtp", "query")) - } - async fn is_local_domain(&self, domain: &str) -> crate::Result { Ok(self.domains.contains(domain)) } diff --git a/crates/directory/src/smtp/mod.rs b/crates/directory/src/smtp/mod.rs index 9d68dc2c..2e10833e 100644 --- a/crates/directory/src/smtp/mod.rs +++ b/crates/directory/src/smtp/mod.rs @@ -25,17 +25,16 @@ pub mod config; pub mod lookup; pub mod pool; -use bb8::Pool; +use ahash::AHashSet; +use deadpool::managed::Pool; use mail_send::SmtpClientBuilder; use smtp_proto::EhloResponse; use tokio::net::TcpStream; use tokio_rustls::client::TlsStream; -use crate::LookupList; - pub struct SmtpDirectory { pool: Pool, - domains: LookupList, + domains: AHashSet, } pub struct SmtpConnectionManager { diff --git a/crates/directory/src/smtp/pool.rs b/crates/directory/src/smtp/pool.rs index 6bda981c..34ef0ddb 100644 --- a/crates/directory/src/smtp/pool.rs +++ b/crates/directory/src/smtp/pool.rs @@ -21,18 +21,18 @@ * for more details. */ -use bb8::ManageConnection; +use async_trait::async_trait; +use deadpool::managed; use mail_send::{smtp::AssertReply, Error}; use super::{SmtpClient, SmtpConnectionManager}; -#[async_trait::async_trait] -impl ManageConnection for SmtpConnectionManager { - type Connection = SmtpClient; +#[async_trait] +impl managed::Manager for SmtpConnectionManager { + type Type = SmtpClient; type Error = Error; - /// Attempts to create a new connection. - async fn connect(&self) -> Result { + async fn create(&self) -> Result { let mut client = self.builder.connect().await?; let capabilities = client .capabilities(&self.builder.local_host, self.builder.is_lmtp) @@ -49,16 +49,22 @@ impl ManageConnection for SmtpConnectionManager { }) } - /// Determines if the connection is still connected to the database. - async fn is_valid(&self, conn: &mut Self::Connection) -> Result<(), Self::Error> { - conn.client - .cmd(b"NOOP\r\n") - .await? - .assert_positive_completion() - } - - /// Synchronously determine if the connection is no longer usable, if possible. - fn has_broken(&self, conn: &mut Self::Connection) -> bool { - conn.num_auth_failures >= conn.max_auth_errors + async fn recycle( + &self, + conn: &mut SmtpClient, + _: &managed::Metrics, + ) -> managed::RecycleResult { + if conn.num_auth_failures < conn.max_auth_errors { + conn.client + .cmd(b"NOOP\r\n") + .await? + .assert_positive_completion() + .map(|_| ()) + .map_err(managed::RecycleError::Backend) + } else { + Err(managed::RecycleError::StaticMessage( + "No longer valid: Too many authentication failures", + )) + } } } diff --git a/crates/directory/src/sql/config.rs b/crates/directory/src/sql/config.rs index 881c321c..fb758692 100644 --- a/crates/directory/src/sql/config.rs +++ b/crates/directory/src/sql/config.rs @@ -23,7 +23,7 @@ use std::sync::Arc; -use sqlx::any::{install_default_drivers, AnyPoolOptions}; +use store::Stores; use utils::config::{utils::AsKey, Config}; use crate::{cache::CachedDirectory, Directory, DirectoryOptions}; @@ -34,54 +34,19 @@ impl SqlDirectory { pub fn from_config( config: &Config, prefix: impl AsKey, + stores: &Stores, ) -> utils::config::Result> { let prefix = prefix.as_key(); - let address = config.value_require((&prefix, "address"))?; - install_default_drivers(); - let pool = AnyPoolOptions::new() - .max_connections( - config - .property((&prefix, "pool.max-connections"))? - .unwrap_or(10), - ) - .min_connections( - config - .property((&prefix, "pool.min-connections"))? - .unwrap_or(0), - ) - .idle_timeout(config.property((&prefix, "pool.idle-timeout"))?) - .connect_lazy(address) - .map_err(|err| format!("Failed to create connection pool for {address:?}: {err}"))?; + let store_id = config.value_require((&prefix, "store"))?; + let store = stores + .lookup_stores + .get(store_id) + .ok_or_else(|| { + format!("Directory {prefix:?} references a non-existent store {store_id:?}") + })? + .clone(); - let mappings = SqlMappings { - query_name: config - .value((&prefix, "query.name")) - .unwrap_or_default() - .to_string(), - query_members: config - .value((&prefix, "query.members")) - .unwrap_or_default() - .to_string(), - query_recipients: config - .value((&prefix, "query.recipients")) - .unwrap_or_default() - .to_string(), - query_emails: config - .value((&prefix, "query.emails")) - .unwrap_or_default() - .to_string(), - query_verify: config - .value((&prefix, "query.verify")) - .unwrap_or_default() - .to_string(), - query_expand: config - .value((&prefix, "query.expand")) - .unwrap_or_default() - .to_string(), - query_domains: config - .value((&prefix, "query.domains")) - .unwrap_or_default() - .to_string(), + let mut mappings = SqlMappings { column_name: config .value((&prefix, "columns.name")) .unwrap_or_default() @@ -102,13 +67,28 @@ impl SqlDirectory { .value((&prefix, "columns.type")) .unwrap_or_default() .to_string(), + ..Default::default() }; + for (query_id, query) in [ + ("name", &mut mappings.query_name), + ("members", &mut mappings.query_members), + ("recipients", &mut mappings.query_recipients), + ("emails", &mut mappings.query_emails), + ("verify", &mut mappings.query_verify), + ("expand", &mut mappings.query_expand), + ("domains", &mut mappings.query_domains), + ] { + if let Some(query_) = stores.lookups.get(&format!("{}/{}", store_id, query_id)) { + *query = query_.query.to_string(); + } + } + CachedDirectory::try_from_config( config, &prefix, SqlDirectory { - pool, + store, mappings, opt: DirectoryOptions::from_config(config, prefix.as_str())?, }, diff --git a/crates/directory/src/sql/lookup.rs b/crates/directory/src/sql/lookup.rs index da8bd2e6..0a23bb51 100644 --- a/crates/directory/src/sql/lookup.rs +++ b/crates/directory/src/sql/lookup.rs @@ -21,11 +21,10 @@ * for more details. */ -use futures::TryStreamExt; use mail_send::Credentials; -use sqlx::{any::AnyRow, postgres::any::AnyTypeInfoKind, Column, Row}; +use store::{NamedRows, Rows, Value}; -use crate::{DatabaseColumn, Directory, Principal, Type}; +use crate::{Directory, Principal, Type}; use super::{SqlDirectory, SqlMappings}; @@ -49,21 +48,20 @@ impl Directory for SqlDirectory { } async fn principal(&self, name: &str) -> crate::Result> { - let result = sqlx::query(&self.mappings.query_name) - .bind(name) - .fetch(&self.pool) - .try_next() + let result = self + .store + .query::(&self.mappings.query_name, vec![name.into()]) .await?; - if let Some(row) = result { + if !result.rows.is_empty() { // Map row to principal - let mut principal = self.mappings.row_to_principal(row)?; + let mut principal = self.mappings.row_to_principal(result)?; // Obtain members - principal.member_of = sqlx::query_scalar::<_, String>(&self.mappings.query_members) - .bind(name) - .fetch(&self.pool) - .try_collect::>() - .await?; + principal.member_of = self + .store + .query::(&self.mappings.query_members, vec![name.into()]) + .await? + .into(); // Check whether the user is a superuser if let Some(idx) = principal @@ -82,181 +80,134 @@ impl Directory for SqlDirectory { } async fn emails_by_name(&self, name: &str) -> crate::Result> { - sqlx::query_scalar::<_, String>(&self.mappings.query_emails) - .bind(name) - .fetch(&self.pool) - .try_collect::>() + self.store + .query::(&self.mappings.query_emails, vec![name.into()]) .await + .map(Into::into) .map_err(Into::into) } async fn names_by_email(&self, address: &str) -> crate::Result> { - let ids = sqlx::query_scalar::<_, String>(&self.mappings.query_recipients) - .bind(self.opt.subaddressing.to_subaddress(address).as_ref()) - .fetch(&self.pool) - .try_collect::>() + let ids = self + .store + .query::( + &self.mappings.query_recipients, + vec![self + .opt + .subaddressing + .to_subaddress(address) + .into_owned() + .into()], + ) .await?; - if !ids.is_empty() { - Ok(ids) + + if !ids.rows.is_empty() { + Ok(ids.into()) } else if let Some(address) = self.opt.catch_all.to_catch_all(address) { - sqlx::query_scalar::<_, String>(&self.mappings.query_recipients) - .bind(address.as_ref()) - .fetch(&self.pool) - .try_collect::>() + self.store + .query::(&self.mappings.query_recipients, vec![address.into()]) .await + .map(Into::into) .map_err(Into::into) } else { - Ok(ids) + Ok(vec![]) } } async fn rcpt(&self, address: &str) -> crate::Result { - let result = sqlx::query(&self.mappings.query_recipients) - .bind(self.opt.subaddressing.to_subaddress(address).as_ref()) - .fetch(&self.pool) - .try_next() - .await; - match result { - Ok(Some(_)) => Ok(true), - Ok(None) => { - if let Some(address) = self.opt.catch_all.to_catch_all(address) { - sqlx::query(&self.mappings.query_recipients) - .bind(address.as_ref()) - .fetch(&self.pool) - .try_next() - .await - .map(|id| id.is_some()) - .map_err(Into::into) - } else { - Ok(false) - } - } - - Err(err) => Err(err.into()), + if self + .store + .query::( + &self.mappings.query_recipients, + vec![self + .opt + .subaddressing + .to_subaddress(address) + .into_owned() + .into()], + ) + .await? + { + Ok(true) + } else if let Some(address) = self.opt.catch_all.to_catch_all(address) { + self.store + .query::( + &self.mappings.query_recipients, + vec![address.into_owned().into()], + ) + .await + .map_err(Into::into) + } else { + Ok(false) } } async fn vrfy(&self, address: &str) -> crate::Result> { - sqlx::query_scalar::<_, String>(&self.mappings.query_verify) - .bind(self.opt.subaddressing.to_subaddress(address).as_ref()) - .fetch(&self.pool) - .try_collect::>() + self.store + .query::( + &self.mappings.query_verify, + vec![self + .opt + .subaddressing + .to_subaddress(address) + .into_owned() + .into()], + ) .await + .map(Into::into) .map_err(Into::into) } async fn expn(&self, address: &str) -> crate::Result> { - sqlx::query_scalar::<_, String>(&self.mappings.query_expand) - .bind(self.opt.subaddressing.to_subaddress(address).as_ref()) - .fetch(&self.pool) - .try_collect::>() + self.store + .query::( + &self.mappings.query_expand, + vec![self + .opt + .subaddressing + .to_subaddress(address) + .into_owned() + .into()], + ) .await + .map(Into::into) .map_err(Into::into) } - async fn lookup(&self, query: &str, params: &[DatabaseColumn<'_>]) -> crate::Result { - self.query_(query, params).await.map(|row| row.is_some()) - } - - async fn query( - &self, - query: &str, - params: &[DatabaseColumn<'_>], - ) -> crate::Result>> { - self.query_(query, params).await.map(|row| { - if let Some(row) = row { - let mut columns = Vec::with_capacity(row.columns().len()); - for col in row.columns() { - let idx = col.ordinal(); - columns.push(match col.type_info().kind() { - AnyTypeInfoKind::Bool => { - DatabaseColumn::Bool(row.try_get(idx).unwrap_or_default()) - } - AnyTypeInfoKind::SmallInt - | AnyTypeInfoKind::Integer - | AnyTypeInfoKind::BigInt => { - DatabaseColumn::Integer(row.try_get(idx).unwrap_or_default()) - } - AnyTypeInfoKind::Real | AnyTypeInfoKind::Double => { - DatabaseColumn::Float(row.try_get(idx).unwrap_or_default()) - } - AnyTypeInfoKind::Text => DatabaseColumn::Text( - row.try_get::(idx).unwrap_or_default().into(), - ), - AnyTypeInfoKind::Blob => DatabaseColumn::Blob( - row.try_get::, _>(idx).unwrap_or_default().into(), - ), - AnyTypeInfoKind::Null => row - .try_get::(idx) - .map_or(DatabaseColumn::Null, DatabaseColumn::from), - }); - } - columns - } else { - vec![] - } - }) - } - async fn is_local_domain(&self, domain: &str) -> crate::Result { - sqlx::query(&self.mappings.query_domains) - .bind(domain) - .fetch(&self.pool) - .try_next() + self.store + .query::(&self.mappings.query_domains, vec![domain.into()]) .await - .map(|id| id.is_some()) .map_err(Into::into) } } -impl SqlDirectory { - async fn query_( - &self, - query: &str, - params: &[DatabaseColumn<'_>], - ) -> crate::Result> { - tracing::trace!(context = "directory", event = "query", query = query, params = ?params); - let mut q = sqlx::query(query); - for param in params { - q = match param { - DatabaseColumn::Text(v) => q.bind(v.as_ref()), - DatabaseColumn::Integer(v) => q.bind(v), - DatabaseColumn::Bool(v) => q.bind(v), - DatabaseColumn::Float(v) => q.bind(v), - DatabaseColumn::Blob(v) => { - q.bind(std::str::from_utf8(v.as_ref()).unwrap_or_default()) - } - DatabaseColumn::Null => q.bind(""), - } - } - - q.fetch(&self.pool).try_next().await.map_err(Into::into) - } -} - impl SqlMappings { - pub fn row_to_principal(&self, row: AnyRow) -> crate::Result { + pub fn row_to_principal(&self, rows: NamedRows) -> crate::Result { let mut principal = Principal::default(); - for col in row.columns() { - let idx = col.ordinal(); - let name = col.name(); - - if name.eq_ignore_ascii_case(&self.column_name) { - principal.name = row.try_get::(idx)?; - } else if name.eq_ignore_ascii_case(&self.column_secret) { - if let Ok(secret) = row.try_get::(idx) { - principal.secrets.push(secret); + if let Some(row) = rows.rows.into_iter().next() { + for (name, value) in rows.names.into_iter().zip(row.values) { + if name.eq_ignore_ascii_case(&self.column_name) { + principal.name = value.into_string(); + } else if name.eq_ignore_ascii_case(&self.column_secret) { + if let Value::Text(secret) = value { + principal.secrets.push(secret.into_owned()); + } + } else if name.eq_ignore_ascii_case(&self.column_type) { + match value.to_str().as_ref() { + "individual" | "person" | "user" => principal.typ = Type::Individual, + "group" => principal.typ = Type::Group, + _ => (), + } + } else if name.eq_ignore_ascii_case(&self.column_description) { + if let Value::Text(text) = value { + principal.description = text.into_owned().into(); + } + } else if name.eq_ignore_ascii_case(&self.column_quota) { + if let Value::Integer(quota) = value { + principal.quota = quota as u32; + } } - } else if name.eq_ignore_ascii_case(&self.column_type) { - match row.try_get::(idx)?.as_str() { - "individual" | "person" | "user" => principal.typ = Type::Individual, - "group" => principal.typ = Type::Group, - _ => (), - } - } else if name.eq_ignore_ascii_case(&self.column_description) { - principal.description = row.try_get::(idx).ok(); - } else if name.eq_ignore_ascii_case(&self.column_quota) { - principal.quota = row.try_get::(idx).unwrap_or_default() as u32; } } diff --git a/crates/directory/src/sql/mod.rs b/crates/directory/src/sql/mod.rs index 7d358123..956ab768 100644 --- a/crates/directory/src/sql/mod.rs +++ b/crates/directory/src/sql/mod.rs @@ -21,7 +21,7 @@ * for more details. */ -use sqlx::{Any, Pool}; +use store::LookupStore; use crate::DirectoryOptions; @@ -29,12 +29,12 @@ pub mod config; pub mod lookup; pub struct SqlDirectory { - pool: Pool, + store: LookupStore, mappings: SqlMappings, opt: DirectoryOptions, } -#[derive(Debug)] +#[derive(Debug, Default)] pub(crate) struct SqlMappings { query_name: String, query_members: String, diff --git a/crates/install/Cargo.toml b/crates/install/Cargo.toml index 54ad89fc..e8c2e0a9 100644 --- a/crates/install/Cargo.toml +++ b/crates/install/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" [dependencies] reqwest = { version = "0.11", default-features = false, features = ["rustls-tls-webpki-roots", "blocking"] } -rusqlite = { version = "0.29.0", features = ["bundled"] } +rusqlite = { version = "0.30.0", features = ["bundled"] } rpassword = "7.0" indicatif = "0.17.0" dialoguer = "0.11" diff --git a/crates/jmap/Cargo.toml b/crates/jmap/Cargo.toml index 75a3b144..9412f097 100644 --- a/crates/jmap/Cargo.toml +++ b/crates/jmap/Cargo.toml @@ -29,7 +29,6 @@ aes-gcm-siv = "0.11.1" bincode = "1.3.3" form-data = { version = "0.5.0", features = ["sync"], default-features = false } mime = "0.3.17" -sqlx = { version = "0.7", features = [ "runtime-tokio-rustls", "postgres", "mysql", "sqlite" ] } futures-util = "0.3.28" async-stream = "0.3.5" base64 = "0.21" diff --git a/crates/jmap/src/lib.rs b/crates/jmap/src/lib.rs index 4d85ae18..9c76ed96 100644 --- a/crates/jmap/src/lib.rs +++ b/crates/jmap/src/lib.rs @@ -31,7 +31,7 @@ use auth::{ AccessToken, }; use dashmap::DashMap; -use directory::{Directory, DirectoryConfig}; +use directory::{Directories, Directory}; use jmap_proto::{ error::method::MethodError, method::{ @@ -48,7 +48,6 @@ use services::{ }; use smtp::core::SMTP; use store::{ - backend::{elastic::ElasticSearchStore, rocksdb::RocksDbStore}, fts::FtsFilter, parking_lot::Mutex, query::{sort::Pagination, Comparator, Filter, ResultSet, SortedResultSet}, @@ -57,8 +56,8 @@ use store::{ key::{DeserializeBigEndian, KeySerializer}, BatchBuilder, BitmapClass, TagValue, ToBitmaps, ValueClass, }, - BitmapKey, BlobStore, Deserialize, FtsStore, Key, Serialize, Store, ValueKey, SUBSPACE_VALUES, - U32_LEN, U64_LEN, + BitmapKey, BlobStore, Deserialize, FtsStore, Key, Serialize, Store, Stores, ValueKey, + SUBSPACE_VALUES, U32_LEN, U64_LEN, }; use tokio::sync::mpsc; use utils::{ @@ -185,7 +184,8 @@ pub enum IngestError { impl JMAP { pub async fn init( config: &utils::config::Config, - directory_config: &DirectoryConfig, + stores: &Stores, + directories: &Directories, delivery_rx: mpsc::Receiver, smtp: Arc, ) -> Result, String> { @@ -196,45 +196,9 @@ impl JMAP { .property::("global.shared-map.shard")? .unwrap_or(32) .next_power_of_two() as usize; - /*let store = Store::PostgreSQL(Arc::new( - PostgresStore::open(config) - .await - .failed("Unable to open database"), - ));*/ - /*let store = Store::SQLite(Arc::new( - SqliteStore::open(config) - .await - .failed("Unable to open database"), - ));*/ - /*let store = Store::FoundationDb(Arc::new( - FdbStore::open(config) - .await - .failed("Unable to open database"), - ));*/ - /*let store = Store::MySQL(Arc::new( - MysqlStore::open(config) - .await - .failed("Unable to open database"), - ));*/ - let store = Store::RocksDb(Arc::new( - RocksDbStore::open(config) - .await - .failed("Unable to open database"), - )); - let blob_store = store.clone().into(); - /*let blob_store = BlobStore::Fs(Arc::new( - FsStore::open(config) - .await - .failed("Unable to open blob store"), - ));*/ - let fts_store = FtsStore::Store(store.clone()); - /*let fts_store = ElasticSearchStore::open(config) - .await - .failed("Unable to open FTS store") - .into();*/ let jmap_server = Arc::new(JMAP { - directory: directory_config + directory: directories .directories .get(config.value_require("jmap.directory")?) .failed(&format!( @@ -246,9 +210,30 @@ impl JMAP { .property::("global.node-id")? .map(SnowflakeIdGenerator::with_node_id) .unwrap_or_else(SnowflakeIdGenerator::new), - fts_store, - store, - blob_store, + store: stores + .stores + .get(config.value_require("jmap.store.data")?) + .failed(&format!( + "Unable to find data store '{}'", + config.value_require("jmap.store.data")? + )) + .clone(), + fts_store: stores + .fts_stores + .get(config.value_require("jmap.store.fts")?) + .failed(&format!( + "Unable to find full text store '{}'", + config.value_require("jmap.store.fts")? + )) + .clone(), + blob_store: stores + .blob_stores + .get(config.value_require("jmap.store.blob")?) + .failed(&format!( + "Unable to find blob store '{}'", + config.value_require("jmap.store.blob")? + )) + .clone(), config: Config::new(config).failed("Invalid configuration file"), sessions: TtlDashMap::with_capacity( config.property("jmap.session.cache.size")?.unwrap_or(100), diff --git a/crates/main/Cargo.toml b/crates/main/Cargo.toml index f838c19c..bc6e8d48 100644 --- a/crates/main/Cargo.toml +++ b/crates/main/Cargo.toml @@ -32,7 +32,7 @@ jemallocator = "0.5.0" [features] #default = ["sqlite", "foundationdb", "postgres", "mysql", "rocks", "elastic", "s3"] -default = ["rocks", "elastic"] +default = ["sqlite", "postgres", "mysql"] sqlite = ["store/sqlite"] foundationdb = ["store/foundation"] postgres = ["store/postgres"] diff --git a/crates/main/src/main.rs b/crates/main/src/main.rs index df20ceae..ce1527b3 100644 --- a/crates/main/src/main.rs +++ b/crates/main/src/main.rs @@ -28,6 +28,7 @@ use imap::core::{ImapSessionManager, IMAP}; use jmap::{api::JmapSessionManager, services::IPC_CHANNEL_BUFFER, JMAP}; use managesieve::core::ManageSieveSessionManager; use smtp::core::{SmtpSessionManager, SMTP}; +use store::config::ConfigStore; use tokio::sync::mpsc; use utils::{ config::{Config, ServerProtocol}, @@ -45,7 +46,10 @@ static GLOBAL: Jemalloc = Jemalloc; async fn main() -> std::io::Result<()> { let config = Config::init(); let servers = config.parse_servers().failed("Invalid configuration"); - let directory = config.parse_directory().failed("Invalid configuration"); + let stores = config.parse_stores().await.failed("Invalid configuration"); + let directory = config + .parse_directory(&stores) + .failed("Invalid configuration"); // Bind ports and drop privileges servers.bind(&config); @@ -62,10 +66,10 @@ async fn main() -> std::io::Result<()> { // Init servers let (delivery_tx, delivery_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); - let smtp = SMTP::init(&config, &servers, &directory, delivery_tx) + let smtp = SMTP::init(&config, &servers, &stores, &directory, delivery_tx) .await .failed("Invalid configuration file"); - let jmap = JMAP::init(&config, &directory, delivery_rx, smtp.clone()) + let jmap = JMAP::init(&config, &stores, &directory, delivery_rx, smtp.clone()) .await .failed("Invalid configuration file"); let imap = IMAP::init(&config) @@ -73,7 +77,7 @@ async fn main() -> std::io::Result<()> { .failed("Invalid configuration file"); // Spawn servers - let (shutdown_tx, shutdown_rx) = servers.spawn(|server, shutdown_rx| { + let (shutdown_tx, _shutdown_rx) = servers.spawn(|server, shutdown_rx| { match &server.protocol { ServerProtocol::Smtp | ServerProtocol::Lmtp => { server.spawn(SmtpSessionManager::new(smtp.clone()), shutdown_rx) @@ -95,11 +99,6 @@ async fn main() -> std::io::Result<()> { }; }); - // Spawn scheduled directory queries - for schedule in directory.schedules { - schedule.spawn(shutdown_rx.clone()); - } - // Wait for shutdown signal wait_for_shutdown(&format!( "Shutting down Stalwart Mail Server v{}...", diff --git a/crates/smtp/Cargo.toml b/crates/smtp/Cargo.toml index cbb320a2..7c3790fc 100644 --- a/crates/smtp/Cargo.toml +++ b/crates/smtp/Cargo.toml @@ -12,6 +12,7 @@ edition = "2021" resolver = "2" [dependencies] +store = { path = "../store" } utils = { path = "../utils" } nlp = { path = "../nlp" } directory = { path = "../directory" } @@ -43,7 +44,6 @@ blake3 = "1.3" lru-cache = "0.1.2" rand = "0.8.5" x509-parser = "0.15.0" -sqlx = { version = "0.7", features = [ "runtime-tokio-rustls", "postgres", "mysql", "sqlite" ] } reqwest = { version = "0.11", default-features = false, features = ["rustls-tls-webpki-roots", "blocking"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/crates/smtp/src/config/condition.rs b/crates/smtp/src/config/condition.rs index 9d71c417..4c5f7bad 100644 --- a/crates/smtp/src/config/condition.rs +++ b/crates/smtp/src/config/condition.rs @@ -215,8 +215,8 @@ impl ConfigCondition for Config { })?) } MatchType::Lookup => { - if let Some(lookup) = ctx.directory.lookups.get(value_str) { - ConditionMatch::Lookup(lookup.clone()) + if let Some(lookup) = ctx.stores.lookups.get(value_str) { + ConditionMatch::Lookup(lookup.clone().into()) } else { return Err(format!( "Lookup {:?} not found for property {:?}.", diff --git a/crates/smtp/src/config/mod.rs b/crates/smtp/src/config/mod.rs index 93a7944f..c5cff22a 100644 --- a/crates/smtp/src/config/mod.rs +++ b/crates/smtp/src/config/mod.rs @@ -40,7 +40,7 @@ use std::{ }; use ahash::AHashMap; -use directory::{Directory, DirectoryConfig, Lookup}; +use directory::{Directories, Directory}; use mail_auth::{ common::crypto::{Ed25519Key, RsaKey, Sha256}, dkim::{Canonicalization, Done}, @@ -50,9 +50,10 @@ use mail_send::Credentials; use regex::Regex; use sieve::Sieve; use smtp_proto::MtPriority; +use store::Stores; use utils::config::{DynValue, Rate, Server, ServerProtocol}; -use crate::inbound::milter; +use crate::{core::Lookup, inbound::milter}; #[derive(Debug)] pub struct Host { @@ -96,7 +97,7 @@ pub enum ConditionMatch { UInt(u16), Int(i16), IpAddrMask(IpAddrMask), - Lookup(Arc), + Lookup(Lookup), Regex(Regex), } @@ -534,7 +535,8 @@ pub struct ConfigContext<'x> { pub servers: &'x [Server], pub hosts: AHashMap, pub scripts: AHashMap>, - pub directory: DirectoryConfig, + pub directory: Directories, + pub stores: Stores, pub signers: AHashMap>, pub sealers: AHashMap>, } diff --git a/crates/smtp/src/config/scripts.rs b/crates/smtp/src/config/scripts.rs index 5de38795..f1bf0286 100644 --- a/crates/smtp/src/config/scripts.rs +++ b/crates/smtp/src/config/scripts.rs @@ -119,7 +119,7 @@ impl ConfigSieve for Config { ) .with_max_header_size(10240) .with_valid_notification_uri("mailto") - .with_valid_ext_lists(ctx.directory.lookups.keys().map(|k| k.to_string())) + .with_valid_ext_lists(ctx.stores.lookups.keys().map(|k| k.to_string())) .with_functions(&mut fnc_map); if let Some(value) = self.property("sieve.trusted.limits.redirects")? { @@ -187,7 +187,12 @@ impl ConfigSieve for Config { Ok(SieveCore { runtime, scripts: ctx.scripts.clone(), - lookup: ctx.directory.lookups.clone(), + lookup: ctx + .stores + .lookups + .iter() + .map(|(k, v)| (k.to_string(), v.clone().into())) + .collect(), config: SieveConfig { from_addr: self .value("sieve.trusted.from-addr") @@ -203,6 +208,7 @@ impl ConfigSieve for Config { .to_string(), sign, directories: ctx.directory.directories.clone(), + lookup_stores: ctx.stores.lookup_stores.clone(), }, }) } diff --git a/crates/smtp/src/core/mod.rs b/crates/smtp/src/core/mod.rs index b14017f5..ddf000e7 100644 --- a/crates/smtp/src/core/mod.rs +++ b/crates/smtp/src/core/mod.rs @@ -22,6 +22,7 @@ */ use std::{ + cmp::Ordering, hash::Hash, net::IpAddr, sync::{atomic::AtomicU32, Arc}, @@ -30,12 +31,17 @@ use std::{ use ahash::AHashMap; use dashmap::DashMap; -use directory::{Directory, Lookup}; +use directory::Directory; use mail_auth::{common::lru::LruCache, IprevOutput, Resolver, SpfOutput}; -use sieve::{Runtime, Sieve}; -use smtp_proto::request::receiver::{ - BdatReceiver, DataReceiver, DummyDataReceiver, DummyLineReceiver, LineReceiver, RequestReceiver, +use sieve::{runtime::Variable, Runtime, Sieve}; +use smtp_proto::{ + request::receiver::{ + BdatReceiver, DataReceiver, DummyDataReceiver, DummyLineReceiver, LineReceiver, + RequestReceiver, + }, + IntoString, }; +use store::{LookupStore, Row, Value}; use tokio::{ io::{AsyncRead, AsyncWrite}, sync::mpsc, @@ -106,7 +112,7 @@ pub struct SMTP { pub struct SieveCore { pub runtime: Runtime, pub scripts: AHashMap>, - pub lookup: AHashMap>, + pub lookup: AHashMap, pub config: SieveConfig, } @@ -116,6 +122,7 @@ pub struct SieveConfig { pub return_path: String, pub sign: Vec>, pub directories: AHashMap>, + pub lookup_stores: AHashMap, } pub struct Resolvers { @@ -153,6 +160,9 @@ pub struct TlsConnectors { pub dummy_verify: TlsConnector, } +#[derive(Clone)] +pub struct Lookup(Arc); + pub enum State { Request(RequestReceiver), Bdat(BdatReceiver), @@ -270,6 +280,95 @@ impl SessionData { } } +impl Lookup { + pub async fn contains(&self, item: impl Into>) -> Option { + self.0 + .store + .query::(&self.0.query, vec![item.into()]) + .await + .ok() + } + + pub async fn lookup(&self, items: Vec>) -> Option { + self.0 + .store + .query::>(&self.0.query, items) + .await + .ok() + .map(|row| { + let mut row = row.map(|row| row.values).unwrap_or_default(); + match row.len().cmp(&1) { + Ordering::Equal if !matches!(row.first(), Some(Value::Null)) => { + row.pop().map(into_sieve_value).unwrap() + } + Ordering::Less => Variable::default(), + _ => Variable::Array( + row.into_iter() + .map(into_sieve_value) + .collect::>() + .into(), + ), + } + }) + } + + pub async fn query(&self, items: Vec>) -> Option>> { + self.0 + .store + .query::>(&self.0.query, items) + .await + .ok() + .map(|row| row.map(|row| row.values).unwrap_or_default()) + } +} + +impl PartialEq for Lookup { + fn eq(&self, other: &Self) -> bool { + self.0.query == other.0.query + } +} + +pub fn into_sieve_value(value: Value) -> Variable { + match value { + Value::Integer(v) => Variable::Integer(v), + Value::Bool(v) => Variable::Integer(i64::from(v)), + Value::Float(v) => Variable::Float(v), + Value::Text(v) => Variable::String(v.into_owned().into()), + Value::Blob(v) => Variable::String(v.into_owned().into_string().into()), + Value::Null => Variable::default(), + } +} + +pub fn into_store_value(value: Variable) -> Value<'static> { + match value { + Variable::String(v) => Value::Text(v.to_string().into()), + Variable::Integer(v) => Value::Integer(v), + Variable::Float(v) => Value::Float(v), + v => Value::Text(v.to_string().into_owned().into()), + } +} + +pub fn to_store_value(value: &Variable) -> Value<'static> { + match value { + Variable::String(v) => Value::Text(v.to_string().into()), + Variable::Integer(v) => Value::Integer(*v), + Variable::Float(v) => Value::Float(*v), + v => Value::Text(v.to_string().into_owned().into()), + } +} + +impl AsRef for Lookup { + fn as_ref(&self) -> &LookupStore { + &self.0.store + } +} + +impl From> for Lookup { + fn from(lookup: Arc) -> Self { + Self(lookup) + } +} + impl Default for State { fn default() -> Self { State::Request(RequestReceiver::default()) diff --git a/crates/smtp/src/lib.rs b/crates/smtp/src/lib.rs index 0c3c6a00..e41d0ea4 100644 --- a/crates/smtp/src/lib.rs +++ b/crates/smtp/src/lib.rs @@ -31,10 +31,11 @@ use config::{ resolver::ConfigResolver, scripts::ConfigSieve, session::ConfigSession, ConfigContext, Host, }; use dashmap::DashMap; -use directory::DirectoryConfig; +use directory::Directories; use mail_send::smtp::tls::build_tls_connector; use queue::manager::SpawnQueue; use reporting::scheduler::SpawnReport; +use store::Stores; use tokio::sync::mpsc; use utils::{ config::{Config, ServerProtocol, Servers}, @@ -56,12 +57,14 @@ impl SMTP { pub async fn init( config: &Config, servers: &Servers, - directory: &DirectoryConfig, + stores: &Stores, + directory: &Directories, #[cfg(feature = "local_delivery")] delivery_tx: mpsc::Sender, ) -> Result, String> { // Read configuration parameters let mut config_ctx = ConfigContext::new(&servers.inner); config_ctx.directory = directory.clone(); + config_ctx.stores = stores.clone(); // Parse remote hosts config.parse_remote_hosts(&mut config_ctx)?; diff --git a/crates/smtp/src/scripts/event_loop.rs b/crates/smtp/src/scripts/event_loop.rs index 39e4309a..3750373e 100644 --- a/crates/smtp/src/scripts/event_loop.rs +++ b/crates/smtp/src/scripts/event_loop.rs @@ -23,7 +23,6 @@ use std::{sync::Arc, time::Duration}; -use directory::Lookup; use mail_auth::common::headers::HeaderWriter; use sieve::{ compiler::grammar::actions::action_redirect::{ByMode, ByTime, Notify, NotifyItem, Ret}, @@ -33,6 +32,7 @@ use smtp_proto::{ MAIL_BY_TRACE, MAIL_RET_FULL, MAIL_RET_HDRS, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, }; +use store::backend::memory::MemoryStore; use tokio::runtime::Handle; use crate::{ @@ -166,17 +166,14 @@ impl SMTP { } Recipient::List(list) => { if let Some(list) = self.sieve.lookup.get(&list) { - match list.as_ref() { - Lookup::List { list } => { + if let store::LookupStore::Memory(list) = list.as_ref() { + if let MemoryStore::List(list) = list.as_ref() { for rcpt in &list.set { handle.block_on( message.add_recipient(rcpt, &self.queue.config), ); } } - Lookup::Directory { .. } | Lookup::Map { .. } => { - // Not implemented - } } } else { tracing::warn!( diff --git a/crates/smtp/src/scripts/plugins/bayes.rs b/crates/smtp/src/scripts/plugins/bayes.rs index f3115edf..668a82aa 100644 --- a/crates/smtp/src/scripts/plugins/bayes.rs +++ b/crates/smtp/src/scripts/plugins/bayes.rs @@ -21,7 +21,6 @@ * for more details. */ -use directory::{DatabaseColumn, Lookup}; use nlp::{ bayes::{ cache::BayesTokenCache, tokenize::BayesTokenizer, BayesClassifier, BayesModel, TokenHash, @@ -30,9 +29,10 @@ use nlp::{ tokenizers::osb::{OsbToken, OsbTokenizer}, }; use sieve::{runtime::Variable, FunctionMap}; +use store::Value; use tokio::runtime::Handle; -use crate::config::scripts::SieveContext; +use crate::{config::scripts::SieveContext, core::Lookup}; use super::PluginContext; @@ -109,7 +109,7 @@ fn train(ctx: PluginContext<'_>, is_train: bool) -> Variable { (-(weights.spam as i64), -(weights.ham as i64)) }; if handle - .block_on(lookup_train.lookup(&[ + .block_on(lookup_train.lookup(vec![ hash.h1.into(), hash.h2.into(), s_weight.into(), @@ -130,7 +130,7 @@ fn train(ctx: PluginContext<'_>, is_train: bool) -> Variable { (0i64, train_val) }; if handle - .block_on(lookup_train.query(&[ + .block_on(lookup_train.query(vec![ 0i64.into(), 0i64.into(), spam_count.into(), @@ -322,11 +322,11 @@ impl LookupOrInsert for BayesTokenCache { if let Some(weights) = self.get(&hash) { weights.unwrap_or_default().into() } else if let Some(result) = - handle.block_on(get_token.query(&[hash.h1.into(), hash.h2.into()])) + handle.block_on(get_token.query(vec![hash.h1.into(), hash.h2.into()])) { let mut result = result.into_iter(); match (result.next(), result.next()) { - (Some(DatabaseColumn::Integer(spam)), Some(DatabaseColumn::Integer(ham))) => { + (Some(Value::Integer(spam)), Some(Value::Integer(ham))) => { let weights = Weights { spam: spam as u32, ham: ham as u32, diff --git a/crates/smtp/src/scripts/plugins/lookup.rs b/crates/smtp/src/scripts/plugins/lookup.rs index 618887b8..01df16dd 100644 --- a/crates/smtp/src/scripts/plugins/lookup.rs +++ b/crates/smtp/src/scripts/plugins/lookup.rs @@ -27,12 +27,12 @@ use std::{ time::{Duration, Instant}, }; -use directory::DatabaseColumn; use mail_auth::flate2; use sieve::{runtime::Variable, FunctionMap}; use crate::{ config::scripts::{RemoteList, SieveContext}, + core::to_store_value, USER_AGENT, }; @@ -62,14 +62,20 @@ pub fn exec(ctx: PluginContext<'_>) -> Variable { Variable::Array(items) => { for item in items.iter() { if !item.is_empty() - && ctx.handle.block_on(lookup.contains(item)).unwrap_or(false) + && ctx + .handle + .block_on(lookup.contains(to_store_value(item))) + .unwrap_or(false) { return true.into(); } } false } - v if !v.is_empty() => ctx.handle.block_on(lookup.contains(v)).unwrap_or(false), + v if !v.is_empty() => ctx + .handle + .block_on(lookup.contains(to_store_value(v))) + .unwrap_or(false), _ => false, } } else { @@ -88,8 +94,8 @@ pub fn exec(ctx: PluginContext<'_>) -> Variable { pub fn exec_map(ctx: PluginContext<'_>) -> Variable { let lookup_id = ctx.arguments[0].to_string(); let items = match &ctx.arguments[1] { - Variable::Array(l) => l.iter().map(DatabaseColumn::from).collect(), - v if !v.is_empty() => vec![DatabaseColumn::from(v)], + Variable::Array(l) => l.iter().map(to_store_value).collect(), + v if !v.is_empty() => vec![to_store_value(v)], _ => vec![], }; let span = ctx.span; @@ -98,7 +104,7 @@ pub fn exec_map(ctx: PluginContext<'_>) -> Variable { if let Some(lookup) = ctx.core.sieve.lookup.get(lookup_id.as_ref()) { return ctx .handle - .block_on(lookup.lookup(&items)) + .block_on(lookup.lookup(items)) .unwrap_or_default(); } else { tracing::warn!( diff --git a/crates/smtp/src/scripts/plugins/query.rs b/crates/smtp/src/scripts/plugins/query.rs index 31b55db2..bdb52ebf 100644 --- a/crates/smtp/src/scripts/plugins/query.rs +++ b/crates/smtp/src/scripts/plugins/query.rs @@ -12,7 +12,7 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. + * in the LICENSE file at the top-level store of this distribution. * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . * @@ -21,9 +21,14 @@ * for more details. */ -use crate::config::scripts::SieveContext; -use directory::DatabaseColumn; +use std::cmp::Ordering; + +use crate::{ + config::scripts::SieveContext, + core::{into_sieve_value, to_store_value}, +}; use sieve::{runtime::Variable, FunctionMap}; +use store::{Rows, Value}; use super::PluginContext; @@ -34,21 +39,20 @@ pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap) { pub fn exec(ctx: PluginContext<'_>) -> Variable { let span = ctx.span; - // Obtain directory name - let directory = ctx.arguments[0].to_string(); - let directory = - if let Some(directory_) = ctx.core.sieve.config.directories.get(directory.as_ref()) { - directory_ - } else { - tracing::warn!( - parent: span, - context = "sieve:query", - event = "failed", - reason = "Unknown directory", - directory = %directory, - ); - return false.into(); - }; + // Obtain store name + let store = ctx.arguments[0].to_string(); + let store = if let Some(store_) = ctx.core.sieve.config.lookup_stores.get(store.as_ref()) { + store_ + } else { + tracing::warn!( + parent: span, + context = "sieve:query", + event = "failed", + reason = "Unknown store", + store = %store, + ); + return false.into(); + }; // Obtain query string let query = ctx.arguments[1].to_string(); @@ -64,8 +68,8 @@ pub fn exec(ctx: PluginContext<'_>) -> Variable { // Obtain arguments let arguments = match &ctx.arguments[2] { - Variable::Array(l) => l.iter().map(DatabaseColumn::from).collect(), - v => vec![DatabaseColumn::from(v)], + Variable::Array(l) => l.iter().map(to_store_value).collect(), + v => vec![to_store_value(v)], }; // Run query @@ -74,26 +78,45 @@ pub fn exec(ctx: PluginContext<'_>) -> Variable { .get(..6) .map_or(false, |q| q.eq_ignore_ascii_case(b"SELECT")) { - if let Ok(mut query_columns) = ctx.handle.block_on(directory.query(&query, &arguments)) { - match query_columns.len() { - 1 if !matches!(query_columns.first(), Some(DatabaseColumn::Null)) => { - query_columns.pop().map(Variable::from).unwrap() + if let Ok(mut rows) = ctx.handle.block_on(store.query::(&query, arguments)) { + match rows.rows.len().cmp(&1) { + Ordering::Equal => { + let mut row = rows.rows.pop().unwrap().values; + match row.len().cmp(&1) { + Ordering::Equal if !matches!(row.first(), Some(Value::Null)) => { + row.pop().map(into_sieve_value).unwrap() + } + Ordering::Less => Variable::default(), + _ => Variable::Array( + row.into_iter() + .map(into_sieve_value) + .collect::>() + .into(), + ), + } } - 0 => Variable::default(), - _ => Variable::Array( - query_columns - .into_iter() - .map(Variable::from) - .collect::>() - .into(), - ), + Ordering::Less => Variable::default(), + Ordering::Greater => rows + .rows + .into_iter() + .map(|r| { + Variable::Array( + r.values + .into_iter() + .map(into_sieve_value) + .collect::>() + .into(), + ) + }) + .collect::>() + .into(), } } else { false.into() } } else { ctx.handle - .block_on(directory.lookup(&query, &arguments)) + .block_on(store.query::(&query, arguments)) .is_ok() .into() } diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index c113f33b..d93f9678 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -9,7 +9,7 @@ utils = { path = "../utils" } nlp = { path = "../nlp" } rocksdb = { version = "0.21", optional = true, features = ["multi-threaded-cf"] } foundationdb = { version = "0.8.0", features = ["embedded-fdb-include"], optional = true } -rusqlite = { version = "0.29.0", features = ["bundled"], optional = true } +rusqlite = { version = "0.30.0", features = ["bundled"], optional = true } rust-s3 = { version = "0.33.0", default-features = false, features = ["tokio-rustls-tls"], optional = true } tokio = { version = "1.23", features = ["sync", "fs", "io-util"] } r2d2 = { version = "0.8.10", optional = true } @@ -33,9 +33,14 @@ tokio-postgres = { version = "0.7.10", optional = true } tokio-rustls = { version = "0.24.0", optional = true } rustls = { version = "0.21.0", optional = true } ring = { version = "0.17", optional = true } +bytes = { version = "1.0", optional = true } mysql_async = { version = "0.33", default-features = false, features = ["default-rustls"], optional = true } elasticsearch = { version = "8.5.0-alpha.1", default-features = false, features = ["rustls-tls"], optional = true } serde_json = {version = "1.0.64", optional = true } +regex = "1.7.0" +reqwest = { version = "0.11", default-features = false, features = ["rustls-tls-webpki-roots", "blocking"] } +flate2 = "1.0" +async-trait = "0.1.68" [dev-dependencies] tokio = { version = "1.23", features = ["full"] } @@ -43,7 +48,7 @@ tokio = { version = "1.23", features = ["full"] } [features] rocks = ["rocksdb", "rayon", "num_cpus"] sqlite = ["rusqlite", "rayon", "r2d2", "num_cpus", "lru-cache"] -postgres = ["tokio-postgres", "deadpool-postgres", "tokio-rustls", "rustls", "ring", "futures"] +postgres = ["tokio-postgres", "deadpool-postgres", "tokio-rustls", "rustls", "ring", "futures", "bytes"] elastic = ["elasticsearch", "serde_json"] mysql = ["mysql_async"] s3 = ["rust-s3"] diff --git a/crates/store/src/backend/elastic/mod.rs b/crates/store/src/backend/elastic/mod.rs index 68b3204e..b098dead 100644 --- a/crates/store/src/backend/elastic/mod.rs +++ b/crates/store/src/backend/elastic/mod.rs @@ -32,7 +32,7 @@ use elasticsearch::{ Elasticsearch, Error, }; use serde_json::json; -use utils::config::Config; +use utils::config::{utils::AsKey, Config}; pub mod index; pub mod query; @@ -44,50 +44,57 @@ pub struct ElasticSearchStore { pub(crate) static INDEX_NAMES: &[&str] = &["stalwart_email"]; impl ElasticSearchStore { - pub async fn open(config: &Config) -> crate::Result { - let credentials = if let Some(user) = config.value("store.fts.user") { - let password = config.value_require("store.fts.password")?; + pub async fn open(config: &Config, prefix: impl AsKey) -> crate::Result { + let prefix = prefix.as_key(); + let credentials = if let Some(user) = config.value((&prefix, "user")) { + let password = config.value_require((&prefix, "password"))?; Some(Credentials::Basic(user.to_string(), password.to_string())) } else { None }; - let es = if let Some(url) = config.value("store.fts.url") { + let es = if let Some(url) = config.value((&prefix, "url")) { let url = Url::parse(url).map_err(|e| { - crate::Error::InternalError(format!("Invalid store.fts.url: {}", e)) + crate::Error::InternalError(format!( + "Invalid URL {}: {}", + (&prefix, "url").as_key(), + e + )) })?; let conn_pool = SingleNodeConnectionPool::new(url); let mut builder = TransportBuilder::new(conn_pool); if let Some(credentials) = credentials { builder = builder.auth(credentials); } - if config.property_or_static::("store.fts.allow-invalid-certs", "false")? { + if config.property_or_static::((&prefix, "allow-invalid-certs"), "false")? { builder = builder.cert_validation(CertificateValidation::None); } Self { index: Elasticsearch::new(builder.build()?), } - } else if let Some(cloud_id) = config.value("store.fts.cloud-id") { + } else if let Some(cloud_id) = config.value((&prefix, "cloud-id")) { Self { index: Elasticsearch::new(Transport::cloud( cloud_id, credentials.ok_or_else(|| { - crate::Error::InternalError( - "Missing store.fts.user or store.fts.password".to_string(), - ) + crate::Error::InternalError(format!( + "Missing user and/or password for ElasticSearch store {}", + prefix + )) })?, )?), } } else { - return Err(crate::Error::InternalError( - "Missing store.fts.url or store.fts.cloud_id".to_string(), - )); + return Err(crate::Error::InternalError(format!( + "Missing url or cloud_id for ElasticSearch store {}", + prefix + ))); }; es.create_index( - config.property_or_static("store.fts.shards", "3")?, - config.property_or_static("store.fts.replicas", "0")?, + config.property_or_static((&prefix, "index.shards"), "3")?, + config.property_or_static((&prefix, "index.replicas"), "0")?, ) .await?; @@ -160,7 +167,7 @@ impl ElasticSearchStore { if !response.status_code().is_success() { return Err(crate::Error::InternalError(format!( - "Error while creating ElastiSearch index: {:?}", + "Error while creating ElasticSearch index: {:?}", response ))); } @@ -172,12 +179,12 @@ impl ElasticSearchStore { impl From for crate::Error { fn from(value: Error) -> Self { - crate::Error::InternalError(format!("Elasticsearch error: {}", value)) + crate::Error::InternalError(format!("ElasticSearch error: {}", value)) } } impl From for crate::Error { fn from(value: BuildError) -> Self { - crate::Error::InternalError(format!("Elasticsearch build error: {}", value)) + crate::Error::InternalError(format!("ElasticSearch build error: {}", value)) } } diff --git a/crates/store/src/backend/foundationdb/main.rs b/crates/store/src/backend/foundationdb/main.rs index af7e02e6..6e6ebf04 100644 --- a/crates/store/src/backend/foundationdb/main.rs +++ b/crates/store/src/backend/foundationdb/main.rs @@ -21,16 +21,33 @@ * for more details. */ -use foundationdb::Database; -use utils::config::Config; +use foundationdb::{options::DatabaseOption, Database}; +use utils::config::{utils::AsKey, Config}; use super::FdbStore; impl FdbStore { - pub async fn open(_: &Config) -> crate::Result { - Ok(Self { - guard: unsafe { foundationdb::boot() }, - db: Database::default()?, - }) + pub async fn open(config: &Config, prefix: impl AsKey) -> crate::Result { + let prefix = prefix.as_key(); + let guard = unsafe { foundationdb::boot() }; + + let db = Database::new(config.value((&prefix, "path")))?; + if let Some(value) = config.property((&prefix, "transaction.timeout"))? { + db.set_option(DatabaseOption::TransactionTimeout(value))?; + } + if let Some(value) = config.property((&prefix, "transaction.retry-limit"))? { + db.set_option(DatabaseOption::TransactionRetryLimit(value))?; + } + if let Some(value) = config.property((&prefix, "transaction.max-retry-delay"))? { + db.set_option(DatabaseOption::TransactionMaxRetryDelay(value))?; + } + if let Some(value) = config.property((&prefix, "transaction.machine-id"))? { + db.set_option(DatabaseOption::MachineId(value))?; + } + if let Some(value) = config.property((&prefix, "transaction.datacenter-id"))? { + db.set_option(DatabaseOption::DatacenterId(value))?; + } + + Ok(Self { guard, db }) } } diff --git a/crates/store/src/backend/fs/mod.rs b/crates/store/src/backend/fs/mod.rs index fca25bbb..aff54c17 100644 --- a/crates/store/src/backend/fs/mod.rs +++ b/crates/store/src/backend/fs/mod.rs @@ -27,7 +27,10 @@ use tokio::{ fs::{self, File}, io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}, }; -use utils::{codec::base32_custom::Base32Writer, config::Config}; +use utils::{ + codec::base32_custom::Base32Writer, + config::{utils::AsKey, Config}, +}; pub struct FsStore { path: PathBuf, @@ -35,15 +38,13 @@ pub struct FsStore { } impl FsStore { - pub async fn open(config: &Config) -> crate::Result { - let path = config.property_require::("store.blob.local.path")?; + pub async fn open(config: &Config, prefix: impl AsKey) -> crate::Result { + let prefix = prefix.as_key(); + let path = config.property_require::((&prefix, "path"))?; if path.exists() { Ok(FsStore { path, - hash_levels: std::cmp::min( - config.property_or_static("store.blob.local.depth", "2")?, - 5, - ), + hash_levels: std::cmp::min(config.property_or_static((&prefix, "depth"), "2")?, 5), }) } else { Err(crate::Error::InternalError(format!( diff --git a/crates/store/src/backend/memory/glob.rs b/crates/store/src/backend/memory/glob.rs new file mode 100644 index 00000000..513599e4 --- /dev/null +++ b/crates/store/src/backend/memory/glob.rs @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2020-2023, Stalwart Labs Ltd. + * + * This file is part of the Stalwart Sieve Interpreter. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GlobPattern { + pattern: Vec, + to_lower: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PatternChar { + WildcardMany { num: usize, match_pos: usize }, + WildcardSingle { match_pos: usize }, + Char { char: char, match_pos: usize }, +} + +impl GlobPattern { + pub fn compile(pattern: &str, to_lower: bool) -> Self { + let mut chars = Vec::new(); + let mut is_escaped = false; + let mut str = pattern.chars().peekable(); + + while let Some(char) = str.next() { + match char { + '*' if !is_escaped => { + let mut num = 1; + while let Some('*') = str.peek() { + num += 1; + str.next(); + } + chars.push(PatternChar::WildcardMany { num, match_pos: 0 }); + } + '?' if !is_escaped => { + chars.push(PatternChar::WildcardSingle { match_pos: 0 }); + } + '\\' if !is_escaped => { + is_escaped = true; + continue; + } + _ => { + if is_escaped { + is_escaped = false; + } + if to_lower && char.is_uppercase() { + for char in char.to_lowercase() { + chars.push(PatternChar::Char { char, match_pos: 0 }); + } + } else { + chars.push(PatternChar::Char { char, match_pos: 0 }); + } + } + } + } + + GlobPattern { + pattern: chars, + to_lower, + } + } + + // Credits: Algorithm ported from https://research.swtch.com/glob + pub fn matches(&self, value: &str) -> bool { + let value = if self.to_lower { + value.to_lowercase().chars().collect::>() + } else { + value.chars().collect::>() + }; + + let mut px = 0; + let mut nx = 0; + let mut next_px = 0; + let mut next_nx = 0; + + while px < self.pattern.len() || nx < value.len() { + match self.pattern.get(px) { + Some(PatternChar::Char { char, .. }) => { + if matches!(value.get(nx), Some(nc) if nc == char ) { + px += 1; + nx += 1; + continue; + } + } + Some(PatternChar::WildcardSingle { .. }) => { + if nx < value.len() { + px += 1; + nx += 1; + continue; + } + } + Some(PatternChar::WildcardMany { .. }) => { + next_px = px; + next_nx = nx + 1; + px += 1; + continue; + } + _ => (), + } + if 0 < next_nx && next_nx <= value.len() { + px = next_px; + nx = next_nx; + continue; + } + return false; + } + true + } +} diff --git a/crates/store/src/backend/memory/lookup.rs b/crates/store/src/backend/memory/lookup.rs new file mode 100644 index 00000000..41a4dc08 --- /dev/null +++ b/crates/store/src/backend/memory/lookup.rs @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use crate::{IntoRows, QueryResult, QueryType, Row, Value}; + +use super::{LookupList, MatchType, MemoryStore}; + +impl MemoryStore { + pub(crate) fn query( + &self, + _: &str, + params: Vec>, + ) -> crate::Result { + let exists = match T::query_type() { + QueryType::Exists => true, + QueryType::QueryOne => false, + QueryType::QueryAll | QueryType::Execute => { + return Err(crate::Error::InternalError( + "Unsupported query type".to_string(), + )) + } + }; + + let needle = params.first().map(|v| v.to_str()).unwrap_or_default(); + + match self { + MemoryStore::List(list) => { + let found = list.contains(needle.as_ref()); + if exists { + Ok(T::from_exists(found)) + } else { + Ok(T::from_query_one(Some(Row { + values: vec![Value::Bool(found)], + }))) + } + } + MemoryStore::Map(map) => { + if let Some(value) = map.get(needle.as_ref()) { + if exists { + Ok(T::from_exists(true)) + } else { + Ok(T::from_query_one(Some(Row { + values: vec![value.clone()], + }))) + } + } else if exists { + Ok(T::from_exists(false)) + } else { + Ok(T::from_query_one(None::)) + } + } + } + } +} + +impl IntoRows for Option { + fn into_row(self) -> Option { + self + } + + fn into_rows(self) -> crate::Rows { + unreachable!() + } + + fn into_named_rows(self) -> crate::NamedRows { + unreachable!() + } +} + +impl LookupList { + pub fn contains(&self, value: &str) -> bool { + if self.set.contains(value) { + true + } else { + for match_type in &self.matches { + let result = match match_type { + MatchType::StartsWith(s) => value.starts_with(s), + MatchType::EndsWith(s) => value.ends_with(s), + MatchType::Glob(g) => g.matches(value), + MatchType::Regex(r) => r.is_match(value), + }; + if result { + return true; + } + } + false + } + } + + pub fn extend(&mut self, other: Self) { + self.set.extend(other.set); + self.matches.extend(other.matches); + } +} diff --git a/crates/store/src/backend/memory/main.rs b/crates/store/src/backend/memory/main.rs new file mode 100644 index 00000000..9274331f --- /dev/null +++ b/crates/store/src/backend/memory/main.rs @@ -0,0 +1,298 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + fs::File, + io::{BufRead, BufReader}, +}; + +use utils::config::{ + utils::{AsKey, ParseValue}, + Config, +}; + +use crate::Value; + +use super::{glob::GlobPattern, LookupList, LookupMap, MatchType, MemoryStore}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LookupType { + List, + Glob, + Regex, + Map, +} + +#[derive(Debug, Clone)] +pub struct LookupFormat { + pub lookup_type: LookupType, + pub comment: Option, + pub separator: Option, +} + +impl MemoryStore { + pub async fn open(config: &Config, prefix: impl AsKey) -> crate::Result { + let prefix = prefix.as_key(); + + let lookup_type = config.property_require::((&prefix, "type"))?; + let format = LookupFormat { + lookup_type, + comment: config.value((&prefix, "comment")).map(|s| s.to_string()), + separator: config.value((&prefix, "separator")).map(|s| s.to_string()), + }; + + Ok(match lookup_type { + LookupType::Map => { + MemoryStore::Map(parse_lookup_list(config, (&prefix, "values"), format)?) + } + _ => MemoryStore::List(parse_lookup_list(config, (&prefix, "values"), format)?), + }) + } +} + +fn parse_lookup_list( + config: &Config, + key: K, + format: LookupFormat, +) -> utils::config::Result { + let mut list = T::default(); + let mut last_failed = false; + for (_, mut value) in config.values(key.clone()) { + if let Some(new_value) = value.strip_prefix("fallback+") { + if last_failed { + value = new_value; + } else { + continue; + } + } + last_failed = false; + + if value.starts_with("https://") || value.starts_with("http://") { + match tokio::task::block_in_place(|| { + reqwest::blocking::get(value).and_then(|r| { + if r.status().is_success() { + r.bytes().map(Ok) + } else { + Ok(Err(r)) + } + }) + }) { + Ok(Ok(bytes)) => { + match list.insert_lines(&*bytes, &format, value.ends_with(".gz")) { + Ok(_) => continue, + Err(err) => { + tracing::warn!( + "Failed to read list {key:?} from {value:?}: {err}", + key = key.as_key(), + value = value, + err = err + ); + } + } + } + Ok(Err(response)) => { + tracing::warn!( + "Failed to fetch list {key:?} from {value:?}: Status {status}", + key = key.as_key(), + value = value, + status = response.status() + ); + } + Err(err) => { + tracing::warn!( + "Failed to fetch list {key:?} from {value:?}: {err}", + key = key.as_key(), + value = value, + err = err + ); + } + } + last_failed = true; + } else if let Some(path) = value.strip_prefix("file://") { + list.insert_lines( + File::open(path).map_err(|err| { + format!( + "Failed to read file {path:?} for list {}: {err}", + key.as_key() + ) + })?, + &format, + value.ends_with(".gz"), + ) + .map_err(|err| { + format!( + "Failed to read file {path:?} for list {}: {err}", + key.as_key() + ) + })?; + } else { + list.insert(value.to_string(), &format); + } + } + Ok(list) +} + +pub trait InsertLine: Default { + fn insert(&mut self, entry: String, format: &LookupFormat); + fn insert_lines( + &mut self, + reader: R, + format: &LookupFormat, + decompress: bool, + ) -> Result<(), std::io::Error> { + let reader: Box = if decompress { + Box::new(flate2::read::GzDecoder::new(reader)) + } else { + Box::new(reader) + }; + + for line in BufReader::new(reader).lines() { + let line_ = line?; + let line = line_.trim(); + if !line.is_empty() + && format + .comment + .as_ref() + .map_or(true, |c| !line.starts_with(c)) + { + self.insert(line.to_string(), format); + } + } + Ok(()) + } +} + +impl InsertLine for LookupList { + fn insert(&mut self, entry: String, format: &LookupFormat) { + match format.lookup_type { + LookupType::List => { + self.set.insert(entry); + } + LookupType::Glob => { + let n_wildcards = entry + .as_bytes() + .iter() + .filter(|&&ch| ch == b'*' || ch == b'?') + .count(); + if n_wildcards > 0 { + if n_wildcards == 1 { + if let Some(s) = entry.strip_prefix('*') { + if !s.is_empty() { + self.matches.push(MatchType::EndsWith(s.to_string())); + } + return; + } else if let Some(s) = entry.strip_suffix('*') { + if !s.is_empty() { + self.matches.push(MatchType::StartsWith(s.to_string())); + } + return; + } + } + self.matches + .push(MatchType::Glob(GlobPattern::compile(&entry, false))); + } else { + self.set.insert(entry); + } + } + LookupType::Regex => match regex::Regex::new(&entry) { + Ok(regex) => { + self.matches.push(MatchType::Regex(regex)); + } + Err(err) => { + tracing::warn!("Invalid regular expression {:?}: {}", entry, err); + } + }, + LookupType::Map => unreachable!(), + } + } +} + +impl InsertLine for LookupMap { + fn insert(&mut self, entry: String, format: &LookupFormat) { + let (key, value) = entry + .split_once(format.separator.as_deref().unwrap_or(" ")) + .unwrap_or((entry.as_str(), "")); + let key = key.trim(); + if key.is_empty() { + return; + } else if value.is_empty() { + self.insert(key.to_string(), Value::Null); + return; + } + let mut has_digit = false; + let mut has_dots = false; + let mut has_other = false; + + for (pos, ch) in value.bytes().enumerate() { + if ch.is_ascii_digit() { + has_digit = true; + } else if ch == b'.' { + has_dots = true; + } else if pos > 0 || ch != b'-' { + has_other = true; + } + } + + let value = if has_other || !has_digit { + Value::Text(value.to_string().into()) + } else if has_dots { + value + .parse() + .map(Value::Float) + .unwrap_or_else(|_| Value::Text(value.to_string().into())) + } else { + value + .parse() + .map(Value::Integer) + .unwrap_or_else(|_| Value::Text(value.to_string().into())) + }; + + self.insert(key.to_string(), value); + } +} + +impl Default for LookupFormat { + fn default() -> Self { + Self { + lookup_type: LookupType::Glob, + comment: Default::default(), + separator: Default::default(), + } + } +} + +impl ParseValue for LookupType { + fn parse_value(key: impl AsKey, value: &str) -> utils::config::Result { + match value { + "list" => Ok(LookupType::List), + "glob" => Ok(LookupType::Glob), + "regex" => Ok(LookupType::Regex), + "map" => Ok(LookupType::Map), + _ => Err(format!( + "Invalid value for lookup type {key:?}: {value:?}", + key = key.as_key(), + value = value + )), + } + } +} diff --git a/crates/store/src/backend/memory/mod.rs b/crates/store/src/backend/memory/mod.rs new file mode 100644 index 00000000..a6651ce0 --- /dev/null +++ b/crates/store/src/backend/memory/mod.rs @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +pub mod glob; +pub mod lookup; +pub mod main; + +use ahash::{AHashMap, AHashSet}; + +use crate::Value; + +use self::glob::GlobPattern; + +pub enum MemoryStore { + List(LookupList), + Map(LookupMap), +} + +#[derive(Default)] +pub struct LookupList { + pub set: AHashSet, + pub matches: Vec, +} + +pub type LookupMap = AHashMap>; + +pub enum MatchType { + StartsWith(String), + EndsWith(String), + Glob(GlobPattern), + Regex(regex::Regex), +} diff --git a/crates/store/src/backend/mod.rs b/crates/store/src/backend/mod.rs index 0ac1d8bd..d7fe5104 100644 --- a/crates/store/src/backend/mod.rs +++ b/crates/store/src/backend/mod.rs @@ -26,6 +26,7 @@ pub mod elastic; #[cfg(feature = "foundation")] pub mod foundationdb; pub mod fs; +pub mod memory; #[cfg(feature = "mysql")] pub mod mysql; #[cfg(feature = "postgres")] diff --git a/crates/store/src/backend/mysql/lookup.rs b/crates/store/src/backend/mysql/lookup.rs new file mode 100644 index 00000000..8bc7f442 --- /dev/null +++ b/crates/store/src/backend/mysql/lookup.rs @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use mysql_async::{prelude::Queryable, Params, Row}; + +use crate::{IntoRows, QueryResult, QueryType, Value}; + +use super::MysqlStore; + +impl MysqlStore { + pub(crate) async fn query( + &self, + query: &str, + params: Vec>, + ) -> crate::Result { + let mut conn = self.conn_pool.get_conn().await?; + let s = conn.prep(query).await?; + let params = Params::Positional(params.into_iter().map(Into::into).collect()); + + match T::query_type() { + QueryType::Execute => conn.exec_drop(s, params).await.map_or_else( + |e| Err(e.into()), + |_| Ok(T::from_exec(conn.affected_rows() as usize)), + ), + QueryType::Exists => conn + .exec_first::(s, params) + .await + .map_or_else(|e| Err(e.into()), |r| Ok(T::from_exists(r.is_some()))), + QueryType::QueryOne => conn + .exec_first::(s, params) + .await + .map_or_else(|e| Err(e.into()), |r| Ok(T::from_query_one(r))), + QueryType::QueryAll => conn + .exec::(s, params) + .await + .map_or_else(|e| Err(e.into()), |r| Ok(T::from_query_all(r))), + } + } +} + +impl From> for mysql_async::Value { + fn from(value: crate::Value) -> Self { + match value { + crate::Value::Integer(i) => mysql_async::Value::Int(i), + crate::Value::Bool(b) => mysql_async::Value::Int(b as i64), + crate::Value::Float(f) => mysql_async::Value::Double(f), + crate::Value::Text(t) => mysql_async::Value::Bytes(t.into_owned().into_bytes()), + crate::Value::Blob(b) => mysql_async::Value::Bytes(b.into_owned()), + crate::Value::Null => mysql_async::Value::NULL, + } + } +} + +impl From for crate::Value<'static> { + fn from(value: mysql_async::Value) -> Self { + match value { + mysql_async::Value::Int(i) => Self::Integer(i), + mysql_async::Value::UInt(i) => Self::Integer(i as i64), + mysql_async::Value::Double(f) => Self::Float(f), + mysql_async::Value::Bytes(b) => String::from_utf8(b).map_or_else( + |e| Self::Blob(e.into_bytes().into()), + |s| Self::Text(s.into()), + ), + mysql_async::Value::NULL => Self::Null, + mysql_async::Value::Float(f) => Self::Float(f as f64), + mysql_async::Value::Date(_, _, _, _, _, _, _) + | mysql_async::Value::Time(_, _, _, _, _, _) => Self::Text(value.as_sql(true).into()), + } + } +} + +impl IntoRows for Vec { + fn into_rows(self) -> crate::Rows { + crate::Rows { + rows: self + .into_iter() + .map(|r| crate::Row { + values: r + .unwrap_raw() + .into_iter() + .flatten() + .map(Into::into) + .collect(), + }) + .collect(), + } + } + + fn into_named_rows(self) -> crate::NamedRows { + crate::NamedRows { + names: self + .first() + .map(|r| r.columns().iter().map(|c| c.name_str().into()).collect()) + .unwrap_or_default(), + rows: self + .into_iter() + .map(|r| crate::Row { + values: r + .unwrap_raw() + .into_iter() + .flatten() + .map(Into::into) + .collect(), + }) + .collect(), + } + } + + fn into_row(self) -> Option { + unreachable!() + } +} + +impl IntoRows for Option { + fn into_row(self) -> Option { + self.map(|row| crate::Row { + values: row + .unwrap_raw() + .into_iter() + .flatten() + .map(Into::into) + .collect(), + }) + } + + fn into_rows(self) -> crate::Rows { + unreachable!() + } + + fn into_named_rows(self) -> crate::NamedRows { + unreachable!() + } +} diff --git a/crates/store/src/backend/mysql/main.rs b/crates/store/src/backend/mysql/main.rs index 57ae3b4e..aee2d503 100644 --- a/crates/store/src/backend/mysql/main.rs +++ b/crates/store/src/backend/mysql/main.rs @@ -21,7 +21,8 @@ * for more details. */ -use mysql_async::{prelude::Queryable, OptsBuilder, Pool, PoolConstraints, PoolOpts}; +use mysql_async::{prelude::Queryable, OptsBuilder, Pool, PoolConstraints, PoolOpts, SslOpts}; +use utils::config::utils::AsKey; use crate::{ SUBSPACE_BITMAPS, SUBSPACE_BLOBS, SUBSPACE_BLOB_DATA, SUBSPACE_COUNTERS, SUBSPACE_INDEXES, @@ -31,29 +32,36 @@ use crate::{ use super::MysqlStore; impl MysqlStore { - pub async fn open(config: &utils::config::Config) -> crate::Result { + pub async fn open(config: &utils::config::Config, prefix: impl AsKey) -> crate::Result { + let prefix = prefix.as_key(); let mut opts = OptsBuilder::default() - .ip_or_hostname(config.value_require("store.db.host")?.to_string()) - .user(config.value("store.db.user").map(|s| s.to_string())) - .pass(config.value("store.db.password").map(|s| s.to_string())) + .ip_or_hostname(config.value_require((&prefix, "host"))?.to_string()) + .user(config.value((&prefix, "user")).map(|s| s.to_string())) + .pass(config.value((&prefix, "password")).map(|s| s.to_string())) .db_name( config - .value_require("store.db.database")? + .value_require((&prefix, "database"))? .to_string() .into(), ) - .wait_timeout(config.property("store.db.timeout")?); - if let Some(port) = config.property("store.db.port")? { + .wait_timeout(config.property((&prefix, "timeout.wait"))?); + if let Some(port) = config.property((&prefix, "port"))? { opts = opts.tcp_port(port); } + if config.property_or_static::((&prefix, "tls.allow-invalid-certs"), "false")? { + opts = opts.ssl_opts(Some( + SslOpts::default().with_danger_accept_invalid_certs(true), + )); + } + // Configure connection pool let mut pool_min = PoolConstraints::default().min(); let mut pool_max = PoolConstraints::default().max(); - if let Some(n_size) = config.property::("store.db.pool.min-connections")? { + if let Some(n_size) = config.property::((&prefix, "pool.min-connections"))? { pool_min = n_size; } - if let Some(n_size) = config.property::("store.db.pool.max-connections")? { + if let Some(n_size) = config.property::((&prefix, "pool.max-connections"))? { pool_max = n_size; } opts = opts.pool_opts( diff --git a/crates/store/src/backend/mysql/mod.rs b/crates/store/src/backend/mysql/mod.rs index 115b2ab7..5b9b028d 100644 --- a/crates/store/src/backend/mysql/mod.rs +++ b/crates/store/src/backend/mysql/mod.rs @@ -24,6 +24,7 @@ use mysql_async::Pool; pub mod blob; +pub mod lookup; pub mod main; pub mod read; pub mod write; diff --git a/crates/store/src/backend/postgres/lookup.rs b/crates/store/src/backend/postgres/lookup.rs new file mode 100644 index 00000000..cce2dda9 --- /dev/null +++ b/crates/store/src/backend/postgres/lookup.rs @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use crate::{QueryResult, QueryType}; + +use bytes::BytesMut; +use futures::{pin_mut, TryStreamExt}; +use tokio_postgres::types::{FromSql, ToSql, Type}; + +use crate::IntoRows; + +use super::PostgresStore; + +impl PostgresStore { + pub(crate) async fn query( + &self, + query: &str, + params_: Vec>, + ) -> crate::Result { + let conn = self.conn_pool.get().await?; + let s = conn.prepare_cached(query).await?; + let params = params_ + .iter() + .map(|v| v as &(dyn tokio_postgres::types::ToSql + Sync)) + .collect::>(); + + match T::query_type() { + QueryType::Execute => conn + .execute(&s, params.as_slice()) + .await + .map_or_else(|e| Err(e.into()), |r| Ok(T::from_exec(r as usize))), + QueryType::Exists => { + let rows = conn.query_raw(&s, params.into_iter()).await?; + pin_mut!(rows); + rows.try_next() + .await + .map_or_else(|e| Err(e.into()), |r| Ok(T::from_exists(r.is_some()))) + } + QueryType::QueryOne => conn + .query_opt(&s, params.as_slice()) + .await + .map_or_else(|e| Err(e.into()), |r| Ok(T::from_query_one(r))), + QueryType::QueryAll => conn + .query(&s, params.as_slice()) + .await + .map_or_else(|e| Err(e.into()), |r| Ok(T::from_query_all(r))), + } + } +} + +impl ToSql for crate::Value<'_> { + fn to_sql( + &self, + ty: &tokio_postgres::types::Type, + out: &mut BytesMut, + ) -> Result> + where + Self: Sized, + { + match self { + crate::Value::Integer(v) => match *ty { + Type::CHAR => (*v as i8).to_sql(ty, out), + Type::INT2 => (*v as i16).to_sql(ty, out), + Type::INT4 => (*v as i32).to_sql(ty, out), + _ => v.to_sql(ty, out), + }, + crate::Value::Bool(v) => v.to_sql(ty, out), + crate::Value::Float(v) => { + if matches!(ty, &Type::FLOAT4) { + (*v as f32).to_sql(ty, out) + } else { + v.to_sql(ty, out) + } + } + crate::Value::Text(v) => v.to_sql(ty, out), + crate::Value::Blob(v) => v.to_sql(ty, out), + crate::Value::Null => None::.to_sql(ty, out), + } + } + + fn accepts(_: &tokio_postgres::types::Type) -> bool + where + Self: Sized, + { + true + } + + fn to_sql_checked( + &self, + ty: &tokio_postgres::types::Type, + out: &mut BytesMut, + ) -> Result> { + match self { + crate::Value::Integer(v) => match *ty { + Type::CHAR => (*v as i8).to_sql_checked(ty, out), + Type::INT2 => (*v as i16).to_sql_checked(ty, out), + Type::INT4 => (*v as i32).to_sql_checked(ty, out), + _ => v.to_sql_checked(ty, out), + }, + crate::Value::Bool(v) => v.to_sql_checked(ty, out), + crate::Value::Float(v) => { + if matches!(ty, &Type::FLOAT4) { + (*v as f32).to_sql_checked(ty, out) + } else { + v.to_sql_checked(ty, out) + } + } + crate::Value::Text(v) => v.to_sql_checked(ty, out), + crate::Value::Blob(v) => v.to_sql_checked(ty, out), + crate::Value::Null => None::.to_sql_checked(ty, out), + } + } +} + +impl IntoRows for Vec { + fn into_rows(self) -> crate::Rows { + crate::Rows { + rows: self + .into_iter() + .map(|r| crate::Row { + values: (0..r.len()) + .map(|idx| r.try_get(idx).unwrap_or(crate::Value::Null)) + .collect(), + }) + .collect(), + } + } + + fn into_named_rows(self) -> crate::NamedRows { + crate::NamedRows { + names: self + .first() + .map(|r| r.columns().iter().map(|c| c.name().to_string()).collect()) + .unwrap_or_default(), + rows: self + .into_iter() + .map(|r| crate::Row { + values: (0..r.len()) + .map(|idx| r.try_get(idx).unwrap_or(crate::Value::Null)) + .collect(), + }) + .collect(), + } + } + + fn into_row(self) -> Option { + unreachable!() + } +} + +impl IntoRows for Option { + fn into_row(self) -> Option { + self.map(|row| crate::Row { + values: (0..row.len()) + .map(|idx| row.try_get(idx).unwrap_or(crate::Value::Null)) + .collect(), + }) + } + + fn into_rows(self) -> crate::Rows { + unreachable!() + } + + fn into_named_rows(self) -> crate::NamedRows { + unreachable!() + } +} + +impl FromSql<'_> for crate::Value<'static> { + fn from_sql( + ty: &tokio_postgres::types::Type, + raw: &'_ [u8], + ) -> Result> { + match ty { + &Type::VARCHAR | &Type::TEXT | &Type::BPCHAR | &Type::NAME | &Type::UNKNOWN => { + String::from_sql(ty, raw).map(|s| crate::Value::Text(s.into())) + } + &Type::BOOL => bool::from_sql(ty, raw).map(crate::Value::Bool), + &Type::CHAR => i8::from_sql(ty, raw).map(|v| crate::Value::Integer(v as i64)), + &Type::INT2 => i16::from_sql(ty, raw).map(|v| crate::Value::Integer(v as i64)), + &Type::INT4 => i32::from_sql(ty, raw).map(|v| crate::Value::Integer(v as i64)), + &Type::INT8 | &Type::OID => i64::from_sql(ty, raw).map(crate::Value::Integer), + &Type::FLOAT4 | &Type::FLOAT8 => f64::from_sql(ty, raw).map(crate::Value::Float), + ty if (ty.name() == "citext" + || ty.name() == "ltree" + || ty.name() == "lquery" + || ty.name() == "ltxtquery") => + { + String::from_sql(ty, raw).map(|s| crate::Value::Text(s.into())) + } + _ => Vec::::from_sql(ty, raw).map(|b| crate::Value::Blob(b.into())), + } + } + + fn accepts(_: &tokio_postgres::types::Type) -> bool { + true + } +} diff --git a/crates/store/src/backend/postgres/main.rs b/crates/store/src/backend/postgres/main.rs index d12e31d7..0a4fb52a 100644 --- a/crates/store/src/backend/postgres/main.rs +++ b/crates/store/src/backend/postgres/main.rs @@ -33,32 +33,33 @@ use deadpool_postgres::{ Config, CreatePoolError, ManagerConfig, PoolConfig, RecyclingMethod, Runtime, }; use tokio_postgres::NoTls; -use utils::rustls_client_config; +use utils::{config::utils::AsKey, rustls_client_config}; impl PostgresStore { - pub async fn open(config: &utils::config::Config) -> crate::Result { + pub async fn open(config: &utils::config::Config, prefix: impl AsKey) -> crate::Result { + let prefix = prefix.as_key(); let mut cfg = Config::new(); cfg.dbname = config - .value_require("store.db.database")? + .value_require((&prefix, "database"))? .to_string() .into(); - cfg.host = config.value("store.db.host").map(|s| s.to_string()); - cfg.user = config.value("store.db.user").map(|s| s.to_string()); - cfg.password = config.value("store.db.password").map(|s| s.to_string()); - cfg.port = config.property("store.db.port")?; - cfg.connect_timeout = config.property("store.db.timeout")?; + cfg.host = config.value((&prefix, "host")).map(|s| s.to_string()); + cfg.user = config.value((&prefix, "user")).map(|s| s.to_string()); + cfg.password = config.value((&prefix, "password")).map(|s| s.to_string()); + cfg.port = config.property((&prefix, "port"))?; + cfg.connect_timeout = config.property((&prefix, "timeout.connect"))?; cfg.manager = Some(ManagerConfig { recycling_method: RecyclingMethod::Fast, }); - if let Some(max_conn) = config.property::("store.db.pool.max-connections")? { + if let Some(max_conn) = config.property::((&prefix, "pool.max-connections"))? { cfg.pool = PoolConfig::new(max_conn).into(); } let db = Self { - conn_pool: if config.property_or_static::("store.db.tls.enable", "false")? { + conn_pool: if config.property_or_static::((&prefix, "tls.enable"), "false")? { cfg.create_pool( Some(Runtime::Tokio1), MakeRustlsConnect::new(rustls_client_config( - config.property_or_static("store.db.tls.allow-invalid-certs", "false")?, + config.property_or_static((&prefix, "tls.allow-invalid-certs"), "false")?, )), )? } else { diff --git a/crates/store/src/backend/postgres/mod.rs b/crates/store/src/backend/postgres/mod.rs index ef723f81..eec30df9 100644 --- a/crates/store/src/backend/postgres/mod.rs +++ b/crates/store/src/backend/postgres/mod.rs @@ -24,6 +24,7 @@ use deadpool_postgres::{Pool, PoolError}; pub mod blob; +pub mod lookup; pub mod main; pub mod read; pub mod tls; diff --git a/crates/store/src/backend/rocksdb/main.rs b/crates/store/src/backend/rocksdb/main.rs index ede25e1b..5eb34e10 100644 --- a/crates/store/src/backend/rocksdb/main.rs +++ b/crates/store/src/backend/rocksdb/main.rs @@ -30,7 +30,10 @@ use rocksdb::{ }; use tokio::sync::oneshot; -use utils::{config::Config, UnwrapFailure}; +use utils::{ + config::{utils::AsKey, Config}, + UnwrapFailure, +}; use crate::{Deserialize, Error}; @@ -40,11 +43,12 @@ use super::{ }; impl RocksDbStore { - pub async fn open(config: &Config) -> crate::Result { + pub async fn open(config: &Config, prefix: impl AsKey) -> crate::Result { + let prefix = prefix.as_key(); // Create the database directory if it doesn't exist let idx_path: PathBuf = PathBuf::from( config - .value_require("store.db.path") + .value_require((&prefix, "path")) .failed("Invalid configuration file"), ); std::fs::create_dir_all(&idx_path).map_err(|err| { @@ -72,7 +76,7 @@ impl RocksDbStore { // Blobs let mut cf_opts = Options::default(); cf_opts.set_enable_blob_files(true); - cf_opts.set_min_blob_size(config.property_or_static("store.db.min-blob-size", "16834")?); + cf_opts.set_min_blob_size(config.property_or_static((&prefix, "min-blob-size"), "16834")?); cfs.push(ColumnFamilyDescriptor::new(CF_BLOB_DATA, cf_opts)); // Other cfs @@ -86,7 +90,7 @@ impl RocksDbStore { db_opts.create_if_missing(true); db_opts.set_max_background_jobs(std::cmp::max(num_cpus::get() as i32, 3)); db_opts.set_write_buffer_size( - config.property_or_static("store.db.write-buffer-size", "134217728")?, + config.property_or_static((&prefix, "write-buffer-size"), "134217728")?, ); Ok(RocksDbStore { @@ -96,7 +100,7 @@ impl RocksDbStore { worker_pool: rayon::ThreadPoolBuilder::new() .num_threads( config - .property::("store.db.pool.workers")? + .property::((&prefix, "pool.workers"))? .filter(|v| *v > 0) .unwrap_or_else(num_cpus::get), ) diff --git a/crates/store/src/backend/s3/mod.rs b/crates/store/src/backend/s3/mod.rs index 9aa3bba8..d683ec92 100644 --- a/crates/store/src/backend/s3/mod.rs +++ b/crates/store/src/backend/s3/mod.rs @@ -28,17 +28,21 @@ use s3::{ error::S3Error, Bucket, Region, }; -use utils::{codec::base32_custom::Base32Writer, config::Config}; +use utils::{ + codec::base32_custom::Base32Writer, + config::{utils::AsKey, Config}, +}; pub struct S3Store { bucket: Bucket, } impl S3Store { - pub async fn open(config: &Config) -> crate::Result { + pub async fn open(config: &Config, prefix: impl AsKey) -> crate::Result { // Obtain region and endpoint from config - let region = config.value_require("store.blob.s3.region")?; - let region = if let Some(endpoint) = config.value("store.blob.s3.endpoint") { + let prefix = prefix.as_key(); + let region = config.value_require((&prefix, "region"))?; + let region = if let Some(endpoint) = config.value((&prefix, "endpoint")) { Region::Custom { region: region.to_string(), endpoint: endpoint.to_string(), @@ -47,17 +51,17 @@ impl S3Store { region.parse().unwrap() }; let credentials = Credentials::new( - config.value("store.blob.s3.access-key"), - config.value("store.blob.s3.secret-key"), - config.value("store.blob.s3.security-token"), - config.value("store.blob.s3.session-token"), - config.value("store.blob.s3.profile"), + config.value((&prefix, "access-key")), + config.value((&prefix, "secret-key")), + config.value((&prefix, "security-token")), + config.value((&prefix, "session-token")), + config.value((&prefix, "profile")), )?; - let timeout = config.property_or_static::("store.blob.s3.timeout", "30s")?; + let timeout = config.property_or_static::((&prefix, "timeout"), "30s")?; Ok(S3Store { bucket: Bucket::new( - config.value_require("store.blob.s3.bucket")?, + config.value_require((&prefix, "bucket"))?, region, credentials, )? diff --git a/crates/store/src/backend/sqlite/lookup.rs b/crates/store/src/backend/sqlite/lookup.rs new file mode 100644 index 00000000..c9dc38bd --- /dev/null +++ b/crates/store/src/backend/sqlite/lookup.rs @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use rusqlite::{types::FromSql, Row, Rows, ToSql}; + +use crate::{IntoRows, QueryResult, QueryType, Value}; + +use super::SqliteStore; + +impl SqliteStore { + pub(crate) async fn query( + &self, + query: &str, + params_: Vec>, + ) -> crate::Result { + let conn = self.conn_pool.get()?; + self.spawn_worker(move || { + let mut s = conn.prepare_cached(query)?; + let params = params_ + .iter() + .map(|v| v as &(dyn rusqlite::types::ToSql)) + .collect::>(); + + match T::query_type() { + QueryType::Execute => s + .execute(params.as_slice()) + .map_or_else(|e| Err(e.into()), |r| Ok(T::from_exec(r))), + QueryType::Exists => s + .exists(params.as_slice()) + .map(T::from_exists) + .map_err(Into::into), + QueryType::QueryOne => s + .query(params.as_slice()) + .and_then(|mut rows| Ok(T::from_query_one(rows.next()?))) + .map_err(Into::into), + QueryType::QueryAll => Ok(T::from_query_all(s.query(params.as_slice())?)), + } + }) + .await + } +} + +impl ToSql for Value<'_> { + fn to_sql(&self) -> rusqlite::Result> { + match self { + Value::Integer(value) => value.to_sql(), + Value::Bool(value) => value.to_sql(), + Value::Float(value) => value.to_sql(), + Value::Text(value) => value.to_sql(), + Value::Blob(value) => value.to_sql(), + Value::Null => Ok(rusqlite::types::ToSqlOutput::Owned( + rusqlite::types::Value::Null, + )), + } + } +} + +impl FromSql for Value<'static> { + fn column_result(value: rusqlite::types::ValueRef<'_>) -> rusqlite::types::FromSqlResult { + Ok(match value { + rusqlite::types::ValueRef::Null => Value::Null, + rusqlite::types::ValueRef::Integer(v) => Value::Integer(v), + rusqlite::types::ValueRef::Real(v) => Value::Float(v), + rusqlite::types::ValueRef::Text(v) => { + Value::Text(String::from_utf8_lossy(v).into_owned().into()) + } + rusqlite::types::ValueRef::Blob(v) => Value::Blob(v.to_vec().into()), + }) + } +} + +impl IntoRows for Rows<'_> { + fn into_rows(mut self) -> crate::Rows { + let column_count = self.as_ref().map(|s| s.column_count()).unwrap_or_default(); + let mut rows = crate::Rows { rows: Vec::new() }; + + while let Ok(Some(row)) = self.next() { + rows.rows.push(crate::Row { + values: (0..column_count) + .map(|idx| row.get::<_, Value>(idx).unwrap_or(Value::Null)) + .collect(), + }); + } + + rows + } + + fn into_named_rows(mut self) -> crate::NamedRows { + let (column_count, names) = self + .as_ref() + .map(|s| { + ( + s.column_count(), + s.column_names() + .into_iter() + .map(String::from) + .collect::>(), + ) + }) + .unwrap_or((0, Vec::new())); + + let mut rows = crate::NamedRows { + names, + rows: Vec::new(), + }; + + while let Ok(Some(row)) = self.next() { + rows.rows.push(crate::Row { + values: (0..column_count) + .map(|idx| row.get::<_, Value>(idx).unwrap_or(Value::Null)) + .collect(), + }); + } + + rows + } + + fn into_row(self) -> Option { + unreachable!() + } +} + +impl IntoRows for Option<&Row<'_>> { + fn into_row(self) -> Option { + self.map(|row| crate::Row { + values: (0..row.as_ref().column_count()) + .map(|idx| row.get::<_, Value>(idx).unwrap_or(Value::Null)) + .collect(), + }) + } + + fn into_rows(self) -> crate::Rows { + todo!() + } + + fn into_named_rows(self) -> crate::NamedRows { + todo!() + } +} diff --git a/crates/store/src/backend/sqlite/main.rs b/crates/store/src/backend/sqlite/main.rs index 10e5e153..e6829992 100644 --- a/crates/store/src/backend/sqlite/main.rs +++ b/crates/store/src/backend/sqlite/main.rs @@ -23,7 +23,10 @@ use r2d2::Pool; use tokio::sync::oneshot; -use utils::{config::Config, UnwrapFailure}; +use utils::{ + config::{utils::AsKey, Config}, + UnwrapFailure, +}; use crate::{ SUBSPACE_BITMAPS, SUBSPACE_BLOBS, SUBSPACE_BLOB_DATA, SUBSPACE_COUNTERS, SUBSPACE_INDEXES, @@ -33,14 +36,15 @@ use crate::{ use super::{pool::SqliteConnectionManager, SqliteStore}; impl SqliteStore { - pub async fn open(config: &Config) -> crate::Result { + pub async fn open(config: &Config, prefix: impl AsKey) -> crate::Result { + let prefix = prefix.as_key(); let db = Self { conn_pool: Pool::builder() - .max_size(config.property_or_static("store.db.pool.max-connections", "10")?) + .max_size(config.property_or_static((&prefix, "pool.max-connections"), "10")?) .build( SqliteConnectionManager::file( config - .value_require("store.db.path") + .value_require((&prefix, "path")) .failed("Invalid configuration file"), ) .with_init(|c| { @@ -55,7 +59,7 @@ impl SqliteStore { worker_pool: rayon::ThreadPoolBuilder::new() .num_threads( config - .property::("store.db.pool.workers")? + .property::((&prefix, "pool.workers"))? .filter(|v| *v > 0) .unwrap_or_else(num_cpus::get), ) diff --git a/crates/store/src/backend/sqlite/mod.rs b/crates/store/src/backend/sqlite/mod.rs index 74ef4acb..c4de6d10 100644 --- a/crates/store/src/backend/sqlite/mod.rs +++ b/crates/store/src/backend/sqlite/mod.rs @@ -26,6 +26,7 @@ use r2d2::Pool; use self::pool::SqliteConnectionManager; pub mod blob; +pub mod lookup; pub mod main; pub mod pool; pub mod read; diff --git a/crates/store/src/config.rs b/crates/store/src/config.rs new file mode 100644 index 00000000..a7f4f9f5 --- /dev/null +++ b/crates/store/src/config.rs @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::sync::Arc; + +use async_trait::async_trait; +use utils::config::{utils::AsKey, Config}; + +use crate::{ + backend::{fs::FsStore, memory::MemoryStore}, + Lookup, LookupStore, Store, Stores, +}; + +#[cfg(feature = "s3")] +use crate::backend::s3::S3Store; + +#[cfg(feature = "postgres")] +use crate::backend::postgres::PostgresStore; + +#[cfg(feature = "mysql")] +use crate::backend::mysql::MysqlStore; + +#[cfg(feature = "sqlite")] +use crate::backend::sqlite::SqliteStore; + +#[cfg(feature = "foundation")] +use crate::backend::foundationdb::FdbStore; + +#[cfg(feature = "rocks")] +use crate::backend::rocksdb::RocksDbStore; + +#[cfg(feature = "elastic")] +use crate::backend::elastic::ElasticSearchStore; + +#[async_trait] +pub trait ConfigStore { + async fn parse_stores(&self) -> utils::config::Result; +} + +#[async_trait] +impl ConfigStore for Config { + async fn parse_stores(&self) -> utils::config::Result { + let mut config = Stores::default(); + + for id in self.sub_keys("store") { + // Parse directory + let protocol = self + .value_require(("store", id, "type"))? + .to_ascii_lowercase(); + let prefix = ("store", id); + let store_id = id.to_string(); + + let lookup_store = match protocol.as_str() { + #[cfg(feature = "rocks")] + "rocksdb" => { + let db: Store = RocksDbStore::open(self, prefix).await?.into(); + config.stores.insert(store_id.clone(), db.clone()); + config + .fts_stores + .insert(store_id.clone(), db.clone().into()); + config + .blob_stores + .insert(store_id.clone(), db.clone().into()); + db + } + #[cfg(feature = "foundation")] + "foundationdb" => { + let db: Store = FdbStore::open(self, prefix).await?.into(); + config.stores.insert(store_id.clone(), db.clone()); + config + .fts_stores + .insert(store_id.clone(), db.clone().into()); + config + .blob_stores + .insert(store_id.clone(), db.clone().into()); + db + } + #[cfg(feature = "postgres")] + "postgresql" => { + let db: Store = PostgresStore::open(self, prefix).await?.into(); + config.stores.insert(store_id.clone(), db.clone()); + config + .fts_stores + .insert(store_id.clone(), db.clone().into()); + config + .blob_stores + .insert(store_id.clone(), db.clone().into()); + db + } + #[cfg(feature = "mysql")] + "mysql" => { + let db: Store = MysqlStore::open(self, prefix).await?.into(); + config.stores.insert(store_id.clone(), db.clone()); + config + .fts_stores + .insert(store_id.clone(), db.clone().into()); + config + .blob_stores + .insert(store_id.clone(), db.clone().into()); + db + } + #[cfg(feature = "sqlite")] + "sqlite" => { + let db: Store = SqliteStore::open(self, prefix).await?.into(); + config.stores.insert(store_id.clone(), db.clone()); + config + .fts_stores + .insert(store_id.clone(), db.clone().into()); + config + .blob_stores + .insert(store_id.clone(), db.clone().into()); + db + } + "fs" => { + config + .blob_stores + .insert(store_id, FsStore::open(self, prefix).await?.into()); + continue; + } + #[cfg(feature = "s3")] + "s3" => { + config + .blob_stores + .insert(store_id, S3Store::open(self, prefix).await?.into()); + continue; + } + #[cfg(feature = "elastic")] + "elasticsearch" => { + config.fts_stores.insert( + store_id, + ElasticSearchStore::open(self, prefix).await?.into(), + ); + continue; + } + "memory" => { + let prefix = prefix.as_key(); + for lookup_id in self.sub_keys((&prefix, "lookup")) { + config.lookups.insert( + format!("{store_id}/{lookup_id}"), + Arc::new(Lookup { + store: MemoryStore::open( + self, + (prefix.as_str(), "lookup", lookup_id), + ) + .await? + .into(), + query: String::new(), + }), + ); + } + continue; + } + + unknown => { + tracing::debug!("Unknown directory type: {unknown:?}"); + continue; + } + }; + + // Add queries + let lookup_store: LookupStore = lookup_store.into(); + for lookup_id in self.sub_keys(("store", id, "query")) { + config.lookups.insert( + format!("{store_id}/{lookup_id}"), + Arc::new(Lookup { + store: lookup_store.clone(), + query: self.property_require(("store", id, "query", lookup_id))?, + }), + ); + } + config.lookup_stores.insert(store_id, lookup_store.clone()); + + // Run init queries on database + for (_, query) in self.values(("store", id, "init")) { + if let Err(err) = lookup_store.query::(query, Vec::new()).await { + tracing::warn!("Failed to initialize store {id:?}: {err}"); + } + } + } + + Ok(config) + } +} + +impl From for String { + fn from(err: crate::Error) -> Self { + match err { + crate::Error::InternalError(err) => err, + crate::Error::AssertValueFailed => unimplemented!(), + } + } +} diff --git a/crates/store/src/dispatch.rs b/crates/store/src/dispatch.rs index 9699b58f..8d869906 100644 --- a/crates/store/src/dispatch.rs +++ b/crates/store/src/dispatch.rs @@ -22,6 +22,7 @@ */ use std::{ + f32::consts::E, fmt::Display, ops::{BitAndAssign, Range}, }; @@ -31,9 +32,9 @@ use roaring::RoaringBitmap; use crate::{ fts::{index::FtsDocument, FtsFilter}, write::{key::KeySerializer, Batch, BitmapClass, ValueClass}, - BitmapKey, BlobStore, Deserialize, FtsStore, IterateParams, Key, Store, ValueKey, - SUBSPACE_BITMAPS, SUBSPACE_INDEXES, SUBSPACE_INDEX_VALUES, SUBSPACE_LOGS, SUBSPACE_VALUES, - U32_LEN, + BitmapKey, BlobStore, Deserialize, Error, FtsStore, IterateParams, Key, LookupStore, + QueryResult, Store, Value, ValueKey, SUBSPACE_BITMAPS, SUBSPACE_INDEXES, SUBSPACE_INDEX_VALUES, + SUBSPACE_LOGS, SUBSPACE_VALUES, U32_LEN, }; impl Store { @@ -353,55 +354,61 @@ impl Store { impl BlobStore { pub async fn get_blob(&self, key: &[u8], range: Range) -> crate::Result>> { match self { + Self::Store(store) => match store { + #[cfg(feature = "sqlite")] + Store::SQLite(store) => store.get_blob(key, range).await, + #[cfg(feature = "foundation")] + Store::FoundationDb(store) => store.get_blob(key, range).await, + #[cfg(feature = "postgres")] + Store::PostgreSQL(store) => store.get_blob(key, range).await, + #[cfg(feature = "mysql")] + Store::MySQL(store) => store.get_blob(key, range).await, + #[cfg(feature = "rocks")] + Store::RocksDb(store) => store.get_blob(key, range).await, + }, Self::Fs(store) => store.get_blob(key, range).await, #[cfg(feature = "s3")] Self::S3(store) => store.get_blob(key, range).await, - #[cfg(feature = "sqlite")] - Self::Sqlite(store) => store.get_blob(key, range).await, - #[cfg(feature = "foundation")] - Self::FoundationDb(store) => store.get_blob(key, range).await, - #[cfg(feature = "postgres")] - Self::PostgreSQL(store) => store.get_blob(key, range).await, - #[cfg(feature = "mysql")] - Self::MySQL(store) => store.get_blob(key, range).await, - #[cfg(feature = "rocks")] - Self::RocksDb(store) => store.get_blob(key, range).await, } } pub async fn put_blob(&self, key: &[u8], data: &[u8]) -> crate::Result<()> { match self { + Self::Store(store) => match store { + #[cfg(feature = "sqlite")] + Store::SQLite(store) => store.put_blob(key, data).await, + #[cfg(feature = "foundation")] + Store::FoundationDb(store) => store.put_blob(key, data).await, + #[cfg(feature = "postgres")] + Store::PostgreSQL(store) => store.put_blob(key, data).await, + #[cfg(feature = "mysql")] + Store::MySQL(store) => store.put_blob(key, data).await, + #[cfg(feature = "rocks")] + Store::RocksDb(store) => store.put_blob(key, data).await, + }, Self::Fs(store) => store.put_blob(key, data).await, #[cfg(feature = "s3")] Self::S3(store) => store.put_blob(key, data).await, - #[cfg(feature = "sqlite")] - Self::Sqlite(store) => store.put_blob(key, data).await, - #[cfg(feature = "foundation")] - Self::FoundationDb(store) => store.put_blob(key, data).await, - #[cfg(feature = "postgres")] - Self::PostgreSQL(store) => store.put_blob(key, data).await, - #[cfg(feature = "mysql")] - Self::MySQL(store) => store.put_blob(key, data).await, - #[cfg(feature = "rocks")] - Self::RocksDb(store) => store.put_blob(key, data).await, } } pub async fn delete_blob(&self, key: &[u8]) -> crate::Result { match self { + Self::Store(store) => match store { + #[cfg(feature = "sqlite")] + Store::SQLite(store) => store.delete_blob(key).await, + #[cfg(feature = "foundation")] + Store::FoundationDb(store) => store.delete_blob(key).await, + #[cfg(feature = "postgres")] + Store::PostgreSQL(store) => store.delete_blob(key).await, + #[cfg(feature = "mysql")] + Store::MySQL(store) => store.delete_blob(key).await, + #[cfg(feature = "rocks")] + Store::RocksDb(store) => store.delete_blob(key).await, + }, Self::Fs(store) => store.delete_blob(key).await, #[cfg(feature = "s3")] Self::S3(store) => store.delete_blob(key).await, - #[cfg(feature = "sqlite")] - Self::Sqlite(store) => store.delete_blob(key).await, - #[cfg(feature = "foundation")] - Self::FoundationDb(store) => store.delete_blob(key).await, - #[cfg(feature = "postgres")] - Self::PostgreSQL(store) => store.delete_blob(key).await, - #[cfg(feature = "mysql")] - Self::MySQL(store) => store.delete_blob(key).await, - #[cfg(feature = "rocks")] - Self::RocksDb(store) => store.delete_blob(key).await, } } } @@ -456,3 +463,30 @@ impl FtsStore { } } } + +impl LookupStore { + pub async fn query( + &self, + query: &str, + params: Vec>, + ) -> crate::Result { + let todo = true; + let result = match self { + LookupStore::Store(store) => { + match store { + Store::SQLite(store) => store.query(query, params).await, + //Store::FoundationDb(store) => store.query(query, params).await, + Store::PostgreSQL(store) => store.query(query, params).await, + Store::MySQL(store) => store.query(query, params).await, + //Store::RocksDb(store) => store.query(query, params).await, + _ => todo!(), + } + } + LookupStore::Memory(store) => store.query(query, params), + }; + + tracing::trace!( context = "store", event = "query", query = query, result = ?result); + + result + } +} diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 3d2065cf..df313f82 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -21,16 +21,18 @@ * for more details. */ -use std::{fmt::Display, sync::Arc}; +use std::{borrow::Cow, fmt::Display, sync::Arc}; pub mod backend; +pub mod config; pub mod dispatch; pub mod fts; pub mod query; pub mod write; pub use ahash; -use backend::fs::FsStore; +use ahash::AHashMap; +use backend::{fs::FsStore, memory::MemoryStore}; pub use blake3; pub use parking_lot; pub use rand; @@ -186,6 +188,21 @@ pub struct IterateParams { values: bool, } +#[derive(Clone, Default)] +pub struct Stores { + pub stores: AHashMap, + pub blob_stores: AHashMap, + pub fts_stores: AHashMap, + pub lookup_stores: AHashMap, + pub lookups: AHashMap>, +} + +#[derive(Clone)] +pub struct Lookup { + pub store: LookupStore, + pub query: String, +} + #[derive(Clone)] pub enum Store { #[cfg(feature = "sqlite")] @@ -202,19 +219,10 @@ pub enum Store { #[derive(Clone)] pub enum BlobStore { + Store(Store), Fs(Arc), #[cfg(feature = "s3")] S3(Arc), - #[cfg(feature = "sqlite")] - Sqlite(Arc), - #[cfg(feature = "foundation")] - FoundationDb(Arc), - #[cfg(feature = "postgres")] - PostgreSQL(Arc), - #[cfg(feature = "mysql")] - MySQL(Arc), - #[cfg(feature = "rocks")] - RocksDb(Arc), } #[derive(Clone)] @@ -224,6 +232,12 @@ pub enum FtsStore { ElasticSearch(Arc), } +#[derive(Clone)] +pub enum LookupStore { + Store(Store), + Memory(Arc), +} + #[cfg(feature = "sqlite")] impl From for Store { fn from(store: SqliteStore) -> Self { @@ -287,17 +301,285 @@ impl From for FtsStore { impl From for BlobStore { fn from(store: Store) -> Self { - match store { - #[cfg(feature = "sqlite")] - Store::SQLite(store) => Self::Sqlite(store), - #[cfg(feature = "foundation")] - Store::FoundationDb(store) => Self::FoundationDb(store), - #[cfg(feature = "postgres")] - Store::PostgreSQL(store) => Self::PostgreSQL(store), - #[cfg(feature = "mysql")] - Store::MySQL(store) => Self::MySQL(store), - #[cfg(feature = "rocks")] - Store::RocksDb(store) => Self::RocksDb(store), + Self::Store(store) + } +} + +impl From for LookupStore { + fn from(store: Store) -> Self { + Self::Store(store) + } +} + +impl From for LookupStore { + fn from(store: MemoryStore) -> Self { + Self::Memory(Arc::new(store)) + } +} + +#[derive(Clone, Debug)] +pub enum Value<'x> { + Integer(i64), + Bool(bool), + Float(f64), + Text(Cow<'x, str>), + Blob(Cow<'x, [u8]>), + Null, +} + +impl<'x> Value<'x> { + pub fn to_str<'y: 'x>(&'y self) -> Cow<'x, str> { + match self { + Value::Text(s) => s.as_ref().into(), + Value::Integer(i) => Cow::Owned(i.to_string()), + Value::Bool(b) => Cow::Owned(b.to_string()), + Value::Float(f) => Cow::Owned(f.to_string()), + Value::Blob(b) => String::from_utf8_lossy(b.as_ref()), + Value::Null => Cow::Borrowed(""), } } } + +#[derive(Clone, Debug)] +pub struct Row { + pub values: Vec>, +} + +#[derive(Clone, Debug)] +pub struct Rows { + pub rows: Vec, +} + +#[derive(Clone, Debug)] +pub struct NamedRows { + pub names: Vec, + pub rows: Vec, +} + +#[derive(Clone, Copy)] +pub enum QueryType { + Execute, + Exists, + QueryAll, + QueryOne, +} + +pub trait QueryResult: Sync + Send + 'static { + fn from_exec(items: usize) -> Self; + fn from_exists(exists: bool) -> Self; + fn from_query_one(items: impl IntoRows) -> Self; + fn from_query_all(items: impl IntoRows) -> Self; + + fn query_type() -> QueryType; +} + +pub trait IntoRows { + fn into_row(self) -> Option; + fn into_rows(self) -> Rows; + fn into_named_rows(self) -> NamedRows; +} + +impl QueryResult for Option { + fn query_type() -> QueryType { + QueryType::QueryOne + } + + fn from_exec(_: usize) -> Self { + unreachable!() + } + + fn from_exists(_: bool) -> Self { + unreachable!() + } + + fn from_query_all(_: impl IntoRows) -> Self { + unreachable!() + } + + fn from_query_one(items: impl IntoRows) -> Self { + items.into_row() + } +} + +impl QueryResult for Rows { + fn query_type() -> QueryType { + QueryType::QueryAll + } + + fn from_exec(_: usize) -> Self { + unreachable!() + } + + fn from_exists(_: bool) -> Self { + unreachable!() + } + + fn from_query_all(items: impl IntoRows) -> Self { + items.into_rows() + } + + fn from_query_one(_: impl IntoRows) -> Self { + unreachable!() + } +} + +impl QueryResult for NamedRows { + fn query_type() -> QueryType { + QueryType::QueryAll + } + + fn from_exec(_: usize) -> Self { + unreachable!() + } + + fn from_exists(_: bool) -> Self { + unreachable!() + } + + fn from_query_all(items: impl IntoRows) -> Self { + items.into_named_rows() + } + + fn from_query_one(_: impl IntoRows) -> Self { + unreachable!() + } +} + +impl QueryResult for bool { + fn query_type() -> QueryType { + QueryType::Exists + } + + fn from_exec(_: usize) -> Self { + unreachable!() + } + + fn from_exists(exists: bool) -> Self { + exists + } + + fn from_query_all(_: impl IntoRows) -> Self { + unreachable!() + } + + fn from_query_one(_: impl IntoRows) -> Self { + unreachable!() + } +} + +impl QueryResult for usize { + fn query_type() -> QueryType { + QueryType::Execute + } + + fn from_exec(items: usize) -> Self { + items + } + + fn from_exists(_: bool) -> Self { + unreachable!() + } + + fn from_query_all(_: impl IntoRows) -> Self { + unreachable!() + } + + fn from_query_one(_: impl IntoRows) -> Self { + unreachable!() + } +} + +impl<'x> From<&'x str> for Value<'x> { + fn from(value: &'x str) -> Self { + Self::Text(value.into()) + } +} + +impl<'x> From for Value<'x> { + fn from(value: String) -> Self { + Self::Text(value.into()) + } +} + +impl<'x> From<&'x String> for Value<'x> { + fn from(value: &'x String) -> Self { + Self::Text(value.into()) + } +} + +impl<'x> From> for Value<'x> { + fn from(value: Cow<'x, str>) -> Self { + Self::Text(value) + } +} + +impl<'x> From for Value<'x> { + fn from(value: bool) -> Self { + Self::Bool(value) + } +} + +impl<'x> From for Value<'x> { + fn from(value: i64) -> Self { + Self::Integer(value) + } +} + +impl<'x> From for Value<'x> { + fn from(value: u64) -> Self { + Self::Integer(value as i64) + } +} + +impl<'x> From for Value<'x> { + fn from(value: u32) -> Self { + Self::Integer(value as i64) + } +} + +impl<'x> From for Value<'x> { + fn from(value: f64) -> Self { + Self::Float(value) + } +} + +impl<'x> From<&'x [u8]> for Value<'x> { + fn from(value: &'x [u8]) -> Self { + Self::Blob(value.into()) + } +} + +impl<'x> From> for Value<'x> { + fn from(value: Vec) -> Self { + Self::Blob(value.into()) + } +} + +impl<'x> Value<'x> { + pub fn into_string(self) -> String { + match self { + Value::Text(s) => s.into_owned(), + Value::Integer(i) => i.to_string(), + Value::Bool(b) => b.to_string(), + Value::Float(f) => f.to_string(), + Value::Blob(b) => String::from_utf8_lossy(b.as_ref()).into_owned(), + Value::Null => String::new(), + } + } +} + +impl From for Vec { + fn from(value: Row) -> Self { + value.values.into_iter().map(|v| v.into_string()).collect() + } +} + +impl From for Vec { + fn from(value: Rows) -> Self { + value + .rows + .into_iter() + .flat_map(|v| v.values.into_iter().map(|v| v.into_string())) + .collect() + } +} diff --git a/crates/utils/src/config/utils.rs b/crates/utils/src/config/utils.rs index 70bd2b80..c5947a89 100644 --- a/crates/utils/src/config/utils.rs +++ b/crates/utils/src/config/utils.rs @@ -350,6 +350,18 @@ impl ParseValue for u32 { } } +impl ParseValue for i32 { + fn parse_value(key: impl AsKey, value: &str) -> super::Result { + value.parse().map_err(|_| { + format!( + "Invalid integer value {:?} for property {:?}.", + value, + key.as_key() + ) + }) + } +} + impl ParseValue for IpAddr { fn parse_value(key: impl AsKey, value: &str) -> super::Result { value.parse().map_err(|_| { diff --git a/tests/Cargo.toml b/tests/Cargo.toml index d059480c..af88585b 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -6,7 +6,7 @@ resolver = "2" [features] #default = ["sqlite", "foundationdb", "postgres", "mysql", "rocks", "elastic", "s3"] -default = ["rocks", "elastic"] +default = ["sqlite", "postgres", "mysql"] sqlite = ["store/sqlite"] foundationdb = ["store/foundation"] postgres = ["store/postgres"] @@ -54,7 +54,6 @@ base64 = "0.21" dashmap = "5.4" ahash = { version = "0.8" } serial_test = "2.0.0" -sqlx = { version = "0.7", features = [ "runtime-tokio-rustls", "postgres", "mysql", "sqlite" ] } num_cpus = "1.15.0" async-trait = "0.1.68" chrono = "0.4" diff --git a/tests/resources/smtp/config/rules-dynvalue.toml b/tests/resources/smtp/config/rules-dynvalue.toml index eafdad61..7e750fbe 100644 --- a/tests/resources/smtp/config/rules-dynvalue.toml +++ b/tests/resources/smtp/config/rules-dynvalue.toml @@ -61,44 +61,47 @@ test = [ ] expect = false -[directory."list_mx"] +[store."list_mx"] type = "memory" -[directory."list_mx".lookup] -domains = ["mx"] +[store."list_mx".lookup."domains"] +type = "list" +values = ["mx"] -[directory."list_foo"] +[store."list_foo"] type = "memory" -[directory."list_foo".lookup] -domains = ["foo"] +[store."list_foo".lookup."domains"] +type = "list" +values = ["foo"] -[directory."list_123"] +[store."list_123"] type = "memory" -[directory."list_123".lookup] -domains = ["123"] +[store."list_123".lookup."domains"] +type = "list" +values = ["123"] [maybe-eval."dyn_mx"] test = [ - {if = "mx", matches = "([^.]+)\.(.+)$", then = "list_${1}"}, + {if = "mx", matches = "([^.]+)\.(.+)$", then = "list_${1}/domains"}, {else = false} ] expect = "mx" [maybe-eval."dyn_foo"] test = [ - {if = "sender-domain", matches = "([^.]+)\.(.+)$", then = "list_${1}"}, + {if = "sender-domain", matches = "([^.]+)\.(.+)$", then = "list_${1}/domains"}, {else = false} ] expect = "foo" [maybe-eval."static_mx"] -test = "list_mx" +test = "list_mx/domains" expect = "mx" [maybe-eval."static_foo"] -test = "list_foo" +test = "list_foo/domains" expect = "foo" [maybe-eval."dyn_123"] -test = "list_${listener}" +test = "list_${listener}/domains" expect = "123" diff --git a/tests/resources/smtp/config/rules-eval.toml b/tests/resources/smtp/config/rules-eval.toml index cce78724..3133c327 100644 --- a/tests/resources/smtp/config/rules-eval.toml +++ b/tests/resources/smtp/config/rules-eval.toml @@ -164,8 +164,9 @@ nested-none-of-false = { none-of = [ ]} ]} -[directory."list"] +[store."list"] type = "memory" -[directory."list".lookup] -domains = ["mydomain1.org", "foo.net", "otherdomain.net"] +[store."list".lookup."domains"] +type = "list" +values = ["mydomain1.org", "foo.net", "otherdomain.net"] diff --git a/tests/src/directory/imap.rs b/tests/src/directory/imap.rs index 933ea975..f0651b27 100644 --- a/tests/src/directory/imap.rs +++ b/tests/src/directory/imap.rs @@ -48,11 +48,13 @@ async fn imap_directory() { ) .unwrap();*/ - // Obtain directory handle - let handle = parse_config().directories.remove("imap").unwrap(); - // Spawn mock LMTP server let shutdown = spawn_mock_imap_server(5); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + + // Obtain directory handle + let mut config = parse_config().await; + let handle = config.directories.directories.remove("imap").unwrap(); // Basic lookup let tests = vec![ @@ -105,8 +107,6 @@ async fn imap_directory() { item_clone, expected.append(n), )); - // FOX: This is a workaround for a bb8 bug, see: https://github.com/djc/bb8/issues/167 - tokio::time::sleep(std::time::Duration::from_millis(100)).await; } for (result, item, expected_result) in requests { assert_eq!( diff --git a/tests/src/directory/ldap.rs b/tests/src/directory/ldap.rs index 2ed81cb4..04aa86f2 100644 --- a/tests/src/directory/ldap.rs +++ b/tests/src/directory/ldap.rs @@ -39,17 +39,8 @@ async fn ldap_directory() { .unwrap();*/ // Obtain directory handle - let mut config = parse_config(); - let lookups = config.lookups; - let handle = config.directories.remove("ldap").unwrap(); - - // Text lookup - assert!(lookups - .get("ldap/domains") - .unwrap() - .contains("example.org") - .await - .unwrap()); + let mut config = parse_config().await; + let handle = config.directories.directories.remove("ldap").unwrap(); // Test authentication assert_eq!( diff --git a/tests/src/directory/mod.rs b/tests/src/directory/mod.rs index 598b5b8e..673d10b5 100644 --- a/tests/src/directory/mod.rs +++ b/tests/src/directory/mod.rs @@ -26,36 +26,27 @@ pub mod ldap; pub mod smtp; pub mod sql; -use directory::{config::ConfigDirectory, AddressMapping, DirectoryConfig}; +use ::smtp::core::Lookup; +use directory::{config::ConfigDirectory, AddressMapping, Directories}; use mail_send::Credentials; use rustls::{Certificate, PrivateKey, ServerConfig}; use rustls_pemfile::{certs, pkcs8_private_keys}; use std::{borrow::Cow, io::BufReader, path::PathBuf, sync::Arc}; +use store::{config::ConfigStore, LookupStore, Stores}; use tokio_rustls::TlsAcceptor; -const CONFIG: &str = r#" -[directory."sql"] -type = "sql" -address = "sqlite::memory:" -#address = "mysql://root:secret@localhost:3306/stalwart?ssl_mode=disabled" +use crate::store::TempDir; -[directory."sql".options] +const CONFIG: &str = r#" +[directory."sqlite"] +type = "sql" +store = "sqlite" + +[directory."sqlite".options] catch-all = true subaddressing = true -[directory."sql".pool] -max-connections = 1 - -[directory."sql".query] -name = "SELECT name, type, secret, description, quota FROM accounts WHERE name = ? AND active = true" -members = "SELECT member_of FROM group_members WHERE name = ?" -recipients = "SELECT name FROM emails WHERE address = ?" -emails = "SELECT address FROM emails WHERE name = ? AND type != 'list' ORDER BY type DESC, address ASC" -verify = "SELECT address FROM emails WHERE address LIKE '%' || ? || '%' AND type = 'primary' ORDER BY address LIMIT 5" -expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = ? AND l.type = 'list' ORDER BY p.address LIMIT 50" -domains = "SELECT 1 FROM emails WHERE address LIKE '%@' || ? LIMIT 1" - -[directory."sql".columns] +[directory."sqlite".columns] name = "name" description = "description" secret = "secret" @@ -63,6 +54,91 @@ email = "address" quota = "quota" type = "type" +[store."sqlite"] +type = "sqlite" +path = "{TMP}/auth.db" + +[store."sqlite".query] +name = "SELECT name, type, secret, description, quota FROM accounts WHERE name = ? AND active = true" +members = "SELECT member_of FROM group_members WHERE name = ?" +recipients = "SELECT name FROM emails WHERE address = ? ORDER BY name ASC" +emails = "SELECT address FROM emails WHERE name = ? AND type != 'list' ORDER BY type DESC, address ASC" +verify = "SELECT address FROM emails WHERE address LIKE '%' || ? || '%' AND type = 'primary' ORDER BY address LIMIT 5" +expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = ? AND l.type = 'list' ORDER BY p.address LIMIT 50" +domains = "SELECT 1 FROM emails WHERE address LIKE '%@' || ? LIMIT 1" + +############################################################################## + +[directory."postgresql"] +type = "sql" +store = "postgresql" + +[directory."postgresql".options] +catch-all = true +subaddressing = true + +[directory."postgresql".columns] +name = "name" +description = "description" +secret = "secret" +email = "address" +quota = "quota" +type = "type" + +[store."postgresql"] +type = "postgresql" +host = "localhost" +port = 5432 +database = "stalwart" +user = "postgres" +password = "mysecretpassword" + +[store."postgresql".query] +name = "SELECT name, type, secret, description, quota FROM accounts WHERE name = $1 AND active = true" +members = "SELECT member_of FROM group_members WHERE name = $1" +recipients = "SELECT name FROM emails WHERE address = $1 ORDER BY name ASC" +emails = "SELECT address FROM emails WHERE name = $1 AND type != 'list' ORDER BY type DESC, address ASC" +verify = "SELECT address FROM emails WHERE address LIKE '%' || $1 || '%' AND type = 'primary' ORDER BY address LIMIT 5" +expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = $1 AND l.type = 'list' ORDER BY p.address LIMIT 50" +domains = "SELECT 1 FROM emails WHERE address LIKE '%@' || $1 LIMIT 1" + +############################################################################## + +[directory."mysql"] +type = "sql" +store = "mysql" + +[directory."mysql".options] +catch-all = true +subaddressing = true + +[directory."mysql".columns] +name = "name" +description = "description" +secret = "secret" +email = "address" +quota = "quota" +type = "type" + +[store."mysql"] +type = "mysql" +host = "localhost" +port = 3307 +database = "stalwart" +user = "root" +password = "password" + +[store."mysql".query] +name = "SELECT name, type, secret, description, quota FROM accounts WHERE name = ? AND active = true" +members = "SELECT member_of FROM group_members WHERE name = ?" +recipients = "SELECT name FROM emails WHERE address = ? ORDER BY name ASC" +emails = "SELECT address FROM emails WHERE name = ? AND type != 'list' ORDER BY type DESC, address ASC" +verify = "SELECT address FROM emails WHERE address LIKE CONCAT('%', ?, '%') AND type = 'primary' ORDER BY address LIMIT 5" +expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = ? AND l.type = 'list' ORDER BY p.address LIMIT 50" +domains = "SELECT 1 FROM emails WHERE address LIKE CONCAT('%@', ?) LIMIT 1" + +############################################################################## + [directory."ldap"] type = "ldap" address = "ldap://localhost:3893" @@ -72,6 +148,10 @@ base-dn = "dc=example,dc=org" dn = "cn=serviceuser,ou=svcaccts,dc=example,dc=org" secret = "mysecret" +[directory."ldap".auth-bind] +enable = false +dn = "cn=?,ou=svcaccts,dc=example,dc=org" + [directory."ldap".options] catch-all = true subaddressing = true @@ -99,6 +179,8 @@ email = "mail" email-alias = "givenName" quota = "diskQuota" +############################################################################## + [directory."imap"] type = "imap" address = "127.0.0.1" @@ -111,6 +193,8 @@ max-connections = 5 implicit = true allow-invalid-certs = true +############################################################################## + [directory."smtp"] type = "lmtp" address = "127.0.0.1" @@ -131,6 +215,8 @@ allow-invalid-certs = true entries = 500 ttl = {positive = '10s', negative = '5s'} +############################################################################## + [directory."local"] type = "memory" @@ -170,16 +256,29 @@ description = "Sales Team" name = "support" description = "Support Team" -[directory."local".lookup] -domains = ["example.org"] - "#; -pub fn parse_config() -> DirectoryConfig { - utils::config::Config::new(CONFIG) - .unwrap() - .parse_directory() - .unwrap() +pub struct DirectoryStore { + pub store: LookupStore, +} + +pub struct DirectoryTest { + pub directories: Directories, + pub stores: Stores, + pub temp_dir: TempDir, +} + +pub async fn parse_config() -> DirectoryTest { + let temp_dir = TempDir::new("directory_tests", true); + let config_file = CONFIG.replace("{TMP}", &temp_dir.path.to_string_lossy()); + let config = utils::config::Config::new(&config_file).unwrap(); + let stores = config.parse_stores().await.unwrap(); + + DirectoryTest { + directories: config.parse_directory(&stores).unwrap(), + stores, + temp_dir, + } } const CERT: &str = "-----BEGIN CERTIFICATE----- @@ -390,23 +489,23 @@ impl core::fmt::Debug for Item { #[ignore] async fn lookup_local() { const LOOKUP_CONFIG: &str = r#" - [directory."local"] + [store."local"] type = "memory" - [directory."local".lookup."regex"] + [store."local".lookup."regex"] type = "regex" values = ["^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$", "^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$"] - [directory."local".lookup."glob"] + [store."local".lookup."glob"] type = "glob" values = ["*@example.org", "test@*", "localhost", "*+*@*.domain.net"] - [directory."local".lookup."list"] + [store."local".lookup."list"] type = "list" values = ["abc", "xyz", "123"] - [directory."local".lookup."suffix"] + [store."local".lookup."suffix"] type = "glob" comment = "//" values = ["https://publicsuffix.org/list/public_suffix_list.dat", "fallback+file://%PATH%/public_suffix_list.dat.gz"] @@ -434,7 +533,8 @@ async fn lookup_local() { ), ) .unwrap() - .parse_directory() + .parse_stores() + .await .unwrap() .lookups; @@ -455,9 +555,7 @@ async fn lookup_local() { ("suffix", "coco", false), ] { assert_eq!( - lookups - .get(&format!("local/{lookup}")) - .unwrap() + Lookup::from(lookups.get(&format!("local/{lookup}")).unwrap().clone()) .contains(item) .await .unwrap(), diff --git a/tests/src/directory/smtp.rs b/tests/src/directory/smtp.rs index a32831a1..b5595d9f 100644 --- a/tests/src/directory/smtp.rs +++ b/tests/src/directory/smtp.rs @@ -43,9 +43,11 @@ use super::dummy_tls_acceptor; async fn smtp_directory() { // Spawn mock LMTP server let shutdown = spawn_mock_lmtp_server(5); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; // Obtain directory handle - let handle = parse_config().directories.remove("smtp").unwrap(); + let mut config = parse_config().await; + let handle = config.directories.directories.remove("smtp").unwrap(); // Basic lookup let tests = vec![ @@ -153,10 +155,6 @@ async fn smtp_directory() { // Verify that caching works TcpStream::connect("127.0.0.1:9199").await.unwrap_err(); - assert_eq!( - handle.type_name(), - "directory::cache::CachedDirectory" - ); let mut requests = Vec::new(); for n in 0..100 { diff --git a/tests/src/directory/sql.rs b/tests/src/directory/sql.rs index f94a7fb2..631bc193 100644 --- a/tests/src/directory/sql.rs +++ b/tests/src/directory/sql.rs @@ -21,11 +21,16 @@ * for more details. */ -use directory::{Directory, Principal, Type}; +use ahash::AHashMap; +use directory::{Principal, Type}; use mail_send::Credentials; +use smtp::core::Lookup; +use store::{LookupStore, Store}; use crate::directory::parse_config; +use super::DirectoryStore; + #[tokio::test] async fn sql_directory() { // Enable logging @@ -36,317 +41,426 @@ async fn sql_directory() { ) .unwrap();*/ + // Parse config + let mut config = parse_config().await; + let lookups = config + .stores + .lookups + .into_iter() + .map(|(k, v)| (k, Lookup::from(v))) + .collect::>(); + // Obtain directory handle - let mut config = parse_config(); - let lookups = config.lookups; - let handle = config.directories.remove("sql").unwrap(); + for directory_id in ["sqlite", "postgresql", "mysql"] { + println!("Testing SQL directory {:?}", directory_id); + let handle = config.directories.directories.remove(directory_id).unwrap(); + let store = DirectoryStore { + store: config.stores.lookup_stores.remove(directory_id).unwrap(), + }; - // Create tables - create_test_directory(handle.as_ref()).await; + // Create tables + store.create_test_directory().await; - // Create test users - create_test_user(handle.as_ref(), "john", "12345", "John Doe").await; - create_test_user(handle.as_ref(), "jane", "abcde", "Jane Doe").await; - create_test_user( - handle.as_ref(), - "bill", - "$2y$05$bvIG6Nmid91Mu9RcmmWZfO5HJIMCT8riNW0hEp8f6/FuA2/mHZFpe", - "Bill Foobar", - ) - .await; - set_test_quota(handle.as_ref(), "bill", 500000).await; + // Create test users + store.create_test_user("john", "12345", "John Doe").await; + store.create_test_user("jane", "abcde", "Jane Doe").await; + store + .create_test_user( + "bill", + "$2y$05$bvIG6Nmid91Mu9RcmmWZfO5HJIMCT8riNW0hEp8f6/FuA2/mHZFpe", + "Bill Foobar", + ) + .await; + store.set_test_quota("bill", 500000).await; - // Create test groups - create_test_group(handle.as_ref(), "sales", "Sales Team").await; - create_test_group(handle.as_ref(), "support", "Support Team").await; + // Create test groups + store.create_test_group("sales", "Sales Team").await; + store.create_test_group("support", "Support Team").await; - // Link users to groups - add_to_group(handle.as_ref(), "john", "sales").await; - add_to_group(handle.as_ref(), "jane", "sales").await; - add_to_group(handle.as_ref(), "jane", "support").await; + // Link users to groups + store.add_to_group("john", "sales").await; + store.add_to_group("jane", "sales").await; + store.add_to_group("jane", "support").await; - // Add email addresses - link_test_address(handle.as_ref(), "john", "john@example.org", "primary").await; - link_test_address(handle.as_ref(), "jane", "jane@example.org", "primary").await; - link_test_address(handle.as_ref(), "bill", "bill@example.org", "primary").await; + // Add email addresses + store + .link_test_address("john", "john@example.org", "primary") + .await; + store + .link_test_address("jane", "jane@example.org", "primary") + .await; + store + .link_test_address("bill", "bill@example.org", "primary") + .await; - // Add aliases and lists - link_test_address(handle.as_ref(), "john", "john.doe@example.org", "alias").await; - link_test_address(handle.as_ref(), "john", "jdoe@example.org", "alias").await; - link_test_address(handle.as_ref(), "john", "info@example.org", "list").await; - link_test_address(handle.as_ref(), "jane", "info@example.org", "list").await; - link_test_address(handle.as_ref(), "bill", "info@example.org", "list").await; + // Add aliases and lists + store + .link_test_address("john", "john.doe@example.org", "alias") + .await; + store + .link_test_address("john", "jdoe@example.org", "alias") + .await; + store + .link_test_address("john", "info@example.org", "list") + .await; + store + .link_test_address("jane", "info@example.org", "list") + .await; + store + .link_test_address("bill", "info@example.org", "list") + .await; - // Add catch-all user - create_test_user(handle.as_ref(), "robert", "abcde", "Robert Foobar").await; - link_test_address(handle.as_ref(), "robert", "robert@catchall.org", "primary").await; - link_test_address(handle.as_ref(), "robert", "@catchall.org", "alias").await; + // Add catch-all user + store + .create_test_user("robert", "abcde", "Robert Foobar") + .await; + store + .link_test_address("robert", "robert@catchall.org", "primary") + .await; + store + .link_test_address("robert", "@catchall.org", "alias") + .await; - // Text lookup - assert!(lookups - .get("sql/domains") - .unwrap() - .contains("example.org") - .await - .unwrap()); - - // Test authentication - assert_eq!( - handle - .authenticate(&Credentials::Plain { - username: "john".to_string(), - secret: "12345".to_string() - }) - .await + // Text lookup + assert!(lookups + .get(&format!("{}/domains", directory_id)) .unwrap() - .unwrap(), - Principal { - name: "john".to_string(), - description: "John Doe".to_string().into(), - secrets: vec!["12345".to_string()], - typ: Type::Individual, - member_of: vec!["sales".to_string()], - ..Default::default() - } - ); - assert_eq!( - handle + .contains("example.org") + .await + .unwrap()); + + // Test authentication + assert_eq!( + handle + .authenticate(&Credentials::Plain { + username: "john".to_string(), + secret: "12345".to_string() + }) + .await + .unwrap() + .unwrap(), + Principal { + name: "john".to_string(), + description: "John Doe".to_string().into(), + secrets: vec!["12345".to_string()], + typ: Type::Individual, + member_of: vec!["sales".to_string()], + ..Default::default() + } + ); + assert_eq!( + handle + .authenticate(&Credentials::Plain { + username: "bill".to_string(), + secret: "password".to_string() + }) + .await + .unwrap() + .unwrap(), + Principal { + name: "bill".to_string(), + description: "Bill Foobar".to_string().into(), + secrets: vec![ + "$2y$05$bvIG6Nmid91Mu9RcmmWZfO5HJIMCT8riNW0hEp8f6/FuA2/mHZFpe".to_string() + ], + typ: Type::Individual, + quota: 500000, + ..Default::default() + } + ); + assert!(handle .authenticate(&Credentials::Plain { username: "bill".to_string(), - secret: "password".to_string() + secret: "invalid".to_string() }) .await .unwrap() - .unwrap(), - Principal { - name: "bill".to_string(), - description: "Bill Foobar".to_string().into(), - secrets: vec![ - "$2y$05$bvIG6Nmid91Mu9RcmmWZfO5HJIMCT8riNW0hEp8f6/FuA2/mHZFpe".to_string() - ], - typ: Type::Individual, - quota: 500000, - ..Default::default() - } - ); - assert!(handle - .authenticate(&Credentials::Plain { - username: "bill".to_string(), - secret: "invalid".to_string() - }) - .await - .unwrap() - .is_none()); + .is_none()); - // Get user by name - assert_eq!( - handle.principal("jane").await.unwrap().unwrap(), - Principal { - name: "jane".to_string(), - description: "Jane Doe".to_string().into(), - typ: Type::Individual, - secrets: vec!["abcde".to_string()], - member_of: vec!["sales".to_string(), "support".to_string()], - ..Default::default() - } - ); + // Get user by name + assert_eq!( + handle.principal("jane").await.unwrap().unwrap(), + Principal { + name: "jane".to_string(), + description: "Jane Doe".to_string().into(), + typ: Type::Individual, + secrets: vec!["abcde".to_string()], + member_of: vec!["sales".to_string(), "support".to_string()], + ..Default::default() + } + ); - // Get group by name - assert_eq!( - handle.principal("sales").await.unwrap().unwrap(), - Principal { - name: "sales".to_string(), - description: "Sales Team".to_string().into(), - typ: Type::Group, - ..Default::default() - } - ); + // Get group by name + assert_eq!( + handle.principal("sales").await.unwrap().unwrap(), + Principal { + name: "sales".to_string(), + description: "Sales Team".to_string().into(), + typ: Type::Group, + ..Default::default() + } + ); - // Emails by id - assert_eq!( - handle.emails_by_name("john").await.unwrap(), - vec![ - "john@example.org".to_string(), - "jdoe@example.org".to_string(), - "john.doe@example.org".to_string(), - ] - ); - assert_eq!( - handle.emails_by_name("bill").await.unwrap(), - vec!["bill@example.org".to_string(),] - ); + // Emails by id + assert_eq!( + handle.emails_by_name("john").await.unwrap(), + vec![ + "john@example.org".to_string(), + "jdoe@example.org".to_string(), + "john.doe@example.org".to_string(), + ] + ); + assert_eq!( + handle.emails_by_name("bill").await.unwrap(), + vec!["bill@example.org".to_string(),] + ); - // Ids by email - assert_eq!( - handle.names_by_email("jane@example.org").await.unwrap(), - vec!["jane".to_string()] - ); - assert_eq!( - handle.names_by_email("info@example.org").await.unwrap(), - vec!["bill".to_string(), "jane".to_string(), "john".to_string()] - ); - assert_eq!( - handle - .names_by_email("jane+alias@example.org") - .await - .unwrap(), - vec!["jane".to_string()] - ); - assert_eq!( - handle - .names_by_email("info+alias@example.org") - .await - .unwrap(), - vec!["bill".to_string(), "jane".to_string(), "john".to_string()] - ); - assert_eq!( - handle.names_by_email("unknown@example.org").await.unwrap(), - Vec::::new() - ); - assert_eq!( - handle - .names_by_email("anything@catchall.org") - .await - .unwrap(), - vec!["robert".to_string()] - ); + // Ids by email + assert_eq!( + handle.names_by_email("jane@example.org").await.unwrap(), + vec!["jane".to_string()] + ); + assert_eq!( + handle.names_by_email("info@example.org").await.unwrap(), + vec!["bill".to_string(), "jane".to_string(), "john".to_string()] + ); + assert_eq!( + handle + .names_by_email("jane+alias@example.org") + .await + .unwrap(), + vec!["jane".to_string()] + ); + assert_eq!( + handle + .names_by_email("info+alias@example.org") + .await + .unwrap(), + vec!["bill".to_string(), "jane".to_string(), "john".to_string()] + ); + assert_eq!( + handle.names_by_email("unknown@example.org").await.unwrap(), + Vec::::new() + ); + assert_eq!( + handle + .names_by_email("anything@catchall.org") + .await + .unwrap(), + vec!["robert".to_string()] + ); - // Domain validation - assert!(handle.is_local_domain("example.org").await.unwrap()); - assert!(!handle.is_local_domain("other.org").await.unwrap()); + // Domain validation + assert!(handle.is_local_domain("example.org").await.unwrap()); + assert!(!handle.is_local_domain("other.org").await.unwrap()); - // RCPT TO - assert!(handle.rcpt("jane@example.org").await.unwrap()); - assert!(handle.rcpt("info@example.org").await.unwrap()); - assert!(handle.rcpt("jane+alias@example.org").await.unwrap()); - assert!(handle.rcpt("info+alias@example.org").await.unwrap()); - assert!(handle.rcpt("random_user@catchall.org").await.unwrap()); - assert!(!handle.rcpt("invalid@example.org").await.unwrap()); + // RCPT TO + assert!(handle.rcpt("jane@example.org").await.unwrap()); + assert!(handle.rcpt("info@example.org").await.unwrap()); + assert!(handle.rcpt("jane+alias@example.org").await.unwrap()); + assert!(handle.rcpt("info+alias@example.org").await.unwrap()); + assert!(handle.rcpt("random_user@catchall.org").await.unwrap()); + assert!(!handle.rcpt("invalid@example.org").await.unwrap()); - // VRFY - assert_eq!( - handle.vrfy("jane").await.unwrap(), - vec!["jane@example.org".to_string()] - ); - assert_eq!( - handle.vrfy("john").await.unwrap(), - vec!["john@example.org".to_string()] - ); - assert_eq!( - handle.vrfy("jane+alias@example").await.unwrap(), - vec!["jane@example.org".to_string()] - ); - assert_eq!(handle.vrfy("info").await.unwrap(), Vec::::new()); - assert_eq!(handle.vrfy("invalid").await.unwrap(), Vec::::new()); + // VRFY + assert_eq!( + handle.vrfy("jane").await.unwrap(), + vec!["jane@example.org".to_string()] + ); + assert_eq!( + handle.vrfy("john").await.unwrap(), + vec!["john@example.org".to_string()] + ); + assert_eq!( + handle.vrfy("jane+alias@example").await.unwrap(), + vec!["jane@example.org".to_string()] + ); + assert_eq!(handle.vrfy("info").await.unwrap(), Vec::::new()); + assert_eq!(handle.vrfy("invalid").await.unwrap(), Vec::::new()); - // EXPN - assert_eq!( - handle.expn("info@example.org").await.unwrap(), - vec![ - "bill@example.org".to_string(), - "jane@example.org".to_string(), - "john@example.org".to_string() - ] - ); - assert_eq!( - handle.expn("john@example.org").await.unwrap(), - Vec::::new() - ); -} - -pub async fn create_test_directory(handle: &dyn Directory) { - // Create tables - for query in [ - "CREATE TABLE accounts (name TEXT PRIMARY KEY, secret TEXT, description TEXT, type TEXT NOT NULL, quota INTEGER DEFAULT 0, active BOOLEAN DEFAULT 1)", - "CREATE TABLE group_members (name TEXT NOT NULL, member_of TEXT NOT NULL, PRIMARY KEY (name, member_of))", - "CREATE TABLE emails (name TEXT NOT NULL, address TEXT NOT NULL, type TEXT, PRIMARY KEY (name, address))", - "INSERT INTO accounts (name, secret, type) VALUES ('admin', 'secret', 'individual')", - ] { - handle.query(query, &[]).await.unwrap_or_else(|_| panic!("failed for {query}")); + // EXPN + assert_eq!( + handle.expn("info@example.org").await.unwrap(), + vec![ + "bill@example.org".to_string(), + "jane@example.org".to_string(), + "john@example.org".to_string() + ] + ); + assert_eq!( + handle.expn("john@example.org").await.unwrap(), + Vec::::new() + ); } } -pub async fn create_test_user(handle: &dyn Directory, login: &str, secret: &str, name: &str) { - handle - .query( - "INSERT OR IGNORE INTO accounts (name, secret, description, type, active) VALUES (?, ?, ?, 'individual', true)", - &[login.into(), secret.into(), name.into()], - ) - .await - .unwrap(); -} +impl DirectoryStore { + pub async fn create_test_directory(&self) { + // Create tables + for table in ["accounts", "group_members", "emails"] { + self.store + .query::(&format!("DROP TABLE IF EXISTS {table}"), vec![]) + .await + .unwrap(); + } + for query in [ + concat!( + "CREATE TABLE accounts (name TEXT PRIMARY KEY, secret TEXT, description TEXT,", + " type TEXT NOT NULL, quota INTEGER ", + "DEFAULT 0, active BOOLEAN DEFAULT TRUE)" + ), + concat!( + "CREATE TABLE group_members (name TEXT NOT NULL, member_of ", + "TEXT NOT NULL, PRIMARY KEY (name, member_of))" + ), + concat!( + "CREATE TABLE emails (name TEXT NOT NULL, address TEXT NOT", + " NULL, type TEXT, PRIMARY KEY (name, address))" + ), + "INSERT INTO accounts (name, secret, type) VALUES ('admin', 'secret', 'individual')", + ] { + let query = if matches!(self.store, LookupStore::Store(Store::MySQL(_))) { + query.replace("TEXT", "VARCHAR(255)") + } else { + query.to_string() + }; -pub async fn create_test_user_with_email( - handle: &dyn Directory, - login: &str, - secret: &str, - name: &str, -) { - create_test_user(handle, login, secret, name).await; - link_test_address(handle, login, login, "primary").await; -} + self.store + .query::(&query, vec![]) + .await + .unwrap_or_else(|_| panic!("failed for {query}")); + } + } -pub async fn create_test_group(handle: &dyn Directory, login: &str, name: &str) { - handle - .query( - "INSERT OR IGNORE INTO accounts (name, description, type, active) VALUES (?, ?, 'group', true)", - &[login.into(), name.into()], - ) - .await - .unwrap(); -} + pub async fn create_test_user(&self, login: &str, secret: &str, name: &str) { + self.store + .query::( + if matches!(self.store, LookupStore::Store(Store::PostgreSQL(_))) { + concat!( + "INSERT INTO accounts (name, secret, description, ", + "type, active) VALUES ($1, $2, $3, 'individual', true) ON CONFLICT (name) DO NOTHING" + ) + } else if matches!(self.store, LookupStore::Store(Store::MySQL(_))) { + concat!( + "INSERT IGNORE INTO accounts (name, secret, description, ", + "type, active) VALUES (?, ?, ?, 'individual', true)" + ) + } else { + concat!( + "INSERT OR IGNORE INTO accounts (name, secret, description, ", + "type, active) VALUES (?, ?, ?, 'individual', true)" + ) + }, + vec![login.into(), secret.into(), name.into()], + ) + .await + .unwrap(); + } -pub async fn create_test_group_with_email(handle: &dyn Directory, login: &str, name: &str) { - create_test_group(handle, login, name).await; - link_test_address(handle, login, login, "primary").await; -} + pub async fn create_test_user_with_email(&self, login: &str, secret: &str, name: &str) { + self.create_test_user(login, secret, name).await; + self.link_test_address(login, login, "primary").await; + } -pub async fn link_test_address(handle: &dyn Directory, login: &str, address: &str, typ: &str) { - handle - .query( - "INSERT OR IGNORE INTO emails (name, address, type) VALUES (?, ?, ?)", - &[login.into(), address.into(), typ.into()], - ) - .await - .unwrap(); -} + pub async fn create_test_group(&self, login: &str, name: &str) { + self.store + .query::( + if matches!(self.store, LookupStore::Store(Store::PostgreSQL(_))) { + concat!( + "INSERT INTO accounts (name, description, ", + "type, active) VALUES ($1, $2, $3, $4) ON CONFLICT (name) DO NOTHING" + ) + } else if matches!(self.store, LookupStore::Store(Store::MySQL(_))) { + concat!( + "INSERT IGNORE INTO accounts (name, description, ", + "type, active) VALUES (?, ?, ?, ?)" + ) + } else { + concat!( + "INSERT OR IGNORE INTO accounts (name, description, ", + "type, active) VALUES (?, ?, ?, ?)" + ) + }, + vec![login.into(), name.into(), "group".into(), true.into()], + ) + .await + .unwrap(); + } -pub async fn set_test_quota(handle: &dyn Directory, login: &str, quota: u32) { - handle - .query( - &format!("UPDATE accounts SET quota = {} where name = ?", quota,), - &[login.into()], - ) - .await - .unwrap(); -} + pub async fn create_test_group_with_email(&self, login: &str, name: &str) { + self.create_test_group(login, name).await; + self.link_test_address(login, login, "primary").await; + } -pub async fn add_to_group(handle: &dyn Directory, login: &str, group: &str) { - handle - .query( - "INSERT INTO group_members (name, member_of) VALUES (?, ?)", - &[login.into(), group.into()], - ) - .await - .unwrap(); -} + pub async fn link_test_address(&self, login: &str, address: &str, typ: &str) { + self.store + .query::( + if matches!(self.store, LookupStore::Store(Store::PostgreSQL(_))) { + "INSERT INTO emails (name, address, type) VALUES ($1, $2, $3) ON CONFLICT (name, address) DO NOTHING" + } else if matches!(self.store, LookupStore::Store(Store::MySQL(_))) { + "INSERT IGNORE INTO emails (name, address, type) VALUES (?, ?, ?)" + } else { + "INSERT OR IGNORE INTO emails (name, address, type) VALUES (?, ?, ?)" + }, + vec![login.into(), address.into(), typ.into()], + ) + .await + .unwrap(); + } -pub async fn remove_from_group(handle: &dyn Directory, login: &str, group: &str) { - handle - .query( - "DELETE FROM group_members WHERE name = ? AND member_of = ?", - &[login.into(), group.into()], - ) - .await - .unwrap(); -} + pub async fn set_test_quota(&self, login: &str, quota: u32) { + self.store + .query::( + if matches!(self.store, LookupStore::Store(Store::PostgreSQL(_))) { + "UPDATE accounts SET quota = $1 where name = $2" + } else { + "UPDATE accounts SET quota = ? where name = ?" + }, + vec![quota.into(), login.into()], + ) + .await + .unwrap(); + } -pub async fn remove_test_alias(handle: &dyn Directory, login: &str, alias: &str) { - handle - .query( - "DELETE FROM emails WHERE name = ? AND address = ?", - &[login.into(), alias.into()], - ) - .await - .unwrap(); + pub async fn add_to_group(&self, login: &str, group: &str) { + self.store + .query::( + if matches!(self.store, LookupStore::Store(Store::PostgreSQL(_))) { + "INSERT INTO group_members (name, member_of) VALUES ($1, $2)" + } else { + "INSERT INTO group_members (name, member_of) VALUES (?, ?)" + }, + vec![login.into(), group.into()], + ) + .await + .unwrap(); + } + + pub async fn remove_from_group(&self, login: &str, group: &str) { + self.store + .query::( + if matches!(self.store, LookupStore::Store(Store::PostgreSQL(_))) { + "DELETE FROM group_members WHERE name = $1 AND member_of = $2" + } else { + "DELETE FROM group_members WHERE name = ? AND member_of = ?" + }, + vec![login.into(), group.into()], + ) + .await + .unwrap(); + } + + pub async fn remove_test_alias(&self, login: &str, alias: &str) { + self.store + .query::( + if matches!(self.store, LookupStore::Store(Store::PostgreSQL(_))) { + "DELETE FROM emails WHERE name = $1 AND address = $2" + } else { + "DELETE FROM emails WHERE name = ? AND address = ?" + }, + vec![login.into(), alias.into()], + ) + .await + .unwrap(); + } } diff --git a/tests/src/imap/mod.rs b/tests/src/imap/mod.rs index 8b0b05be..e89b9727 100644 --- a/tests/src/imap/mod.rs +++ b/tests/src/imap/mod.rs @@ -38,6 +38,7 @@ pub mod thread; use std::{path::PathBuf, sync::Arc, time::Duration}; use ::managesieve::core::ManageSieveSessionManager; +use ::store::config::ConfigStore; use ahash::AHashSet; use directory::config::ConfigDirectory; use imap::core::{ImapSessionManager, IMAP}; @@ -51,14 +52,7 @@ use tokio::{ }; use utils::{config::ServerProtocol, UnwrapFailure}; -use crate::{ - add_test_certs, - directory::sql::{ - add_to_group, create_test_directory, create_test_group_with_email, create_test_user, - create_test_user_with_email, - }, - store::TempDir, -}; +use crate::{add_test_certs, directory::DirectoryStore, store::TempDir}; const SERVER: &str = r#" [server] @@ -101,7 +95,7 @@ reject-non-fqdn = false [session.rcpt] relay = [ { if = "authenticated-as", ne = "", then = true }, { else = false } ] -directory = "sql" +directory = "auth" [session.rcpt.errors] total = 5 @@ -136,35 +130,51 @@ allow-invalid-certs = true future-release = [ { if = "authenticated-as", ne = "", then = "99999999d"}, { else = false } ] -[store.db] +[store."sqlite"] +type = "sqlite" path = "{TMP}/sqlite.db" + +[store."rocksdb"] +type = "rocksdb" +path = "{TMP}/rocks.db" + +[store."foundationdb"] +type = "foundationdb" + +[store."postgresql"] +type = "postgresql" +host = "localhost" +port = 5432 +database = "stalwart" +user = "postgres" +password = "mysecretpassword" + +[store."mysql"] +type = "mysql" host = "localhost" -#port = 5432 port = 3307 database = "stalwart" -#user = "postgres" -#password = "mysecretpassword" user = "root" password = "password" -[store.fts] +[store."elastic"] +type = "elasticsearch" url = "https://localhost:9200" user = "elastic" password = "RtQ-Lu6+o4rxx=XJplVJ" allow-invalid-certs = true -[store.blob] -type = "local" - -[store.blob.local] -path = "{TMP}" - [certificate.default] cert = "file://{CERT}" private-key = "file://{PK}" [jmap] -directory = "sql" +directory = "auth" + +[jmap.store] +data = "sqlite" +fts = "sqlite" +blob = "sqlite" [jmap.protocol] set.max-objects = 100000 @@ -196,14 +206,11 @@ throttle = "500ms" throttle = "500ms" attempts.interval = "500ms" -[directory."sql"] -type = "sql" -address = "sqlite::memory:" +[store."auth"] +type = "sqlite" +path = "{TMP}/auth.db" -[directory."sql".pool] -max-connections = 1 - -[directory."sql".query] +[store."auth".query] name = "SELECT name, type, secret, description, quota FROM accounts WHERE name = ? AND active = true" members = "SELECT member_of FROM group_members WHERE name = ?" recipients = "SELECT name FROM emails WHERE address = ?" @@ -212,7 +219,11 @@ verify = "SELECT address FROM emails WHERE address LIKE '%' || ? || '%' AND type expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = ? AND l.type = 'list' ORDER BY p.address LIMIT 50" domains = "SELECT 1 FROM emails WHERE address LIKE '%@' || ? LIMIT 1" -[directory."sql".columns] +[directory."auth"] +type = "sql" +store = "auth" + +[directory."auth".columns] name = "name" description = "description" secret = "secret" @@ -220,12 +231,16 @@ email = "address" quota = "quota" type = "type" -[directory."local"] +[store."local"] type = "memory" -[directory."local".lookup] -domains = ["example.com"] -remote-domains = ["remote.org", "foobar.com", "test.com", "other_domain.com"] +[store."local".lookup."domains"] +type = "list" +values = ["example.com"] + +[store."local".lookup."remote-domains"] +type = "list" +values = ["remote.org", "foobar.com", "test.com", "other_domain.com"] [oauth] key = "parerga_und_paralipomena" @@ -255,15 +270,16 @@ async fn init_imap_tests(delete_if_exists: bool) -> IMAPTest { ) .unwrap(); let servers = config.parse_servers().unwrap(); - let directory = config.parse_directory().unwrap(); + let stores = config.parse_stores().await.failed("Invalid configuration"); + let directory = config.parse_directory(&stores).unwrap(); // Start JMAP and SMTP servers servers.bind(&config); let (delivery_tx, delivery_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); - let smtp = SMTP::init(&config, &servers, &directory, delivery_tx) + let smtp = SMTP::init(&config, &servers, &stores, &directory, delivery_tx) .await .failed("Invalid configuration file"); - let jmap = JMAP::init(&config, &directory, delivery_rx, smtp.clone()) + let jmap = JMAP::init(&config, &stores, &directory, delivery_rx, smtp.clone()) .await .failed("Invalid configuration file"); let imap: Arc = IMAP::init(&config) @@ -290,42 +306,29 @@ async fn init_imap_tests(delete_if_exists: bool) -> IMAPTest { }); // Create tables and test accounts - create_test_directory(jmap.directory.as_ref()).await; - create_test_user(jmap.directory.as_ref(), "admin", "secret", "Superuser").await; - add_to_group(jmap.directory.as_ref(), "admin", "superuser").await; - create_test_user_with_email( - jmap.directory.as_ref(), - "jdoe@example.com", - "secret", - "John Doe", - ) - .await; - create_test_user_with_email( - jmap.directory.as_ref(), - "jane.smith@example.com", - "secret", - "Jane Smith", - ) - .await; - create_test_user_with_email( - jmap.directory.as_ref(), - "foobar@example.com", - "secret", - "Bill Foobar", - ) - .await; - create_test_group_with_email( - jmap.directory.as_ref(), - "support@example.com", - "Support Group", - ) - .await; - add_to_group( - jmap.directory.as_ref(), - "jane.smith@example.com", - "support@example.com", - ) - .await; + let lookup = DirectoryStore { + store: stores.lookup_stores.get("auth").unwrap().clone(), + }; + lookup.create_test_directory().await; + lookup + .create_test_user("admin", "secret", "Superuser") + .await; + lookup.add_to_group("admin", "superuser").await; + lookup + .create_test_user_with_email("jdoe@example.com", "secret", "John Doe") + .await; + lookup + .create_test_user_with_email("jane.smith@example.com", "secret", "Jane Smith") + .await; + lookup + .create_test_user_with_email("foobar@example.com", "secret", "Bill Foobar") + .await; + lookup + .create_test_group_with_email("support@example.com", "Support Group") + .await; + lookup + .add_to_group("jane.smith@example.com", "support@example.com") + .await; if delete_if_exists { jmap.store.destroy().await; diff --git a/tests/src/jmap/auth_acl.rs b/tests/src/jmap/auth_acl.rs index 1acbc2b6..216ba95d 100644 --- a/tests/src/jmap/auth_acl.rs +++ b/tests/src/jmap/auth_acl.rs @@ -21,14 +21,8 @@ * for more details. */ -use std::sync::Arc; - -use jmap::{ - mailbox::{INBOX_ID, TRASH_ID}, - JMAP, -}; +use jmap::mailbox::{INBOX_ID, TRASH_ID}; use jmap_client::{ - client::Client, core::{ error::{MethodError, MethodErrorType}, set::{SetError, SetErrorType}, @@ -41,25 +35,34 @@ use jmap_proto::types::id::Id; use std::fmt::Debug; use store::ahash::AHashMap; -use crate::{ - directory::sql::{ - add_to_group, create_test_group_with_email, create_test_user_with_email, remove_from_group, - }, - jmap::{assert_is_empty, mailbox::destroy_all_mailboxes, test_account_login}, -}; +use crate::jmap::{assert_is_empty, mailbox::destroy_all_mailboxes, test_account_login}; -pub async fn test(server: Arc, admin_client: &mut Client) { +use super::JMAPTest; + +pub async fn test(params: &mut JMAPTest) { println!("Running ACL tests..."); + let server = params.server.clone(); // Create a group and three test accounts let inbox_id = Id::new(INBOX_ID as u64).to_string(); let trash_id = Id::new(TRASH_ID as u64).to_string(); - let directory = server.directory.as_ref(); - create_test_user_with_email(directory, "jdoe@example.com", "12345", "John Doe").await; - create_test_user_with_email(directory, "jane.smith@example.com", "abcde", "Jane Smith").await; - create_test_user_with_email(directory, "bill@example.com", "098765", "Bill Foobar").await; - create_test_group_with_email(directory, "sales@example.com", "Sales Group").await; + params + .directory + .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") + .await; + params + .directory + .create_test_user_with_email("jane.smith@example.com", "abcde", "Jane Smith") + .await; + params + .directory + .create_test_user_with_email("bill@example.com", "098765", "Bill Foobar") + .await; + params + .directory + .create_test_group_with_email("sales@example.com", "Sales Group") + .await; let john_id: Id = server .get_account_id("jdoe@example.com") .await @@ -92,7 +95,7 @@ pub async fn test(server: Arc, admin_client: &mut Client) { (&mut john_client, &john_id, "john"), (&mut jane_client, &jane_id, "jane"), (&mut bill_client, &bill_id, "bill"), - (admin_client, &sales_id, "sales"), + (&mut params.client, &sales_id, "sales"), ] { let user_name = client.session().username().to_string(); let mut ids = Vec::with_capacity(2); @@ -667,7 +670,10 @@ pub async fn test(server: Arc, admin_client: &mut Client) { // Add John and Jane to the Sales group for name in ["jdoe@example.com", "jane.smith@example.com"] { - add_to_group(directory, name, "sales@example.com").await; + params + .directory + .add_to_group(name, "sales@example.com") + .await; } server.access_tokens.clear(); john_client.refresh_session().await.unwrap(); @@ -763,7 +769,10 @@ pub async fn test(server: Arc, admin_client: &mut Client) { ); // Remove John from the sales group - remove_from_group(directory, "jdoe@example.com", "sales@example.com").await; + params + .directory + .remove_from_group("jdoe@example.com", "sales@example.com") + .await; server.sessions.clear(); assert_forbidden( john_client @@ -774,8 +783,8 @@ pub async fn test(server: Arc, admin_client: &mut Client) { // Destroy test account data for id in [john_id, bill_id, jane_id, sales_id] { - admin_client.set_default_account_id(&id.to_string()); - destroy_all_mailboxes(admin_client).await; + params.client.set_default_account_id(&id.to_string()); + destroy_all_mailboxes(¶ms.client).await; } assert_is_empty(server).await; } diff --git a/tests/src/jmap/auth_limits.rs b/tests/src/jmap/auth_limits.rs index 0d1ca20f..1677a4b4 100644 --- a/tests/src/jmap/auth_limits.rs +++ b/tests/src/jmap/auth_limits.rs @@ -23,7 +23,6 @@ use std::{sync::Arc, time::Duration}; -use jmap::JMAP; use jmap_client::{ client::{Client, Credentials}, core::set::{SetError, SetErrorType}, @@ -31,25 +30,24 @@ use jmap_client::{ }; use jmap_proto::types::id::Id; -use crate::{ - directory::sql::{create_test_user_with_email, link_test_address}, - jmap::{assert_is_empty, mailbox::destroy_all_mailboxes}, -}; +use crate::jmap::{assert_is_empty, mailbox::destroy_all_mailboxes}; -pub async fn test(server: Arc, admin_client: &mut Client) { +use super::JMAPTest; + +pub async fn test(params: &mut JMAPTest) { println!("Running Authorization tests..."); // Create test account - let directory = server.directory.as_ref(); - create_test_user_with_email(directory, "jdoe@example.com", "12345", "John Doe").await; + let server = params.server.clone(); + params + .directory + .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") + .await; let account_id = Id::from(server.get_account_id("jdoe@example.com").await.unwrap()).to_string(); - link_test_address( - directory, - "jdoe@example.com", - "john.doe@example.com", - "alias", - ) - .await; + params + .directory + .link_test_address("jdoe@example.com", "john.doe@example.com", "alias") + .await; // Reset rate limiters server.rate_limit_auth.clear(); @@ -200,7 +198,7 @@ pub async fn test(server: Arc, admin_client: &mut Client) { Err(jmap_client::Error::Problem(err)) if err.status() == Some(400))); // Destroy test accounts - admin_client.set_default_account_id(&account_id); - destroy_all_mailboxes(admin_client).await; + params.client.set_default_account_id(&account_id); + destroy_all_mailboxes(¶ms.client).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/auth_oauth.rs b/tests/src/jmap/auth_oauth.rs index ea36702d..beed2597 100644 --- a/tests/src/jmap/auth_oauth.rs +++ b/tests/src/jmap/auth_oauth.rs @@ -21,16 +21,10 @@ * for more details. */ -use std::{ - sync::Arc, - time::{Duration, Instant}, -}; +use std::time::{Duration, Instant}; use bytes::Bytes; -use jmap::{ - auth::oauth::{DeviceAuthResponse, ErrorType, OAuthMetadata, TokenResponse}, - JMAP, -}; +use jmap::auth::oauth::{DeviceAuthResponse, ErrorType, OAuthMetadata, TokenResponse}; use jmap_client::{ client::{Client, Credentials}, mailbox::query::Filter, @@ -40,17 +34,19 @@ use reqwest::{header, redirect::Policy}; use serde::de::DeserializeOwned; use store::ahash::AHashMap; -use crate::{ - directory::sql::create_test_user_with_email, - jmap::{assert_is_empty, mailbox::destroy_all_mailboxes}, -}; +use crate::jmap::{assert_is_empty, mailbox::destroy_all_mailboxes}; -pub async fn test(server: Arc, admin_client: &mut Client) { +use super::JMAPTest; + +pub async fn test(params: &mut JMAPTest) { println!("Running OAuth tests..."); // Create test account - let directory = server.directory.as_ref(); - create_test_user_with_email(directory, "jdoe@example.com", "12345", "John Doe").await; + let server = params.server.clone(); + params + .directory + .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") + .await; let john_id = Id::from(server.get_account_id("jdoe@example.com").await.unwrap()).to_string(); // Obtain OAuth metadata @@ -308,8 +304,8 @@ pub async fn test(server: Arc, admin_client: &mut Client) { ); // Destroy test accounts - admin_client.set_default_account_id(john_id); - destroy_all_mailboxes(admin_client).await; + params.client.set_default_account_id(john_id); + destroy_all_mailboxes(¶ms.client).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/blob.rs b/tests/src/jmap/blob.rs index a40bb937..0fa691b7 100644 --- a/tests/src/jmap/blob.rs +++ b/tests/src/jmap/blob.rs @@ -21,22 +21,21 @@ * for more details. */ -use std::sync::Arc; - -use jmap::{mailbox::INBOX_ID, JMAP}; -use jmap_client::client::Client; +use jmap::mailbox::INBOX_ID; use jmap_proto::types::id::Id; use serde_json::Value; -use crate::{ - directory::sql::create_test_user_with_email, - jmap::{assert_is_empty, jmap_json_request, mailbox::destroy_all_mailboxes}, -}; +use crate::jmap::{assert_is_empty, jmap_json_request, mailbox::destroy_all_mailboxes}; -pub async fn test(server: Arc, admin_client: &mut Client) { +use super::JMAPTest; + +pub async fn test(params: &mut JMAPTest) { println!("Running blob tests..."); - let directory = server.directory.as_ref(); - create_test_user_with_email(directory, "jdoe@example.com", "12345", "John Doe").await; + let server = params.server.clone(); + params + .directory + .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") + .await; let account_id = Id::from(server.get_account_id("jdoe@example.com").await.unwrap()); server.store.blob_hash_expire_all().await; @@ -425,8 +424,9 @@ pub async fn test(server: Arc, admin_client: &mut Client) { server.store.blob_hash_expire_all().await; // Blob/lookup - admin_client.set_default_account_id(account_id.to_string()); - let blob_id = admin_client + params.client.set_default_account_id(account_id.to_string()); + let blob_id = params + .client .email_import( concat!( "From: bill@example.com\r\n", @@ -487,7 +487,7 @@ pub async fn test(server: Arc, admin_client: &mut Client) { } // Remove test data - admin_client.set_default_account_id(account_id.to_string()); - destroy_all_mailboxes(admin_client).await; + params.client.set_default_account_id(account_id.to_string()); + destroy_all_mailboxes(¶ms.client).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/crypto.rs b/tests/src/jmap/crypto.rs index 8fd81c85..2c29f844 100644 --- a/tests/src/jmap/crypto.rs +++ b/tests/src/jmap/crypto.rs @@ -21,32 +21,29 @@ * for more details. */ -use std::{path::PathBuf, sync::Arc, time::Duration}; +use std::{path::PathBuf, time::Duration}; use ahash::AHashMap; -use jmap::{ - email::crypto::{ - try_parse_certs, Algorithm, EncryptMessage, EncryptionMethod, EncryptionParams, - }, - JMAP, +use jmap::email::crypto::{ + try_parse_certs, Algorithm, EncryptMessage, EncryptionMethod, EncryptionParams, }; -use jmap_client::client::Client; use jmap_proto::types::id::Id; use mail_parser::{MessageParser, MimeHeaders}; -use crate::{directory::sql::create_test_user_with_email, jmap::delivery::SmtpConnection}; +use crate::jmap::delivery::SmtpConnection; -pub async fn test(server: Arc, client: &mut Client) { +use super::JMAPTest; + +pub async fn test(params: &mut JMAPTest) { println!("Running Encryption-at-rest tests..."); // Create test account - create_test_user_with_email( - server.directory.as_ref(), - "jdoe@example.com", - "12345", - "John Doe", - ) - .await; + let server = params.server.clone(); + let client = &mut params.client; + params + .directory + .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") + .await; let account_id = Id::from(server.get_account_id("jdoe@example.com").await.unwrap()).to_string(); // Update diff --git a/tests/src/jmap/delivery.rs b/tests/src/jmap/delivery.rs index 33e0873d..e2cb88a7 100644 --- a/tests/src/jmap/delivery.rs +++ b/tests/src/jmap/delivery.rs @@ -21,10 +21,8 @@ * for more details. */ -use std::{sync::Arc, time::Duration}; +use std::time::Duration; -use jmap::JMAP; -use jmap_client::client::Client; use jmap_proto::types::{collection::Collection, id::Id}; use tokio::{ @@ -32,37 +30,51 @@ use tokio::{ net::TcpStream, }; -use crate::{ - directory::sql::{create_test_user_with_email, link_test_address, remove_test_alias}, - jmap::{assert_is_empty, mailbox::destroy_all_mailboxes}, -}; +use crate::jmap::{assert_is_empty, mailbox::destroy_all_mailboxes}; -pub async fn test(server: Arc, client: &mut Client) { +use super::JMAPTest; + +pub async fn test(params: &mut JMAPTest) { println!("Running message delivery tests..."); // Create a domain name and a test account - let directory = server.directory.as_ref(); - create_test_user_with_email(directory, "jdoe@example.com", "12345", "John Doe").await; - create_test_user_with_email(directory, "jane@example.com", "abcdef", "Jane Smith").await; - create_test_user_with_email(directory, "bill@example.com", "098765", "Bill Foobar").await; + let server = params.server.clone(); + params + .directory + .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") + .await; + params + .directory + .create_test_user_with_email("jane@example.com", "abcdef", "Jane Smith") + .await; + params + .directory + .create_test_user_with_email("bill@example.com", "098765", "Bill Foobar") + .await; let account_id_1 = Id::from(server.get_account_id("jdoe@example.com").await.unwrap()).to_string(); let account_id_2 = Id::from(server.get_account_id("jane@example.com").await.unwrap()).to_string(); let account_id_3 = Id::from(server.get_account_id("bill@example.com").await.unwrap()).to_string(); - link_test_address( - directory, - "jdoe@example.com", - "john.doe@example.com", - "alias", - ) - .await; + params + .directory + .link_test_address("jdoe@example.com", "john.doe@example.com", "alias") + .await; // Create a mailing list - link_test_address(directory, "jdoe@example.com", "members@example.com", "list").await; - link_test_address(directory, "jane@example.com", "members@example.com", "list").await; - link_test_address(directory, "bill@example.com", "members@example.com", "list").await; + params + .directory + .link_test_address("jdoe@example.com", "members@example.com", "list") + .await; + params + .directory + .link_test_address("jane@example.com", "members@example.com", "list") + .await; + params + .directory + .link_test_address("bill@example.com", "members@example.com", "list") + .await; // Delivering to individuals let mut lmtp = SmtpConnection::connect().await; @@ -172,7 +184,10 @@ pub async fn test(server: Arc, client: &mut Client) { } // Removing members from the mailing list and chunked ingest - remove_test_alias(directory, "jdoe@example.com", "members@example.com").await; + params + .directory + .remove_test_alias("jdoe@example.com", "members@example.com") + .await; lmtp.ingest_chunked( "bill@example.com", &["members@example.com"], @@ -245,8 +260,8 @@ pub async fn test(server: Arc, client: &mut Client) { // Remove test data for account_id in [&account_id_1, &account_id_2, &account_id_3] { - client.set_default_account_id(account_id); - destroy_all_mailboxes(client).await; + params.client.set_default_account_id(account_id); + destroy_all_mailboxes(¶ms.client).await; } assert_is_empty(server).await; } diff --git a/tests/src/jmap/email_changes.rs b/tests/src/jmap/email_changes.rs index 1fd6d943..28384856 100644 --- a/tests/src/jmap/email_changes.rs +++ b/tests/src/jmap/email_changes.rs @@ -21,10 +21,6 @@ * for more details. */ -use std::sync::Arc; - -use jmap::JMAP; -use jmap_client::client::Client; use jmap_proto::{ parser::{json::Parser, JsonObjectParser}, types::{collection::Collection, id::Id, state::State}, @@ -36,10 +32,13 @@ use store::{ use crate::jmap::assert_is_empty; -pub async fn test(server: Arc, client: &mut Client) { +use super::JMAPTest; + +pub async fn test(params: &mut JMAPTest) { println!("Running Email Changes tests..."); - client.set_default_account_id(Id::new(1)); + let server = params.server.clone(); + params.client.set_default_account_id(Id::new(1)); let mut states = vec![State::Initial]; for (change_id, (changes, expected_changelog)) in [ @@ -183,7 +182,11 @@ pub async fn test(server: Arc, client: &mut Client) { let mut new_state = State::Initial; for (test_num, state) in (states).iter().enumerate() { - let changes = client.email_changes(state.to_string(), None).await.unwrap(); + let changes = params + .client + .email_changes(state.to_string(), None) + .await + .unwrap(); assert_eq!( expected_changelog[test_num], @@ -224,7 +227,8 @@ pub async fn test(server: Arc, client: &mut Client) { let mut int_state = state.clone(); for _ in 0..100 { - let changes = client + let changes = params + .client .email_changes(int_state.to_string(), max_changes.into()) .await .unwrap(); @@ -303,7 +307,8 @@ pub async fn test(server: Arc, client: &mut Client) { states.push(new_state); } - let changes = client + let changes = params + .client .email_changes(State::Initial.to_string(), 0.into()) .await .unwrap(); diff --git a/tests/src/jmap/email_copy.rs b/tests/src/jmap/email_copy.rs index 6eb3beaa..e8c6e5ff 100644 --- a/tests/src/jmap/email_copy.rs +++ b/tests/src/jmap/email_copy.rs @@ -21,19 +21,20 @@ * for more details. */ -use std::sync::Arc; - -use jmap::JMAP; -use jmap_client::{client::Client, mailbox::Role}; +use jmap_client::mailbox::Role; use jmap_proto::types::id::Id; use crate::jmap::{assert_is_empty, mailbox::destroy_all_mailboxes}; -pub async fn test(server: Arc, client: &mut Client) { +use super::JMAPTest; + +pub async fn test(params: &mut JMAPTest) { println!("Running Email Copy tests..."); + let server = params.server.clone(); // Create a mailbox on account 1 - let ac1_mailbox_id = client + let ac1_mailbox_id = params + .client .set_default_account_id(Id::new(1).to_string()) .mailbox_create("Copy Test Ac# 1", None::, Role::None) .await @@ -41,7 +42,8 @@ pub async fn test(server: Arc, client: &mut Client) { .take_id(); // Insert a message on account 1 - let ac1_email_id = client + let ac1_email_id = params + .client .email_import( concat!( "From: bill@example.com\r\n", @@ -62,7 +64,8 @@ pub async fn test(server: Arc, client: &mut Client) { .take_id(); // Create a mailbox on account 2 - let ac2_mailbox_id = client + let ac2_mailbox_id = params + .client .set_default_account_id(Id::new(2).to_string()) .mailbox_create("Copy Test Ac# 2", None::, Role::None) .await @@ -70,7 +73,7 @@ pub async fn test(server: Arc, client: &mut Client) { .take_id(); // Copy the email and delete it from the first account - let mut request = client.build(); + let mut request = params.client.build(); request .copy_email(Id::new(1).to_string()) .on_success_destroy_original(true) @@ -90,7 +93,8 @@ pub async fn test(server: Arc, client: &mut Client) { .take_id(); // Check that the email was copied - let email = client + let email = params + .client .email_get(&ac2_email_id, None::>) .await .unwrap() @@ -105,7 +109,8 @@ pub async fn test(server: Arc, client: &mut Client) { assert_eq!(email.received_at().unwrap(), 311923920); // Check that the email was deleted - assert!(client + assert!(params + .client .set_default_account_id(Id::new(1).to_string()) .email_get(&ac1_email_id, None::>) .await @@ -113,8 +118,8 @@ pub async fn test(server: Arc, client: &mut Client) { .is_none()); // Empty store - destroy_all_mailboxes(client).await; - client.set_default_account_id(Id::new(2).to_string()); - destroy_all_mailboxes(client).await; + destroy_all_mailboxes(¶ms.client).await; + params.client.set_default_account_id(Id::new(2).to_string()); + destroy_all_mailboxes(¶ms.client).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/email_get.rs b/tests/src/jmap/email_get.rs index db4fd507..cb9aaf38 100644 --- a/tests/src/jmap/email_get.rs +++ b/tests/src/jmap/email_get.rs @@ -21,20 +21,20 @@ * for more details. */ -use std::{fs, path::PathBuf, sync::Arc}; +use std::{fs, path::PathBuf}; -use jmap::{mailbox::INBOX_ID, JMAP}; -use jmap_client::{ - client::Client, - email::{self, import::EmailImportResponse, Header, HeaderForm}, -}; +use jmap::mailbox::INBOX_ID; +use jmap_client::email::{self, import::EmailImportResponse, Header, HeaderForm}; use jmap_proto::types::id::Id; use mail_parser::HeaderName; use crate::jmap::{assert_is_empty, mailbox::destroy_all_mailboxes, replace_blob_ids}; -pub async fn test(server: Arc, client: &mut Client) { +use super::JMAPTest; + +pub async fn test(params: &mut JMAPTest) { println!("Running Email Get tests..."); + let server = params.server.clone(); let mut test_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); test_dir.push("resources"); @@ -42,7 +42,7 @@ pub async fn test(server: Arc, client: &mut Client) { test_dir.push("email_get"); let mailbox_id = Id::from(INBOX_ID).to_string(); - client.set_default_account_id(Id::from(1u64)); + params.client.set_default_account_id(Id::from(1u64)); for file_name in fs::read_dir(&test_dir).unwrap() { let mut file_name = file_name.as_ref().unwrap().path(); @@ -55,12 +55,13 @@ pub async fn test(server: Arc, client: &mut Client) { let blob_len = blob.len(); // Import email - let mut request = client.build(); + let mut request = params.client.build(); let import_request = request .import_email() .account_id(Id::from(1u64).to_string()) .email( - client + params + .client .upload(None, blob, None) .await .unwrap() @@ -74,7 +75,7 @@ pub async fn test(server: Arc, client: &mut Client) { assert_ne!(response.old_state(), Some(response.new_state())); let email = response.created(&id).unwrap(); - let mut request = client.build(); + let mut request = params.client.build(); request .get_email() .ids([email.id().unwrap()]) @@ -157,7 +158,7 @@ pub async fn test(server: Arc, client: &mut Client) { if is_headers_test { for property in all_headers() { - let mut request = client.build(); + let mut request = params.client.build(); request .get_email() .ids([email.id().unwrap()]) @@ -187,7 +188,7 @@ pub async fn test(server: Arc, client: &mut Client) { } } - destroy_all_mailboxes(client).await; + destroy_all_mailboxes(¶ms.client).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/email_parse.rs b/tests/src/jmap/email_parse.rs index 4d85e15b..93b4d789 100644 --- a/tests/src/jmap/email_parse.rs +++ b/tests/src/jmap/email_parse.rs @@ -21,11 +21,9 @@ * for more details. */ -use std::{fs, path::PathBuf, sync::Arc}; +use std::{fs, path::PathBuf}; -use jmap::JMAP; use jmap_client::{ - client::Client, email::{self, Header, HeaderForm}, mailbox::Role, }; @@ -35,15 +33,19 @@ use crate::jmap::{ assert_is_empty, email_get::all_headers, mailbox::destroy_all_mailboxes, replace_blob_ids, }; -pub async fn test(server: Arc, client: &mut Client) { +use super::JMAPTest; + +pub async fn test(params: &mut JMAPTest) { println!("Running Email Parse tests..."); + let server = params.server.clone(); let mut test_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); test_dir.push("resources"); test_dir.push("jmap"); test_dir.push("email_parse"); - let mailbox_id = client + let mailbox_id = params + .client .set_default_account_id(Id::new(1).to_string()) .mailbox_create("JMAP Parse", None::, Role::None) .await @@ -55,7 +57,8 @@ pub async fn test(server: Arc, client: &mut Client) { let mut test_file = test_dir.clone(); test_file.push(test_name); - let email = client + let email = params + .client .email_import( fs::read(&test_file).unwrap(), [mailbox_id.clone()], @@ -65,7 +68,8 @@ pub async fn test(server: Arc, client: &mut Client) { .await .unwrap(); - let blob_id = client + let blob_id = params + .client .email_get(email.id().unwrap(), Some([email::Property::Attachments])) .await .unwrap() @@ -78,7 +82,8 @@ pub async fn test(server: Arc, client: &mut Client) { .unwrap() .to_string(); - let email = client + let email = params + .client .email_parse( &blob_id, [ @@ -137,7 +142,7 @@ pub async fn test(server: Arc, client: &mut Client) { for part in parts { let blob_id = part.blob_id().unwrap(); - let inner_blob = client.download(blob_id).await.unwrap(); + let inner_blob = params.client.download(blob_id).await.unwrap(); test_file.set_extension(format!("part{}", part.part_id().unwrap())); @@ -168,13 +173,15 @@ pub async fn test(server: Arc, client: &mut Client) { // Test header parsing on a temporary blob let mut test_file = test_dir; test_file.push("headers.eml"); - let blob_id = client + let blob_id = params + .client .upload(None, fs::read(&test_file).unwrap(), None) .await .unwrap() .take_blob_id(); - let mut email = client + let mut email = params + .client .email_parse( &blob_id, [ @@ -225,7 +232,8 @@ pub async fn test(server: Arc, client: &mut Client) { for property in all_headers() { email.headers.extend( - client + params + .client .email_parse(&blob_id, [property].into(), [].into(), None) .await .unwrap() @@ -244,6 +252,6 @@ pub async fn test(server: Arc, client: &mut Client) { panic!("Test failed, output saved to {}", test_file.display()); } - destroy_all_mailboxes(client).await; + destroy_all_mailboxes(¶ms.client).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/email_query.rs b/tests/src/jmap/email_query.rs index e69e76f5..6240ca7b 100644 --- a/tests/src/jmap/email_query.rs +++ b/tests/src/jmap/email_query.rs @@ -21,13 +21,12 @@ * for more details. */ -use std::{collections::hash_map::Entry, sync::Arc, time::Instant}; +use std::{collections::hash_map::Entry, time::Instant}; use crate::{ jmap::{assert_is_empty, mailbox::destroy_all_mailboxes, wait_for_index}, store::{deflate_artwork_data, query::FIELDS}, }; -use jmap::JMAP; use jmap_client::{ client::Client, core::query::{Comparator, Filter}, @@ -38,12 +37,16 @@ use mail_parser::HeaderName; use store::{ahash::AHashMap, write::BatchBuilder}; +use super::JMAPTest; + const MAX_THREADS: usize = 100; const MAX_MESSAGES: usize = 1000; const MAX_MESSAGES_PER_THREAD: usize = 100; -pub async fn test(server: Arc, client: &mut Client, insert: bool) { +pub async fn test(params: &mut JMAPTest, insert: bool) { println!("Running Email Query tests..."); + let server = params.server.clone(); + let client = &mut params.client; client.set_default_account_id(Id::new(1)); if insert { // Add some "virtual" mailbox ids so create doesn't fail @@ -117,7 +120,7 @@ pub async fn test(server: Arc, client: &mut Client, insert: bool) { .unwrap_set_email() .unwrap(); - destroy_all_mailboxes(client).await; + destroy_all_mailboxes(¶ms.client).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/email_query_changes.rs b/tests/src/jmap/email_query_changes.rs index 54612a00..4786d65b 100644 --- a/tests/src/jmap/email_query_changes.rs +++ b/tests/src/jmap/email_query_changes.rs @@ -21,15 +21,12 @@ * for more details. */ -use jmap::JMAP; use jmap_client::{ - client::Client, core::query::{Comparator, Filter}, email, mailbox::Role, }; use jmap_proto::types::{collection::Collection, id::Id, property::Property, state::State}; -use std::sync::Arc; use store::{ ahash::{AHashMap, AHashSet}, @@ -42,15 +39,20 @@ use crate::jmap::{ mailbox::destroy_all_mailboxes, }; -pub async fn test(server: Arc, client: &mut Client) { +use super::JMAPTest; + +pub async fn test(params: &mut JMAPTest) { println!("Running Email QueryChanges tests..."); - let mailbox1_id = client + let server = params.server.clone(); + let mailbox1_id = params + .client .set_default_account_id(Id::new(1).to_string()) .mailbox_create("JMAP Changes 1", None::, Role::None) .await .unwrap() .take_id(); - let mailbox2_id = client + let mailbox2_id = params + .client .mailbox_create("JMAP Changes 2", None::, Role::None) .await .unwrap() @@ -101,7 +103,8 @@ pub async fn test(server: Arc, client: &mut Client) { match &change { LogAction::Insert(id) => { let jmap_id = Id::from_bytes( - client + params + .client .email_import( format!( "From: test_{}\nSubject: test_{}\n\ntest", @@ -139,7 +142,7 @@ pub async fn test(server: Arc, client: &mut Client) { } LogAction::Delete(id) => { let id = *id_map.get(id).unwrap(); - client.email_destroy(&id.to_string()).await.unwrap(); + params.client.email_destroy(&id.to_string()).await.unwrap(); removed_ids.insert(id); } LogAction::Move(from, to) => { @@ -217,7 +220,7 @@ pub async fn test(server: Arc, client: &mut Client) { if test_num == 3 && query.up_to_id.is_none() { continue; } - let mut request = client.build(); + let mut request = params.client.build(); let query_request = request .query_email_changes(query.since_query_state.to_string()) .sort(query.sort); @@ -273,7 +276,7 @@ pub async fn test(server: Arc, client: &mut Client) { states.push(new_state); } - destroy_all_mailboxes(client).await; + destroy_all_mailboxes(¶ms.client).await; // Delete virtual threads let mut batch = BatchBuilder::new(); diff --git a/tests/src/jmap/email_search_snippet.rs b/tests/src/jmap/email_search_snippet.rs index e32e823a..42c09edf 100644 --- a/tests/src/jmap/email_search_snippet.rs +++ b/tests/src/jmap/email_search_snippet.rs @@ -21,19 +21,22 @@ * for more details. */ -use std::{fs, path::PathBuf, sync::Arc}; +use std::{fs, path::PathBuf}; use crate::jmap::{assert_is_empty, mailbox::destroy_all_mailboxes, wait_for_index}; -use jmap::{mailbox::INBOX_ID, JMAP}; -use jmap_client::{client::Client, core::query, email::query::Filter}; +use jmap::mailbox::INBOX_ID; +use jmap_client::{core::query, email::query::Filter}; use jmap_proto::types::id::Id; use store::ahash::AHashMap; -pub async fn test(server: Arc, client: &mut Client) { +use super::JMAPTest; + +pub async fn test(params: &mut JMAPTest) { println!("Running SearchSnippet tests..."); + let server = params.server.clone(); let mailbox_id = Id::from(INBOX_ID).to_string(); - client.set_default_account_id(Id::from(1u64)); + params.client.set_default_account_id(Id::from(1u64)); let mut email_ids = AHashMap::default(); @@ -52,7 +55,8 @@ pub async fn test(server: Arc, client: &mut Client) { ] { let mut file_name = test_dir.clone(); file_name.push(format!("{}.eml", email_name)); - let email_id = client + let email_id = params + .client .email_import( fs::read(&file_name).unwrap(), [&mailbox_id], @@ -148,7 +152,7 @@ pub async fn test(server: Arc, client: &mut Client) { )), ), ] { - let mut request = client.build(); + let mut request = params.client.build(); let result_ref = request .query_email() .filter(filter.clone()) @@ -179,6 +183,6 @@ pub async fn test(server: Arc, client: &mut Client) { } // Destroy test data - destroy_all_mailboxes(client).await; + destroy_all_mailboxes(¶ms.client).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/email_set.rs b/tests/src/jmap/email_set.rs index fceb47eb..fde4f2c8 100644 --- a/tests/src/jmap/email_set.rs +++ b/tests/src/jmap/email_set.rs @@ -21,10 +21,10 @@ * for more details. */ -use std::{fs, path::PathBuf, sync::Arc}; +use std::{fs, path::PathBuf}; use crate::jmap::{assert_is_empty, mailbox::destroy_all_mailboxes}; -use jmap::{mailbox::INBOX_ID, JMAP}; +use jmap::mailbox::INBOX_ID; use jmap_client::{ client::Client, core::set::{SetError, SetErrorType}, @@ -34,18 +34,19 @@ use jmap_client::{ }; use jmap_proto::types::id::Id; -use super::{find_values, replace_blob_ids, replace_boundaries, replace_values}; +use super::{find_values, replace_blob_ids, replace_boundaries, replace_values, JMAPTest}; -pub async fn test(server: Arc, client: &mut Client) { +pub async fn test(params: &mut JMAPTest) { println!("Running Email Set tests..."); + let server = params.server.clone(); let mailbox_id = Id::from(INBOX_ID).to_string(); - client.set_default_account_id(Id::from(1u64)); + params.client.set_default_account_id(Id::from(1u64)); - create(client, &mailbox_id).await; - update(client, &mailbox_id).await; + create(&mut params.client, &mailbox_id).await; + update(&mut params.client, &mailbox_id).await; - destroy_all_mailboxes(client).await; + destroy_all_mailboxes(¶ms.client).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/email_submission.rs b/tests/src/jmap/email_submission.rs index 8915b1e0..be96c292 100644 --- a/tests/src/jmap/email_submission.rs +++ b/tests/src/jmap/email_submission.rs @@ -22,9 +22,7 @@ */ use ahash::AHashMap; -use jmap::JMAP; use jmap_client::{ - client::Client, core::set::{SetError, SetErrorType, SetObject}, email_submission::{query::Filter, Address, Delivered, DeliveryStatus, Displayed, UndoStatus}, mailbox::Role, @@ -44,11 +42,12 @@ use tokio::{ sync::mpsc, }; -use crate::{ - directory::sql::create_test_user_with_email, - jmap::{assert_is_empty, email_set::assert_email_properties, mailbox::destroy_all_mailboxes}, +use crate::jmap::{ + assert_is_empty, email_set::assert_email_properties, mailbox::destroy_all_mailboxes, }; +use super::JMAPTest; + #[derive(Default, Debug, PartialEq, Eq)] pub struct MockMessage { pub mail_from: String, @@ -79,9 +78,11 @@ pub struct MockSMTPSettings { } #[allow(clippy::disallowed_types)] -pub async fn test(server: Arc, client: &mut Client) { +pub async fn test(params: &mut JMAPTest) { println!("Running E-mail submissions tests..."); // Start mock SMTP server + let server = params.server.clone(); + let client = &mut params.client; let (mut smtp_rx, smtp_settings) = spawn_mock_smtp_server(); server.smtp.resolvers.dns.ipv4_add( "localhost", @@ -90,8 +91,11 @@ pub async fn test(server: Arc, client: &mut Client) { ); // Create a test account - let directory = server.directory.as_ref(); - create_test_user_with_email(directory, "jdoe@example.com", "12345", "John Doe").await; + let server = params.server.clone(); + params + .directory + .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") + .await; let account_id = Id::from(server.get_account_id("jdoe@example.com").await.unwrap()).to_string(); // Create an identity without using a valid address should fail @@ -470,7 +474,7 @@ pub async fn test(server: Arc, client: &mut Client) { { client.email_submission_destroy(&id).await.unwrap(); } - destroy_all_mailboxes(client).await; + destroy_all_mailboxes(¶ms.client).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/event_source.rs b/tests/src/jmap/event_source.rs index ace050c7..c31f7858 100644 --- a/tests/src/jmap/event_source.rs +++ b/tests/src/jmap/event_source.rs @@ -21,29 +21,30 @@ * for more details. */ -use std::{sync::Arc, time::Duration}; +use std::time::Duration; -use crate::{ - directory::sql::create_test_user_with_email, - jmap::{ - assert_is_empty, delivery::SmtpConnection, mailbox::destroy_all_mailboxes, - test_account_login, - }, +use crate::jmap::{ + assert_is_empty, delivery::SmtpConnection, mailbox::destroy_all_mailboxes, test_account_login, }; use futures::StreamExt; -use jmap::{mailbox::INBOX_ID, JMAP}; -use jmap_client::{client::Client, event_source::Changes, mailbox::Role, TypeState}; +use jmap::mailbox::INBOX_ID; +use jmap_client::{event_source::Changes, mailbox::Role, TypeState}; use jmap_proto::types::id::Id; use store::ahash::AHashSet; use tokio::sync::mpsc; -pub async fn test(server: Arc, admin_client: &mut Client) { +use super::JMAPTest; + +pub async fn test(params: &mut JMAPTest) { println!("Running EventSource tests..."); // Create test account - let directory = server.directory.as_ref(); - create_test_user_with_email(directory, "jdoe@example.com", "12345", "John Doe").await; + let server = params.server.clone(); + params + .directory + .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") + .await; let account_id = Id::from(server.get_account_id("jdoe@example.com").await.unwrap()).to_string(); let client = test_account_login("jdoe@example.com", "12345").await; @@ -118,8 +119,11 @@ pub async fn test(server: Arc, admin_client: &mut Client) { assert_state(&mut event_rx, &account_id, &[TypeState::Mailbox]).await; // Destroy Inbox - admin_client.set_default_account_id(&account_id.to_string()); - admin_client + params + .client + .set_default_account_id(&account_id.to_string()); + params + .client .mailbox_destroy(&Id::from(INBOX_ID).to_string(), true) .await .unwrap(); @@ -132,7 +136,7 @@ pub async fn test(server: Arc, admin_client: &mut Client) { assert_ping(&mut event_rx).await; assert_ping(&mut event_rx).await; - destroy_all_mailboxes(admin_client).await; + destroy_all_mailboxes(¶ms.client).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/mailbox.rs b/tests/src/jmap/mailbox.rs index 37501bd4..b7c60cdd 100644 --- a/tests/src/jmap/mailbox.rs +++ b/tests/src/jmap/mailbox.rs @@ -21,9 +21,6 @@ * for more details. */ -use std::sync::Arc; - -use jmap::JMAP; use jmap_client::{ client::Client, core::{ @@ -39,8 +36,12 @@ use store::ahash::AHashMap; use crate::jmap::assert_is_empty; -pub async fn test(server: Arc, client: &mut Client) { +use super::JMAPTest; + +pub async fn test(params: &mut JMAPTest) { println!("Running Mailbox tests..."); + let server = params.server.clone(); + let client = &mut params.client; // Create test mailboxes client.set_default_account_id(Id::from(0u64)); diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index ee9f8f8a..91a755da 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -34,14 +34,11 @@ use jmap_client::client::{Client, Credentials}; use jmap_proto::types::id::Id; use reqwest::header; use smtp::core::{SmtpSessionManager, SMTP}; +use store::config::ConfigStore; use tokio::sync::{mpsc, watch}; use utils::{config::ServerProtocol, UnwrapFailure}; -use crate::{ - add_test_certs, - directory::sql::{add_to_group, create_test_directory, create_test_user}, - store::TempDir, -}; +use crate::{add_test_certs, directory::DirectoryStore, store::TempDir}; pub mod auth_acl; pub mod auth_limits; @@ -99,7 +96,7 @@ reject-non-fqdn = false [session.rcpt] relay = [ { if = "authenticated-as", ne = "", then = true }, { else = false } ] -directory = "sql" +directory = "auth" [session.rcpt.errors] total = 5 @@ -134,35 +131,51 @@ allow-invalid-certs = true future-release = [ { if = "authenticated-as", ne = "", then = "99999999d"}, { else = false } ] -[store.db] +[store."sqlite"] +type = "sqlite" path = "{TMP}/sqlite.db" + +[store."rocksdb"] +type = "rocksdb" +path = "{TMP}/rocks.db" + +[store."foundationdb"] +type = "foundationdb" + +[store."postgresql"] +type = "postgresql" +host = "localhost" +port = 5432 +database = "stalwart" +user = "postgres" +password = "mysecretpassword" + +[store."mysql"] +type = "mysql" host = "localhost" -#port = 5432 port = 3307 database = "stalwart" -#user = "postgres" -#password = "mysecretpassword" user = "root" password = "password" -[store.fts] +[store."elastic"] +type = "elasticsearch" url = "https://localhost:9200" user = "elastic" password = "RtQ-Lu6+o4rxx=XJplVJ" allow-invalid-certs = true -[store.blob] -type = "local" - -[store.blob.local] -path = "{TMP}" - [certificate.default] cert = "file://{CERT}" private-key = "file://{PK}" [jmap] -directory = "sql" +directory = "auth" + +[jmap.store] +data = "sqlite" +fts = "sqlite" +blob = "sqlite" [jmap.protocol] set.max-objects = 100000 @@ -194,14 +207,11 @@ throttle = "500ms" throttle = "500ms" attempts.interval = "500ms" -[directory."sql"] -type = "sql" -address = "sqlite::memory:" +[store."auth"] +type = "sqlite" +path = "{TMP}/auth.db" -[directory."sql".pool] -max-connections = 1 - -[directory."sql".query] +[store."auth".query] name = "SELECT name, type, secret, description, quota FROM accounts WHERE name = ? AND active = true" members = "SELECT member_of FROM group_members WHERE name = ?" recipients = "SELECT name FROM emails WHERE address = ?" @@ -210,7 +220,11 @@ verify = "SELECT address FROM emails WHERE address LIKE '%' || ? || '%' AND type expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = ? AND l.type = 'list' ORDER BY p.address LIMIT 50" domains = "SELECT 1 FROM emails WHERE address LIKE '%@' || ? LIMIT 1" -[directory."sql".columns] +[directory."auth"] +type = "sql" +store = "auth" + +[directory."auth".columns] name = "name" description = "description" secret = "secret" @@ -218,12 +232,16 @@ email = "address" quota = "quota" type = "type" -[directory."local"] +[store."local"] type = "memory" -[directory."local".lookup] -domains = ["example.com"] -remote-domains = ["remote.org", "foobar.com", "test.com", "other_domain.com"] +[store."local".lookup."domains"] +type = "list" +values = ["example.com"] + +[store."local".lookup."remote-domains"] +type = "list" +values = ["remote.org", "foobar.com", "test.com", "other_domain.com"] [oauth] key = "parerga_und_paralipomena" @@ -256,30 +274,30 @@ pub async fn jmap_tests() { let delete = true; let mut params = init_jmap_tests(delete).await; - email_query::test(params.server.clone(), &mut params.client, delete).await; - email_get::test(params.server.clone(), &mut params.client).await; - email_set::test(params.server.clone(), &mut params.client).await; - email_parse::test(params.server.clone(), &mut params.client).await; - email_search_snippet::test(params.server.clone(), &mut params.client).await; - email_changes::test(params.server.clone(), &mut params.client).await; - email_query_changes::test(params.server.clone(), &mut params.client).await; - email_copy::test(params.server.clone(), &mut params.client).await; - thread_get::test(params.server.clone(), &mut params.client).await; - thread_merge::test(params.server.clone(), &mut params.client).await; - mailbox::test(params.server.clone(), &mut params.client).await; - delivery::test(params.server.clone(), &mut params.client).await; - auth_acl::test(params.server.clone(), &mut params.client).await; - auth_limits::test(params.server.clone(), &mut params.client).await; - auth_oauth::test(params.server.clone(), &mut params.client).await; - event_source::test(params.server.clone(), &mut params.client).await; - push_subscription::test(params.server.clone(), &mut params.client).await; - sieve_script::test(params.server.clone(), &mut params.client).await; - vacation_response::test(params.server.clone(), &mut params.client).await; - email_submission::test(params.server.clone(), &mut params.client).await; - websocket::test(params.server.clone(), &mut params.client).await; - quota::test(params.server.clone(), &mut params.client).await; - crypto::test(params.server.clone(), &mut params.client).await; - blob::test(params.server.clone(), &mut params.client).await; + email_query::test(&mut params, delete).await; + email_get::test(&mut params).await; + email_set::test(&mut params).await; + email_parse::test(&mut params).await; + email_search_snippet::test(&mut params).await; + email_changes::test(&mut params).await; + email_query_changes::test(&mut params).await; + email_copy::test(&mut params).await; + thread_get::test(&mut params).await; + thread_merge::test(&mut params).await; + mailbox::test(&mut params).await; + delivery::test(&mut params).await; + auth_acl::test(&mut params).await; + auth_limits::test(&mut params).await; + auth_oauth::test(&mut params).await; + event_source::test(&mut params).await; + push_subscription::test(&mut params).await; + sieve_script::test(&mut params).await; + vacation_response::test(&mut params).await; + email_submission::test(&mut params).await; + websocket::test(&mut params).await; + quota::test(&mut params).await; + crypto::test(&mut params).await; + blob::test(&mut params).await; if delete { params.temp_dir.delete(); @@ -302,9 +320,10 @@ pub async fn jmap_stress_tests() { } #[allow(dead_code)] -struct JMAPTest { +pub struct JMAPTest { server: Arc, client: Client, + directory: DirectoryStore, temp_dir: TempDir, shutdown_tx: watch::Sender, } @@ -344,15 +363,16 @@ async fn init_jmap_tests(delete_if_exists: bool) -> JMAPTest { ) .unwrap(); let servers = config.parse_servers().unwrap(); - let directory = config.parse_directory().unwrap(); + let stores = config.parse_stores().await.failed("Invalid configuration"); + let directory = config.parse_directory(&stores).unwrap(); // Start JMAP and SMTP servers servers.bind(&config); let (delivery_tx, delivery_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); - let smtp = SMTP::init(&config, &servers, &directory, delivery_tx) + let smtp = SMTP::init(&config, &servers, &stores, &directory, delivery_tx) .await .failed("Invalid configuration file"); - let jmap = JMAP::init(&config, &directory, delivery_rx, smtp.clone()) + let jmap = JMAP::init(&config, &stores, &directory, delivery_rx, smtp.clone()) .await .failed("Invalid configuration file"); let (shutdown_tx, _) = servers.spawn(|server, shutdown_rx| { @@ -368,9 +388,14 @@ async fn init_jmap_tests(delete_if_exists: bool) -> JMAPTest { }); // Create tables - create_test_directory(jmap.directory.as_ref()).await; - create_test_user(jmap.directory.as_ref(), "admin", "secret", "Superuser").await; - add_to_group(jmap.directory.as_ref(), "admin", "superusers").await; + let directory = DirectoryStore { + store: stores.lookup_stores.get("auth").unwrap().clone(), + }; + directory.create_test_directory().await; + directory + .create_test_user("admin", "secret", "Superuser") + .await; + directory.add_to_group("admin", "superusers").await; if delete_if_exists { jmap.store.destroy().await; @@ -390,6 +415,7 @@ async fn init_jmap_tests(delete_if_exists: bool) -> JMAPTest { server: jmap, temp_dir, client, + directory, shutdown_tx, } } diff --git a/tests/src/jmap/push_subscription.rs b/tests/src/jmap/push_subscription.rs index 94002e54..38a1c47e 100644 --- a/tests/src/jmap/push_subscription.rs +++ b/tests/src/jmap/push_subscription.rs @@ -40,9 +40,8 @@ use jmap::{ }, auth::AccessToken, push::ece::ece_encrypt, - JMAP, }; -use jmap_client::{client::Client, mailbox::Role, push_subscription::Keys}; +use jmap_client::{mailbox::Role, push_subscription::Keys}; use jmap_proto::types::{id::Id, type_state::DataType}; use store::ahash::AHashSet; @@ -51,10 +50,11 @@ use utils::listener::SessionData; use crate::{ add_test_certs, - directory::sql::create_test_user_with_email, jmap::{assert_is_empty, mailbox::destroy_all_mailboxes, test_account_login}, }; +use super::JMAPTest; + const SERVER: &str = " [server] hostname = 'jmap-push.example.org' @@ -77,14 +77,17 @@ cert = 'file://{CERT}' private-key = 'file://{PK}' "; -pub async fn test(server: Arc, admin_client: &mut Client) { +pub async fn test(params: &mut JMAPTest) { println!("Running Push Subscription tests..."); // Create test account - let directory = server.directory.as_ref(); - create_test_user_with_email(directory, "jdoe@example.com", "12345", "John Doe").await; + let server = params.server.clone(); + params + .directory + .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") + .await; let account_id = Id::from(server.get_account_id("jdoe@example.com").await.unwrap()); - admin_client.set_default_account_id(account_id); + params.client.set_default_account_id(account_id); let client = test_account_login("jdoe@example.com", "12345").await; // Create channels @@ -216,7 +219,7 @@ pub async fn test(server: Arc, admin_client: &mut Client) { client.mailbox_destroy(&mailbox_id, true).await.unwrap(); expect_nothing(&mut event_rx).await; - destroy_all_mailboxes(admin_client).await; + destroy_all_mailboxes(¶ms.client).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/quota.rs b/tests/src/jmap/quota.rs index 6de8bd75..b270507a 100644 --- a/tests/src/jmap/quota.rs +++ b/tests/src/jmap/quota.rs @@ -21,32 +21,40 @@ * for more details. */ -use std::sync::Arc; - -use crate::{ - directory::sql::{add_to_group, create_test_user_with_email, set_test_quota}, - jmap::{ - assert_is_empty, delivery::SmtpConnection, jmap_raw_request, - mailbox::destroy_all_mailboxes, test_account_login, - }, +use crate::jmap::{ + assert_is_empty, delivery::SmtpConnection, jmap_raw_request, mailbox::destroy_all_mailboxes, + test_account_login, }; -use jmap::{blob::upload::DISABLE_UPLOAD_QUOTA, mailbox::INBOX_ID, JMAP}; +use jmap::{blob::upload::DISABLE_UPLOAD_QUOTA, mailbox::INBOX_ID}; use jmap_client::{ - client::Client, core::set::{SetErrorType, SetObject}, email::EmailBodyPart, }; use jmap_proto::types::{collection::Collection, id::Id}; -pub async fn test(server: Arc, admin_client: &mut Client) { +use super::JMAPTest; + +pub async fn test(params: &mut JMAPTest) { println!("Running quota tests..."); - let directory = server.directory.as_ref(); - create_test_user_with_email(directory, "jdoe@example.com", "12345", "John Doe").await; - create_test_user_with_email(directory, "robert@example.com", "aabbcc", "Robert Foobar").await; + let server = params.server.clone(); + params + .directory + .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") + .await; + params + .directory + .create_test_user_with_email("robert@example.com", "aabbcc", "Robert Foobar") + .await; let other_account_id = Id::from(server.get_account_id("jdoe@example.com").await.unwrap()); let account_id = Id::from(server.get_account_id("robert@example.com").await.unwrap()); - set_test_quota(directory, "robert@example.com", 1024).await; - add_to_group(directory, "robert@example.com", "jdoe@example.com").await; + params + .directory + .set_test_quota("robert@example.com", 1024) + .await; + params + .directory + .add_to_group("robert@example.com", "jdoe@example.com") + .await; // Delete temporary blobs from previous tests server.store.blob_hash_expire_all().await; @@ -317,8 +325,8 @@ pub async fn test(server: Arc, admin_client: &mut Client) { // Remove test data for account_id in [&account_id, &other_account_id] { - admin_client.set_default_account_id(account_id.to_string()); - destroy_all_mailboxes(admin_client).await; + params.client.set_default_account_id(account_id.to_string()); + destroy_all_mailboxes(¶ms.client).await; } assert_is_empty(server).await; } diff --git a/tests/src/jmap/sieve_script.rs b/tests/src/jmap/sieve_script.rs index faa56943..56ad8348 100644 --- a/tests/src/jmap/sieve_script.rs +++ b/tests/src/jmap/sieve_script.rs @@ -21,9 +21,7 @@ * for more details. */ -use jmap::JMAP; use jmap_client::{ - client::Client, core::set::{SetError, SetErrorType}, email, mailbox, sieve::query::{Comparator, Filter}, @@ -33,26 +31,28 @@ use jmap_proto::types::id::Id; use std::{ fs, path::PathBuf, - sync::Arc, time::{Duration, Instant}, }; -use crate::{ - directory::sql::create_test_user_with_email, - jmap::{ - assert_is_empty, - delivery::SmtpConnection, - email_submission::{assert_message_delivery, spawn_mock_smtp_server, MockMessage}, - mailbox::destroy_all_mailboxes, - }, +use crate::jmap::{ + assert_is_empty, + delivery::SmtpConnection, + email_submission::{assert_message_delivery, spawn_mock_smtp_server, MockMessage}, + mailbox::destroy_all_mailboxes, }; -pub async fn test(server: Arc, client: &mut Client) { +use super::JMAPTest; + +pub async fn test(params: &mut JMAPTest) { println!("Running Sieve tests..."); + let server = params.server.clone(); + let client = &mut params.client; // Create test account - let directory = server.directory.as_ref(); - create_test_user_with_email(directory, "jdoe@example.com", "12345", "John Doe").await; + params + .directory + .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") + .await; let account_id = Id::from(server.get_account_id("jdoe@example.com").await.unwrap()).to_string(); client.set_default_account_id(&account_id); @@ -486,7 +486,7 @@ pub async fn test(server: Arc, client: &mut Client) { for id in request.send_query_sieve_script().await.unwrap().take_ids() { client.sieve_script_destroy(&id).await.unwrap(); } - destroy_all_mailboxes(client).await; + destroy_all_mailboxes(¶ms.client).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/thread_get.rs b/tests/src/jmap/thread_get.rs index c5397a05..2e8e42cc 100644 --- a/tests/src/jmap/thread_get.rs +++ b/tests/src/jmap/thread_get.rs @@ -21,17 +21,18 @@ * for more details. */ -use std::sync::Arc; - use crate::jmap::{assert_is_empty, mailbox::destroy_all_mailboxes}; -use jmap::JMAP; -use jmap_client::{client::Client, mailbox::Role}; +use jmap_client::mailbox::Role; use jmap_proto::types::id::Id; -pub async fn test(server: Arc, client: &mut Client) { - println!("Running Email Thread tests..."); +use super::JMAPTest; - let mailbox_id = client +pub async fn test(params: &mut JMAPTest) { + println!("Running Email Thread tests..."); + let server = params.server.clone(); + + let mailbox_id = params + .client .set_default_account_id(Id::new(1).to_string()) .mailbox_create("JMAP Get", None::, Role::None) .await @@ -42,7 +43,8 @@ pub async fn test(server: Arc, client: &mut Client) { let mut thread_id = "".to_string(); for num in [5, 3, 1, 2, 4] { - let mut email = client + let mut email = params + .client .email_import( format!("Subject: test\nReferences: <1234>\n\n{}", num).into_bytes(), [&mailbox_id], @@ -56,7 +58,8 @@ pub async fn test(server: Arc, client: &mut Client) { } assert_eq!( - client + params + .client .thread_get(&thread_id) .await .unwrap() @@ -65,6 +68,6 @@ pub async fn test(server: Arc, client: &mut Client) { expected_result ); - destroy_all_mailboxes(client).await; + destroy_all_mailboxes(¶ms.client).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/thread_merge.rs b/tests/src/jmap/thread_merge.rs index b867dcc2..34100a01 100644 --- a/tests/src/jmap/thread_merge.rs +++ b/tests/src/jmap/thread_merge.rs @@ -21,16 +21,17 @@ * for more details. */ -use std::sync::Arc; - use crate::jmap::{assert_is_empty, mailbox::destroy_all_mailboxes}; -use jmap::JMAP; -use jmap_client::{client::Client, email, mailbox::Role}; +use jmap_client::{email, mailbox::Role}; use jmap_proto::types::id::Id; use store::ahash::{AHashMap, AHashSet}; -pub async fn test(server: Arc, client: &mut Client) { +use super::JMAPTest; + +pub async fn test(params: &mut JMAPTest) { println!("Running Email Merge Threads tests..."); + let server = params.server.clone(); + let client = &mut params.client; let mut all_mailboxes = AHashMap::default(); for (base_test_num, test) in [test_1(), test_2(), test_3()].iter().enumerate() { diff --git a/tests/src/jmap/vacation_response.rs b/tests/src/jmap/vacation_response.rs index c9947074..7144b251 100644 --- a/tests/src/jmap/vacation_response.rs +++ b/tests/src/jmap/vacation_response.rs @@ -22,29 +22,31 @@ */ use chrono::{Duration, Utc}; -use jmap::JMAP; -use jmap_client::client::Client; -use jmap_proto::types::id::Id; -use std::{sync::Arc, time::Instant}; -use crate::{ - directory::sql::create_test_user_with_email, - jmap::{ - assert_is_empty, - delivery::SmtpConnection, - email_submission::{ - assert_message_delivery, expect_nothing, spawn_mock_smtp_server, MockMessage, - }, - mailbox::destroy_all_mailboxes, +use jmap_proto::types::id::Id; +use std::time::Instant; + +use crate::jmap::{ + assert_is_empty, + delivery::SmtpConnection, + email_submission::{ + assert_message_delivery, expect_nothing, spawn_mock_smtp_server, MockMessage, }, + mailbox::destroy_all_mailboxes, }; -pub async fn test(server: Arc, client: &mut Client) { +use super::JMAPTest; + +pub async fn test(params: &mut JMAPTest) { println!("Running Vacation Response tests..."); // Create test account - let directory = server.directory.as_ref(); - create_test_user_with_email(directory, "jdoe@example.com", "12345", "John Doe").await; + let server = params.server.clone(); + let client = &mut params.client; + params + .directory + .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") + .await; let account_id = Id::from(server.get_account_id("jdoe@example.com").await.unwrap()).to_string(); client.set_default_account_id(&account_id); @@ -173,6 +175,6 @@ pub async fn test(server: Arc, client: &mut Client) { // Remove test data client.vacation_response_destroy().await.unwrap(); - destroy_all_mailboxes(client).await; + destroy_all_mailboxes(¶ms.client).await; assert_is_empty(server).await; } diff --git a/tests/src/jmap/websocket.rs b/tests/src/jmap/websocket.rs index 599f7f9c..5fb12906 100644 --- a/tests/src/jmap/websocket.rs +++ b/tests/src/jmap/websocket.rs @@ -23,9 +23,7 @@ use ahash::AHashSet; use futures::StreamExt; -use jmap::JMAP; use jmap_client::{ - client::Client, client_ws::WebSocketMessage, core::{ response::{Response, TaggedMethodResponse}, @@ -34,21 +32,23 @@ use jmap_client::{ TypeState, }; use jmap_proto::types::id::Id; -use std::{sync::Arc, time::Duration}; +use std::time::Duration; use tokio::sync::mpsc; -use crate::{ - directory::sql::create_test_user_with_email, - jmap::{assert_is_empty, mailbox::destroy_all_mailboxes, test_account_login}, -}; +use crate::jmap::{assert_is_empty, mailbox::destroy_all_mailboxes, test_account_login}; -pub async fn test(server: Arc, admin_client: &mut Client) { +use super::JMAPTest; + +pub async fn test(params: &mut JMAPTest) { println!("Running WebSockets tests..."); + let server = params.server.clone(); // Authenticate all accounts - let directory = server.directory.as_ref(); - create_test_user_with_email(directory, "jdoe@example.com", "12345", "John Doe").await; + params + .directory + .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") + .await; let account_id = Id::from(server.get_account_id("jdoe@example.com").await.unwrap()).to_string(); let client = test_account_login("jdoe@example.com", "12345").await; @@ -123,8 +123,8 @@ pub async fn test(server: Arc, admin_client: &mut Client) { .unwrap(); expect_nothing(&mut stream_rx).await; - admin_client.set_default_account_id(account_id); - destroy_all_mailboxes(admin_client).await; + params.client.set_default_account_id(account_id); + destroy_all_mailboxes(¶ms.client).await; assert_is_empty(server).await; } diff --git a/tests/src/smtp/config.rs b/tests/src/smtp/config.rs index ffc1e857..0840fdac 100644 --- a/tests/src/smtp/config.rs +++ b/tests/src/smtp/config.rs @@ -30,17 +30,23 @@ use std::{ time::Duration, }; +use store::{ + backend::memory::{LookupList, MemoryStore}, + config::ConfigStore, +}; use tokio::net::TcpSocket; use utils::config::{Config, DynValue, KeyLookup, Listener, Rate, Server, ServerProtocol}; use ahash::AHashMap; -use directory::{config::ConfigDirectory, Lookup, LookupList}; -use smtp::config::{ - condition::ConfigCondition, if_block::ConfigIf, throttle::ConfigThrottle, Condition, - ConditionMatch, Conditions, ConfigContext, EnvelopeKey, IfBlock, IfThen, IpAddrMask, - StringMatch, Throttle, THROTTLE_AUTH_AS, THROTTLE_REMOTE_IP, THROTTLE_SENDER_DOMAIN, +use smtp::{ + config::{ + condition::ConfigCondition, if_block::ConfigIf, throttle::ConfigThrottle, Condition, + ConditionMatch, Conditions, ConfigContext, EnvelopeKey, IfBlock, IfThen, IpAddrMask, + StringMatch, Throttle, THROTTLE_AUTH_AS, THROTTLE_REMOTE_IP, THROTTLE_SENDER_DOMAIN, + }, + core::Lookup, }; use super::add_test_certs; @@ -74,11 +80,12 @@ fn parse_conditions() { ..Default::default() }]; let mut context = ConfigContext::new(&servers); - let list = Arc::new(Lookup::List { - list: LookupList::default(), + let list = Arc::new(store::Lookup { + store: MemoryStore::List(LookupList::default()).into(), + query: "abc".into(), }); context - .directory + .stores .lookups .insert("test-list".to_string(), list.clone()); @@ -118,7 +125,7 @@ fn parse_conditions() { Condition::JumpIfFalse { positions: 1 }, Condition::Match { key: EnvelopeKey::Sender, - value: ConditionMatch::Lookup(list), + value: ConditionMatch::Lookup(list.into()), not: false, }, ], @@ -568,7 +575,7 @@ async fn eval_if() { }, ]; let mut context = ConfigContext::new(&servers); - context.directory = config.parse_directory().unwrap(); + context.stores = config.parse_stores().await.unwrap(); let conditions = config.parse_conditions(&context).unwrap(); let envelope = TestEnvelope::from_config(&config); @@ -602,7 +609,7 @@ async fn eval_dynvalue() { let config = Config::new(&fs::read_to_string(file).unwrap()).unwrap(); let mut context = ConfigContext::new(&[]); - context.directory = config.parse_directory().unwrap(); + context.stores = config.parse_stores().await.unwrap(); let envelope = TestEnvelope::from_config(&config); @@ -664,7 +671,7 @@ async fn eval_dynvalue() { .unwrap() .unwrap() .map_if_block( - &context.directory.directories, + &context.stores.lookups, ("maybe-eval", test_name, "test"), "test", ) @@ -673,14 +680,14 @@ async fn eval_dynvalue() { .value_require(("maybe-eval", test_name, "expect")) .unwrap(); - assert!(if_block + let lookup: Lookup = if_block .eval_and_capture(&envelope) .await .into_value(&envelope) .unwrap() - .is_local_domain(expected) - .await - .unwrap()); + .into(); + + assert!(lookup.contains(expected).await.unwrap()); } } diff --git a/tests/src/smtp/inbound/antispam.rs b/tests/src/smtp/inbound/antispam.rs index 1dc4fb2f..2471ad5d 100644 --- a/tests/src/smtp/inbound/antispam.rs +++ b/tests/src/smtp/inbound/antispam.rs @@ -9,7 +9,6 @@ use std::{ use crate::smtp::session::TestSession; use ahash::AHashMap; -use directory::config::ConfigDirectory; use mail_auth::{dmarc::Policy, DkimResult, DmarcResult, IprevResult, SpfResult, MX}; use sieve::runtime::Variable; use smtp::{ @@ -21,6 +20,7 @@ use smtp::{ ScriptModification, ScriptResult, }, }; +use store::config::ConfigStore; use tokio::runtime::Handle; use utils::config::Config; @@ -42,17 +42,16 @@ cpu = 500000 nested-includes = 5 duplicate-expiry = "7d" -[directory."spamdb"] -type = "sql" -address = "sqlite://%PATH%/test_antispam.db?mode=rwc" -#address = "sqlite:///tmp/test_antispam.db?mode=rwc" +[store."spamdb"] +type = "sqlite" +path = "%PATH%/test_antispam.db" -[directory."spamdb".pool] +[store."spamdb".pool] max-connections = 10 min-connections = 0 idle-timeout = "5m" -[directory."spamdb".lookup] +[store."spamdb".query] token-insert = "INSERT INTO bayes_tokens (h1, h2, ws, wh) VALUES (?, ?, ?, ?) ON CONFLICT(h1, h2) DO UPDATE SET ws = ws + excluded.ws, wh = wh + excluded.wh" @@ -66,46 +65,47 @@ reputation-insert = "INSERT INTO reputation (token, score, count, ttl) VALUES (? reputation-lookup = "SELECT score, count FROM reputation WHERE token = ?" reputation-cleanup = "DELETE FROM reputation WHERE ttl < CURRENT_TIMESTAMP" -[directory."default"] +[store."default"] type = "memory" -[directory."default".lookup] -domains = ["local-domain.org"] +[store."default".lookup."domains"] +type = "list" +values = ["local-domain.org"] -[directory."spam"] +[store."spam"] type = "memory" -[directory."spam".lookup."free-domains"] +[store."spam".lookup."free-domains"] type = "glob" comment = '#' values = ["gmail.com", "googlemail.com", "yahoomail.com", "*.freemail.org"] -[directory."spam".lookup."disposable-domains"] +[store."spam".lookup."disposable-domains"] type = "glob" comment = '#' values = ["guerrillamail.com", "*.disposable.org"] -[directory."spam".lookup."redirectors"] +[store."spam".lookup."redirectors"] type = "glob" comment = '#' values = ["bit.ly", "redirect.io", "redirect.me", "redirect.org", "redirect.com", "redirect.net", "t.ly", "tinyurl.com"] -[directory."spam".lookup."dmarc-allow"] +[store."spam".lookup."dmarc-allow"] type = "glob" comment = '#' values = ["dmarc-allow.org"] -[directory."spam".lookup."spf-dkim-allow"] +[store."spam".lookup."spf-dkim-allow"] type = "glob" comment = '#' values = ["spf-dkim-allow.org"] -[directory."spam".lookup."domains-allow"] +[store."spam".lookup."domains-allow"] type = "glob" values = [] -[directory."spam".lookup."mime-types"] +[store."spam".lookup."mime-types"] type = "map" comment = '#' values = ["html text/html|BAD", @@ -115,12 +115,12 @@ values = ["html text/html|BAD", "js BAD|NZ", "hta BAD|NZ"] -[directory."spam".lookup."trap-address"] +[store."spam".lookup."trap-address"] type = "glob" comment = '#' values = ["spamtrap@*"] -[directory."spam".lookup."scores"] +[store."spam".lookup."scores"] type = "map" values = "file://%CFG_PATH%/maps/scores.map" @@ -254,15 +254,15 @@ async fn antispam() { // Parse config let config = Config::new(&config).unwrap(); let mut ctx = ConfigContext::new(&[]); - ctx.directory = config.parse_directory().unwrap(); + ctx.stores = config.parse_stores().await.unwrap(); core.sieve = config.parse_sieve(&mut ctx).unwrap(); let config = &mut core.session.config; config.rcpt.relay = IfBlock::new(true); // Create tables - let sdb = ctx.directory.directories.get("spamdb").unwrap(); + let sdb = ctx.stores.lookup_stores.get("spamdb").unwrap(); for query in CREATE_TABLES { - sdb.query(query, &[]).await.expect(query); + sdb.query::(query, vec![]).await.expect(query); } // Add mock DNS entries diff --git a/tests/src/smtp/inbound/auth.rs b/tests/src/smtp/inbound/auth.rs index 34f8cd56..a8fe7552 100644 --- a/tests/src/smtp/inbound/auth.rs +++ b/tests/src/smtp/inbound/auth.rs @@ -23,6 +23,7 @@ use directory::config::ConfigDirectory; use smtp_proto::{AUTH_LOGIN, AUTH_PLAIN}; +use store::Stores; use utils::config::{Config, DynValue}; use crate::smtp::{ @@ -59,7 +60,10 @@ member-of = ["sales", "support"] async fn auth() { let mut core = SMTP::test(); let mut ctx = ConfigContext::new(&[]); - ctx.directory = Config::new(DIRECTORY).unwrap().parse_directory().unwrap(); + ctx.directory = Config::new(DIRECTORY) + .unwrap() + .parse_directory(&Stores::default()) + .unwrap(); let config = &mut core.session.config.auth; diff --git a/tests/src/smtp/inbound/data.rs b/tests/src/smtp/inbound/data.rs index 59700d43..5d5e726e 100644 --- a/tests/src/smtp/inbound/data.rs +++ b/tests/src/smtp/inbound/data.rs @@ -22,6 +22,7 @@ */ use directory::config::ConfigDirectory; +use store::Stores; use utils::config::Config; use crate::smtp::{ @@ -62,8 +63,6 @@ description = "Mike Foobar" secret = "p4ssw0rd" email = "mike@test.com" -[directory."local".lookup] -domains = ["foobar.org", "domain.net", "test.com"] "#; #[tokio::test] @@ -79,7 +78,10 @@ async fn data() { // Create temp dir for queue let mut qr = core.init_test_queue("smtp_data_test"); - let directory = Config::new(DIRECTORY).unwrap().parse_directory().unwrap(); + let directory = Config::new(DIRECTORY) + .unwrap() + .parse_directory(&Stores::default()) + .unwrap(); let config = &mut core.session.config.rcpt; config.directory = IfBlock::new(Some(MaybeDynValue::Static( directory.directories.get("local").unwrap().clone(), diff --git a/tests/src/smtp/inbound/dmarc.rs b/tests/src/smtp/inbound/dmarc.rs index debd000d..f175ef24 100644 --- a/tests/src/smtp/inbound/dmarc.rs +++ b/tests/src/smtp/inbound/dmarc.rs @@ -34,6 +34,7 @@ use mail_auth::{ report::DmarcResult, spf::Spf, }; +use store::Stores; use utils::config::{Config, DynValue, Rate}; use crate::smtp::{ @@ -58,8 +59,6 @@ description = "John Doe" secret = "secret" email = ["jdoe@example.com"] -[directory."local".lookup] -domains = ["example.com"] "#; #[tokio::test] @@ -134,7 +133,10 @@ async fn dmarc() { // Create report channels let mut rr = core.init_test_report(); - let directory = Config::new(DIRECTORY).unwrap().parse_directory().unwrap(); + let directory = Config::new(DIRECTORY) + .unwrap() + .parse_directory(&Stores::default()) + .unwrap(); let config = &mut core.session.config.rcpt; config.directory = IfBlock::new(Some(MaybeDynValue::Static( directory.directories.get("local").unwrap().clone(), diff --git a/tests/src/smtp/inbound/rcpt.rs b/tests/src/smtp/inbound/rcpt.rs index e63017f1..1805e27c 100644 --- a/tests/src/smtp/inbound/rcpt.rs +++ b/tests/src/smtp/inbound/rcpt.rs @@ -25,6 +25,7 @@ use std::time::Duration; use directory::config::ConfigDirectory; use smtp_proto::{RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_SUCCESS}; +use store::Stores; use utils::config::Config; use crate::smtp::{ @@ -64,8 +65,6 @@ description = "Mike Foobar" secret = "p4ssw0rd" email = "mike@foobar.org" -[directory."local".lookup] -domains = ["foobar.org"] "#; #[tokio::test] @@ -73,7 +72,10 @@ async fn rcpt() { let mut core = SMTP::test(); let config_ext = &mut core.session.config.extensions; - let directory = Config::new(DIRECTORY).unwrap().parse_directory().unwrap(); + let directory = Config::new(DIRECTORY) + .unwrap() + .parse_directory(&Stores::default()) + .unwrap(); let config = &mut core.session.config.rcpt; config.directory = IfBlock::new(Some(MaybeDynValue::Static( directory.directories.get("local").unwrap().clone(), diff --git a/tests/src/smtp/inbound/rewrite.rs b/tests/src/smtp/inbound/rewrite.rs index e67d70db..93fc2c9f 100644 --- a/tests/src/smtp/inbound/rewrite.rs +++ b/tests/src/smtp/inbound/rewrite.rs @@ -27,6 +27,7 @@ use smtp::{ config::{if_block::ConfigIf, scripts::ConfigSieve, ConfigContext, EnvelopeKey, IfBlock}, core::{Session, SMTP}, }; +use store::Stores; use utils::config::{Config, DynValue}; const CONFIG: &str = r#" @@ -103,7 +104,7 @@ async fn address_rewrite() { let mut core = SMTP::test(); let mut ctx = ConfigContext::new(&[]).parse_signatures(); let settings = Config::new(CONFIG).unwrap(); - ctx.directory = settings.parse_directory().unwrap(); + ctx.directory = settings.parse_directory(&Stores::default()).unwrap(); core.sieve = settings.parse_sieve(&mut ctx).unwrap(); let config = &mut core.session.config; config.mail.script = settings diff --git a/tests/src/smtp/inbound/scripts.rs b/tests/src/smtp/inbound/scripts.rs index b91b9a8c..174cd13c 100644 --- a/tests/src/smtp/inbound/scripts.rs +++ b/tests/src/smtp/inbound/scripts.rs @@ -35,24 +35,26 @@ use smtp::{ core::{Session, SMTP}, scripts::ScriptResult, }; +use store::config::ConfigStore; use tokio::runtime::Handle; use utils::config::Config; const CONFIG: &str = r#" -[directory."sql"] -type = "sql" -address = "sqlite://%PATH%/test.db?mode=rwc" +[store."sql"] +type = "sqlite" +path = "%PATH%/smtp_sieve.db" -[directory."sql".pool] +[store."sql".pool] max-connections = 10 min-connections = 0 idle-timeout = "5m" -[directory."local"] +[store."local"] type = "memory" -[directory."local".lookup] -invalid-ehlos = ["spammer.org", "spammer.net"] +[store."local".lookup."invalid-ehlos"] +type = "list" +values = ["spammer.org", "spammer.net"] [session.data.pipe."test"] command = [ { if = "remote-ip", eq = "10.0.0.123", then = "/bin/bash" }, @@ -132,7 +134,8 @@ async fn sieve_scripts() { ), ) .unwrap(); - ctx.directory = config.parse_directory().unwrap(); + ctx.stores = config.parse_stores().await.unwrap(); + ctx.directory = config.parse_directory(&ctx.stores).unwrap(); let pipes = config.parse_pipes(&ctx, &[EnvelopeKey::RemoteIp]).unwrap(); core.sieve = config.parse_sieve(&mut ctx).unwrap(); let config = &mut core.session.config; diff --git a/tests/src/smtp/inbound/sign.rs b/tests/src/smtp/inbound/sign.rs index 0752cd4f..47135c18 100644 --- a/tests/src/smtp/inbound/sign.rs +++ b/tests/src/smtp/inbound/sign.rs @@ -28,6 +28,7 @@ use mail_auth::{ common::{parse::TxtRecordParser, verify::DomainKey}, spf::Spf, }; +use store::Stores; use utils::config::{Config, DynValue}; use crate::smtp::{ @@ -103,9 +104,6 @@ name = "john" description = "John Doe" secret = "secret" email = ["jdoe@example.com"] - -[directory."local".lookup] -domains = ["example.com"] "#; #[tokio::test] @@ -154,7 +152,10 @@ async fn sign_and_seal() { Instant::now() + Duration::from_secs(5), ); - let directory = Config::new(DIRECTORY).unwrap().parse_directory().unwrap(); + let directory = Config::new(DIRECTORY) + .unwrap() + .parse_directory(&Stores::default()) + .unwrap(); let config = &mut core.session.config.rcpt; config.directory = IfBlock::new(Some(MaybeDynValue::Static( directory.directories.get("local").unwrap().clone(), diff --git a/tests/src/smtp/inbound/vrfy.rs b/tests/src/smtp/inbound/vrfy.rs index 55d184e7..12e6d5f9 100644 --- a/tests/src/smtp/inbound/vrfy.rs +++ b/tests/src/smtp/inbound/vrfy.rs @@ -22,6 +22,7 @@ */ use directory::config::ConfigDirectory; +use store::Stores; use utils::config::Config; use crate::smtp::{ @@ -65,7 +66,10 @@ async fn vrfy_expn() { let mut core = SMTP::test(); let ctx = ConfigContext::new(&[]); - let directory = Config::new(DIRECTORY).unwrap().parse_directory().unwrap(); + let directory = Config::new(DIRECTORY) + .unwrap() + .parse_directory(&Stores::default()) + .unwrap(); let config = &mut core.session.config.rcpt; config.directory = IfBlock::new(Some(MaybeDynValue::Static( directory.directories.get("local").unwrap().clone(), diff --git a/tests/src/smtp/lookup/sql.rs b/tests/src/smtp/lookup/sql.rs index d645a1ae..e542d942 100644 --- a/tests/src/smtp/lookup/sql.rs +++ b/tests/src/smtp/lookup/sql.rs @@ -25,14 +25,16 @@ use std::time::Duration; use directory::config::ConfigDirectory; use smtp_proto::{AUTH_LOGIN, AUTH_PLAIN}; +use store::config::ConfigStore; use utils::config::{Config, DynValue}; use crate::{ - directory::sql::{create_test_directory, create_test_user_with_email, link_test_address}, + directory::DirectoryStore, smtp::{ session::{TestSession, VerifyResponse}, ParseTestConfig, TestConfig, }, + store::TempDir, }; use smtp::{ config::{ConfigContext, EnvelopeKey, IfBlock}, @@ -40,14 +42,11 @@ use smtp::{ }; const CONFIG: &str = r#" -[directory."sql"] -type = "sql" -address = "sqlite::memory:" +[store."sql"] +type = "sqlite" +path = "{TMP}/smtp_sql.db" -[directory."sql".pool] -max-connections = 1 - -[directory."sql".query] +[store."sql".query] name = "SELECT name, type, secret, description, quota FROM accounts WHERE name = ? AND active = true" members = "SELECT member_of FROM group_members WHERE name = ?" recipients = "SELECT name FROM emails WHERE address = ?" @@ -55,6 +54,11 @@ emails = "SELECT address FROM emails WHERE name = ? AND type != 'list' ORDER BY verify = "SELECT address FROM emails WHERE address LIKE '%' || ? || '%' AND type = 'primary' ORDER BY address LIMIT 5" expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = ? AND l.type = 'list' ORDER BY p.address LIMIT 50" domains = "SELECT 1 FROM emails WHERE address LIKE '%@' || ? LIMIT 1" +is_ip_allowed = "SELECT addr FROM allowed_ips WHERE addr = ? LIMIT 1" + +[directory."sql"] +type = "sql" +store = "sql" [directory."sql".columns] name = "name" @@ -64,10 +68,6 @@ email = "address" quota = "quota" type = "type" -[directory."sql".lookup] -domains = "SELECT name FROM domains WHERE name = ? LIMIT 1" -is_ip_allowed = "SELECT addr FROM allowed_ips WHERE addr = ? LIMIT 1" - "#; #[tokio::test] @@ -81,26 +81,47 @@ async fn lookup_sql() { .unwrap();*/ // Parse settings + let temp_dir = TempDir::new("smtp_lookup_tests", true); + let config_file = CONFIG.replace("{TMP}", &temp_dir.path.to_string_lossy()); let mut core = SMTP::test(); let mut ctx = ConfigContext::new(&[]); - let config = Config::new(CONFIG).unwrap(); - ctx.directory = config.parse_directory().unwrap(); + let config = Config::new(&config_file).unwrap(); + ctx.stores = config.parse_stores().await.unwrap(); + ctx.directory = config.parse_directory(&ctx.stores).unwrap(); // Obtain directory handle - let handle = ctx.directory.directories.get("sql").unwrap().as_ref(); + let handle = DirectoryStore { + store: ctx.stores.lookup_stores.get("sql").unwrap().clone(), + }; // Create tables - create_test_directory(handle).await; + handle.create_test_directory().await; // Create test records - create_test_user_with_email(handle, "jane@foobar.org", "s3cr3tp4ss", "Jane").await; - create_test_user_with_email(handle, "john@foobar.org", "mypassword", "John").await; - create_test_user_with_email(handle, "bill@foobar.org", "123456", "Bill").await; - create_test_user_with_email(handle, "mike@foobar.net", "098765", "Mike").await; - link_test_address(handle, "jane@foobar.org", "sales@foobar.org", "list").await; - link_test_address(handle, "john@foobar.org", "sales@foobar.org", "list").await; - link_test_address(handle, "bill@foobar.org", "sales@foobar.org", "list").await; - link_test_address(handle, "mike@foobar.net", "support@foobar.org", "list").await; + handle + .create_test_user_with_email("jane@foobar.org", "s3cr3tp4ss", "Jane") + .await; + handle + .create_test_user_with_email("john@foobar.org", "mypassword", "John") + .await; + handle + .create_test_user_with_email("bill@foobar.org", "123456", "Bill") + .await; + handle + .create_test_user_with_email("mike@foobar.net", "098765", "Mike") + .await; + handle + .link_test_address("jane@foobar.org", "sales@foobar.org", "list") + .await; + handle + .link_test_address("john@foobar.org", "sales@foobar.org", "list") + .await; + handle + .link_test_address("bill@foobar.org", "sales@foobar.org", "list") + .await; + handle + .link_test_address("mike@foobar.net", "support@foobar.org", "list") + .await; for query in [ "CREATE TABLE domains (name TEXT PRIMARY KEY, description TEXT);", @@ -109,7 +130,11 @@ async fn lookup_sql() { "CREATE TABLE allowed_ips (addr TEXT PRIMARY KEY);", "INSERT INTO allowed_ips (addr) VALUES ('10.0.0.50');", ] { - handle.query(query, &[]).await.unwrap(); + handle + .store + .query::(query, Vec::new()) + .await + .unwrap(); } // Enable AUTH diff --git a/tests/src/smtp/management/queue.rs b/tests/src/smtp/management/queue.rs index cfb268a2..eb820029 100644 --- a/tests/src/smtp/management/queue.rs +++ b/tests/src/smtp/management/queue.rs @@ -31,6 +31,7 @@ use directory::config::ConfigDirectory; use mail_auth::MX; use mail_parser::DateTime; use reqwest::{header::AUTHORIZATION, StatusCode}; +use store::Stores; use utils::config::{Config, ServerProtocol}; use crate::smtp::{ @@ -95,7 +96,10 @@ async fn manage_queue() { ); // Start local management interface - let directory = Config::new(DIRECTORY).unwrap().parse_directory().unwrap(); + let directory = Config::new(DIRECTORY) + .unwrap() + .parse_directory(&Stores::default()) + .unwrap(); core.queue.config.management_lookup = directory.directories.get("local").unwrap().clone(); core.session.config.rcpt.relay = IfBlock::new(true); core.session.config.rcpt.max_recipients = IfBlock::new(100); diff --git a/tests/src/smtp/management/report.rs b/tests/src/smtp/management/report.rs index 8e68593b..357059e8 100644 --- a/tests/src/smtp/management/report.rs +++ b/tests/src/smtp/management/report.rs @@ -34,6 +34,7 @@ use mail_auth::{ ActionDisposition, DmarcResult, Record, }, }; +use store::Stores; use tokio::sync::mpsc; use utils::config::{Config, ServerProtocol}; @@ -82,7 +83,10 @@ async fn manage_reports() { config.hash = IfBlock::new(16); config.dmarc_aggregate.max_size = IfBlock::new(1024); config.tls.max_size = IfBlock::new(1024); - let directory = Config::new(DIRECTORY).unwrap().parse_directory().unwrap(); + let directory = Config::new(DIRECTORY) + .unwrap() + .parse_directory(&Stores::default()) + .unwrap(); core.queue.config.management_lookup = directory.directories.get("local").unwrap().clone(); let (report_tx, report_rx) = mpsc::channel(1024); core.report.tx = report_tx; diff --git a/tests/src/smtp/mod.rs b/tests/src/smtp/mod.rs index 20ebe519..0c932112 100644 --- a/tests/src/smtp/mod.rs +++ b/tests/src/smtp/mod.rs @@ -440,6 +440,7 @@ impl TestConfig for SieveCore { return_path: "".to_string(), sign: vec![], directories: Default::default(), + lookup_stores: Default::default(), }, } } diff --git a/tests/src/store/blob.rs b/tests/src/store/blob.rs index 4c1326f8..4feb5bb8 100644 --- a/tests/src/store/blob.rs +++ b/tests/src/store/blob.rs @@ -22,383 +22,338 @@ */ use store::{ - backend::rocksdb::RocksDbStore, + config::ConfigStore, write::{blob::BlobQuota, now, BatchBuilder, BlobOp, F_CLEAR}, - BlobClass, BlobHash, BlobStore, Store, + BlobClass, BlobHash, BlobStore, }; use utils::config::Config; -use crate::store::TempDir; - -const CONFIG_S3: &str = r#" -[store.blob.s3] -access-key = "minioadmin" -secret-key = "minioadmin" -region = "eu-central-1" -endpoint = "http://localhost:9000" -bucket = "tmp" -"#; - -const CONFIG_LOCAL: &str = r#" -[store.blob.local] -path = "{TMP}" -"#; - -const CONFIG_DB: &str = r#" -[store.db] -#path = "{TMP}/sqlite.db" -path = "{TMP}/rocksdb" -host = "localhost" -#port = 5432 -port = 3307 -database = "stalwart" -#user = "postgres" -#password = "mysecretpassword" -user = "root" -password = "password" - -"#; +use crate::store::{TempDir, CONFIG}; #[tokio::test] pub async fn blob_tests() { let temp_dir = TempDir::new("blob_tests", true); + let config = + Config::new(&CONFIG.replace("{TMP}", temp_dir.path.as_path().to_str().unwrap())).unwrap(); + let stores = config.parse_stores().await.unwrap(); - /*for (store_id, store_cfg) in [("s3", CONFIG_S3), ("fs", CONFIG_LOCAL)] { - let config = - Config::new(&store_cfg.replace("{TMP}", temp_dir.path.as_path().to_str().unwrap())) - .unwrap(); - - let blob_store_: BlobStore = match store_id { - "fs" => FsStore::open(&config).await.unwrap().into(), - "s3" => S3Store::open(&config).await.unwrap().into(), - _ => unreachable!(), - }; - + for (store_id, blob_store) in stores.blob_stores { println!("Testing blob store {}...", store_id); - test_store(blob_store_.clone()).await; - }*/ + test_store(blob_store).await; + } - // Init store - //let store: Store = SqliteStore::open( - //let store: Store = FdbStore::open( - //let store: Store = PostgresStore::open( - //let store: Store = MysqlStore::open( - let store: Store = RocksDbStore::open( - &Config::new(&CONFIG_DB.replace("{TMP}", temp_dir.path.as_path().to_str().unwrap())) - .unwrap(), - ) - .await - .unwrap() - .into(); - store.destroy().await; + for (store_id, store) in stores.stores { + println!("Testing blob management on store {}...", store_id); - // Test internal blob store - let blob_store: BlobStore = store.clone().into(); - test_store(blob_store.clone()).await; + // Init store + store.destroy().await; - // Blob hash exists - let hash = BlobHash::from(b"abc".as_slice()); - assert!(!store.blob_hash_exists(&hash).await.unwrap()); + // Test internal blob store + let blob_store: BlobStore = store.clone().into(); - // Reserve blob but mark it as expired - store - .write( - BatchBuilder::new() - .with_account_id(0) - .blob( - hash.clone(), - BlobOp::Reserve { - until: now() - 10, - size: 1024, - }, - 0, - ) - .build_batch(), - ) - .await - .unwrap(); + // Blob hash exists + let hash = BlobHash::from(b"abc".as_slice()); + assert!(!store.blob_hash_exists(&hash).await.unwrap()); - // Uncommitted blob, should not exist - assert!(!store.blob_hash_exists(&hash).await.unwrap()); - - // Write blob to store - blob_store.put_blob(hash.as_ref(), b"abc").await.unwrap(); - - // Commit blob - store - .write( - BatchBuilder::new() - .blob(hash.clone(), BlobOp::Commit, 0) - .build_batch(), - ) - .await - .unwrap(); - - // Blob hash should now exist - assert!(store.blob_hash_exists(&hash).await.unwrap()); - - // AccountId 0 should be able to read blob - assert!(store - .blob_hash_can_read(&hash, BlobClass::Reserved { account_id: 0 }) - .await - .unwrap()); - - // AccountId 1 should not be able to read blob - assert!(!store - .blob_hash_can_read(&hash, BlobClass::Reserved { account_id: 1 }) - .await - .unwrap()); - - // Blob already expired, quota should be 0 - assert_eq!( - store.blob_hash_quota(0).await.unwrap(), - BlobQuota { bytes: 0, count: 0 } - ); - - // Purge expired blobs - store.blob_hash_purge(blob_store.clone()).await.unwrap(); - - // Blob hash should no longer exist - assert!(!store.blob_hash_exists(&hash).await.unwrap()); - - // AccountId 0 should not be able to read blob - assert!(!store - .blob_hash_can_read(&hash, BlobClass::Reserved { account_id: 0 }) - .await - .unwrap()); - - // Blob should no longer be in store - assert!(blob_store - .get_blob(hash.as_ref(), 0..u32::MAX) - .await - .unwrap() - .is_none()); - - // Upload one linked blob to accountId 1, two linked blobs to accountId 0, and three unlinked (reserved) blobs to accountId 2 - for (document_id, (blob, blob_op)) in [ - (b"123", BlobOp::Link), - (b"456", BlobOp::Link), - (b"789", BlobOp::Link), - ( - b"abc", - BlobOp::Reserve { - until: now() - 10, - size: 5000, - }, - ), - ( - b"efg", - BlobOp::Reserve { - until: now() + 10, - size: 1000, - }, - ), - ( - b"hij", - BlobOp::Reserve { - until: now() + 10, - size: 2000, - }, - ), - ] - .into_iter() - .enumerate() - { - let hash = BlobHash::from(blob.as_slice()); + // Reserve blob but mark it as expired + store + .write( + BatchBuilder::new() + .with_account_id(0) + .blob( + hash.clone(), + BlobOp::Reserve { + until: now() - 10, + size: 1024, + }, + 0, + ) + .build_batch(), + ) + .await + .unwrap(); + + // Uncommitted blob, should not exist + assert!(!store.blob_hash_exists(&hash).await.unwrap()); + + // Write blob to store + blob_store.put_blob(hash.as_ref(), b"abc").await.unwrap(); + + // Commit blob store .write( BatchBuilder::new() - .with_account_id(if document_id > 0 { 0 } else { 1 }) - .with_collection(0) - .update_document(document_id as u32) - .blob(hash.clone(), blob_op, 0) .blob(hash.clone(), BlobOp::Commit, 0) .build_batch(), ) .await .unwrap(); - blob_store - .put_blob(hash.as_ref(), blob.as_slice()) + + // Blob hash should now exist + assert!(store.blob_hash_exists(&hash).await.unwrap()); + + // AccountId 0 should be able to read blob + assert!(store + .blob_hash_can_read(&hash, BlobClass::Reserved { account_id: 0 }) + .await + .unwrap()); + + // AccountId 1 should not be able to read blob + assert!(!store + .blob_hash_can_read(&hash, BlobClass::Reserved { account_id: 1 }) + .await + .unwrap()); + + // Blob already expired, quota should be 0 + assert_eq!( + store.blob_hash_quota(0).await.unwrap(), + BlobQuota { bytes: 0, count: 0 } + ); + + // Purge expired blobs + store.blob_hash_purge(blob_store.clone()).await.unwrap(); + + // Blob hash should no longer exist + assert!(!store.blob_hash_exists(&hash).await.unwrap()); + + // AccountId 0 should not be able to read blob + assert!(!store + .blob_hash_can_read(&hash, BlobClass::Reserved { account_id: 0 }) + .await + .unwrap()); + + // Blob should no longer be in store + assert!(blob_store + .get_blob(hash.as_ref(), 0..u32::MAX) + .await + .unwrap() + .is_none()); + + // Upload one linked blob to accountId 1, two linked blobs to accountId 0, and three unlinked (reserved) blobs to accountId 2 + for (document_id, (blob, blob_op)) in [ + (b"123", BlobOp::Link), + (b"456", BlobOp::Link), + (b"789", BlobOp::Link), + ( + b"abc", + BlobOp::Reserve { + until: now() - 10, + size: 5000, + }, + ), + ( + b"efg", + BlobOp::Reserve { + until: now() + 10, + size: 1000, + }, + ), + ( + b"hij", + BlobOp::Reserve { + until: now() + 10, + size: 2000, + }, + ), + ] + .into_iter() + .enumerate() + { + let hash = BlobHash::from(blob.as_slice()); + store + .write( + BatchBuilder::new() + .with_account_id(if document_id > 0 { 0 } else { 1 }) + .with_collection(0) + .update_document(document_id as u32) + .blob(hash.clone(), blob_op, 0) + .blob(hash.clone(), BlobOp::Commit, 0) + .build_batch(), + ) + .await + .unwrap(); + blob_store + .put_blob(hash.as_ref(), blob.as_slice()) + .await + .unwrap(); + } + + // One of the reserved blobs expired and should not count towards quota + assert_eq!( + store.blob_hash_quota(0).await.unwrap(), + BlobQuota { + bytes: 3000, + count: 2 + } + ); + assert_eq!( + store.blob_hash_quota(1).await.unwrap(), + BlobQuota { bytes: 0, count: 0 } + ); + + // Purge expired blobs and make sure nothing else is deleted + store.blob_hash_purge(blob_store.clone()).await.unwrap(); + for (pos, (blob, blob_class)) in [ + (b"abc", BlobClass::Reserved { account_id: 0 }), + ( + b"123", + BlobClass::Linked { + account_id: 1, + collection: 0, + document_id: 0, + }, + ), + ( + b"456", + BlobClass::Linked { + account_id: 0, + collection: 0, + document_id: 1, + }, + ), + ( + b"789", + BlobClass::Linked { + account_id: 0, + collection: 0, + document_id: 2, + }, + ), + (b"efg", BlobClass::Reserved { account_id: 0 }), + (b"hij", BlobClass::Reserved { account_id: 0 }), + ] + .into_iter() + .enumerate() + { + let ct = pos == 0; + let hash = BlobHash::from(blob.as_slice()); + assert!(store.blob_hash_can_read(&hash, blob_class).await.unwrap() ^ ct); + assert!(store.blob_hash_exists(&hash).await.unwrap() ^ ct); + assert!( + blob_store + .get_blob(hash.as_ref(), 0..u32::MAX) + .await + .unwrap() + .is_some() + ^ ct + ); + } + + // AccountId 0 should not have access to accountId 1's blobs + assert!(!store + .blob_hash_can_read( + BlobHash::from(b"123".as_slice()), + BlobClass::Linked { + account_id: 0, + collection: 0, + document_id: 0, + } + ) + .await + .unwrap()); + + // Unlink blob + store + .write( + BatchBuilder::new() + .with_account_id(0) + .with_collection(0) + .update_document(2) + .blob(BlobHash::from(b"789".as_slice()), BlobOp::Link, F_CLEAR) + .build_batch(), + ) .await .unwrap(); - } - // One of the reserved blobs expired and should not count towards quota - assert_eq!( - store.blob_hash_quota(0).await.unwrap(), - BlobQuota { - bytes: 3000, - count: 2 + // Purge and make sure blob is deleted + store.blob_hash_purge(blob_store.clone()).await.unwrap(); + for (pos, (blob, blob_class)) in [ + ( + b"789", + BlobClass::Linked { + account_id: 0, + collection: 0, + document_id: 2, + }, + ), + ( + b"123", + BlobClass::Linked { + account_id: 1, + collection: 0, + document_id: 0, + }, + ), + ( + b"456", + BlobClass::Linked { + account_id: 0, + collection: 0, + document_id: 1, + }, + ), + (b"efg", BlobClass::Reserved { account_id: 0 }), + (b"hij", BlobClass::Reserved { account_id: 0 }), + ] + .into_iter() + .enumerate() + { + let ct = pos == 0; + let hash = BlobHash::from(blob.as_slice()); + assert!(store.blob_hash_can_read(&hash, blob_class).await.unwrap() ^ ct); + assert!(store.blob_hash_exists(&hash).await.unwrap() ^ ct); + assert!( + blob_store + .get_blob(hash.as_ref(), 0..u32::MAX) + .await + .unwrap() + .is_some() + ^ ct + ); } - ); - assert_eq!( - store.blob_hash_quota(1).await.unwrap(), - BlobQuota { bytes: 0, count: 0 } - ); - // Purge expired blobs and make sure nothing else is deleted - store.blob_hash_purge(blob_store.clone()).await.unwrap(); - for (pos, (blob, blob_class)) in [ - (b"abc", BlobClass::Reserved { account_id: 0 }), - ( - b"123", - BlobClass::Linked { - account_id: 1, - collection: 0, - document_id: 0, - }, - ), - ( - b"456", - BlobClass::Linked { - account_id: 0, - collection: 0, - document_id: 1, - }, - ), - ( - b"789", - BlobClass::Linked { - account_id: 0, - collection: 0, - document_id: 2, - }, - ), - (b"efg", BlobClass::Reserved { account_id: 0 }), - (b"hij", BlobClass::Reserved { account_id: 0 }), - ] - .into_iter() - .enumerate() - { - let ct = pos == 0; - let hash = BlobHash::from(blob.as_slice()); - assert!(store.blob_hash_can_read(&hash, blob_class).await.unwrap() ^ ct); - assert!(store.blob_hash_exists(&hash).await.unwrap() ^ ct); - assert!( - blob_store - .get_blob(hash.as_ref(), 0..u32::MAX) - .await - .unwrap() - .is_some() - ^ ct - ); + // Unlink all blobs from accountId 1 and purge + store.blob_hash_unlink_account(1).await.unwrap(); + store.blob_hash_purge(blob_store.clone()).await.unwrap(); + + // Make sure only accountId 0's blobs are left + for (pos, (blob, blob_class)) in [ + ( + b"123", + BlobClass::Linked { + account_id: 1, + collection: 0, + document_id: 0, + }, + ), + ( + b"456", + BlobClass::Linked { + account_id: 0, + collection: 0, + document_id: 1, + }, + ), + (b"efg", BlobClass::Reserved { account_id: 0 }), + (b"hij", BlobClass::Reserved { account_id: 0 }), + ] + .into_iter() + .enumerate() + { + let ct = pos == 0; + let hash = BlobHash::from(blob.as_slice()); + assert!(store.blob_hash_can_read(&hash, blob_class).await.unwrap() ^ ct); + assert!(store.blob_hash_exists(&hash).await.unwrap() ^ ct); + assert!( + blob_store + .get_blob(hash.as_ref(), 0..u32::MAX) + .await + .unwrap() + .is_some() + ^ ct + ); + } } - - // AccountId 0 should not have access to accountId 1's blobs - assert!(!store - .blob_hash_can_read( - BlobHash::from(b"123".as_slice()), - BlobClass::Linked { - account_id: 0, - collection: 0, - document_id: 0, - } - ) - .await - .unwrap()); - - // Unlink blob - store - .write( - BatchBuilder::new() - .with_account_id(0) - .with_collection(0) - .update_document(2) - .blob(BlobHash::from(b"789".as_slice()), BlobOp::Link, F_CLEAR) - .build_batch(), - ) - .await - .unwrap(); - - // Purge and make sure blob is deleted - store.blob_hash_purge(blob_store.clone()).await.unwrap(); - for (pos, (blob, blob_class)) in [ - ( - b"789", - BlobClass::Linked { - account_id: 0, - collection: 0, - document_id: 2, - }, - ), - ( - b"123", - BlobClass::Linked { - account_id: 1, - collection: 0, - document_id: 0, - }, - ), - ( - b"456", - BlobClass::Linked { - account_id: 0, - collection: 0, - document_id: 1, - }, - ), - (b"efg", BlobClass::Reserved { account_id: 0 }), - (b"hij", BlobClass::Reserved { account_id: 0 }), - ] - .into_iter() - .enumerate() - { - let ct = pos == 0; - let hash = BlobHash::from(blob.as_slice()); - assert!(store.blob_hash_can_read(&hash, blob_class).await.unwrap() ^ ct); - assert!(store.blob_hash_exists(&hash).await.unwrap() ^ ct); - assert!( - blob_store - .get_blob(hash.as_ref(), 0..u32::MAX) - .await - .unwrap() - .is_some() - ^ ct - ); - } - - // Unlink all blobs from accountId 1 and purge - store.blob_hash_unlink_account(1).await.unwrap(); - store.blob_hash_purge(blob_store.clone()).await.unwrap(); - - // Make sure only accountId 0's blobs are left - for (pos, (blob, blob_class)) in [ - ( - b"123", - BlobClass::Linked { - account_id: 1, - collection: 0, - document_id: 0, - }, - ), - ( - b"456", - BlobClass::Linked { - account_id: 0, - collection: 0, - document_id: 1, - }, - ), - (b"efg", BlobClass::Reserved { account_id: 0 }), - (b"hij", BlobClass::Reserved { account_id: 0 }), - ] - .into_iter() - .enumerate() - { - let ct = pos == 0; - let hash = BlobHash::from(blob.as_slice()); - assert!(store.blob_hash_can_read(&hash, blob_class).await.unwrap() ^ ct); - assert!(store.blob_hash_exists(&hash).await.unwrap() ^ ct); - assert!( - blob_store - .get_blob(hash.as_ref(), 0..u32::MAX) - .await - .unwrap() - .is_some() - ^ ct - ); - } - temp_dir.delete(); } diff --git a/tests/src/store/mod.rs b/tests/src/store/mod.rs index f3a370e8..06f4c42f 100644 --- a/tests/src/store/mod.rs +++ b/tests/src/store/mod.rs @@ -27,12 +27,7 @@ pub mod query; use std::io::Read; -use ::store::Store; - -use store::{ - backend::{elastic::ElasticSearchStore, rocksdb::RocksDbStore}, - FtsStore, -}; +use store::{config::ConfigStore, FtsStore}; use utils::config::Config; pub struct TempDir { @@ -40,49 +35,62 @@ pub struct TempDir { } const CONFIG: &str = r#" -[store.blob] -type = "local" -local.path = "{TMP}" +[store."s3"] +type = "s3" +access-key = "minioadmin" +secret-key = "minioadmin" +region = "eu-central-1" +endpoint = "http://localhost:9000" +bucket = "tmp" -[store.db] +[store."fs"] +type = "fs" +path = "{TMP}" + +[store."rocksdb"] +type = "rocksdb" +path = "{TMP}/rocksdb" + +[store."foundationdb"] +type = "foundationdb" + +[store."sqlite"] +type = "sqlite" path = "{TMP}/sqlite.db" + +[store."postgresql"] +type = "postgresql" +host = "localhost" +port = 5432 +database = "stalwart" +user = "postgres" +password = "mysecretpassword" + +[store."mysql"] +type = "mysql" host = "localhost" -#port = 5432 port = 3307 database = "stalwart" -#user = "postgres" -#password = "mysecretpassword" user = "root" password = "password" - -[store.fts] -url = "https://localhost:9200" -user = "elastic" -password = "RtQ-Lu6+o4rxx=XJplVJ" -allow-invalid-certs = true - "#; #[tokio::test] pub async fn store_tests() { - //let insert = true; - let insert = false; + let insert = true; let temp_dir = TempDir::new("store_tests", insert); - let config_file = CONFIG.replace("{TMP}", &temp_dir.path.to_string_lossy()); - let config = Config::new(&config_file).unwrap(); - //let db: Store = SqliteStore::open(&Config::new(&config_file).unwrap()) - //let db: Store = FdbStore::open(&Config::new(&config_file).unwrap()) - //let db: Store = PostgresStore::open(&Config::new(&config_file).unwrap()) - //let db: Store = MysqlStore::open(&Config::new(&config_file).unwrap()) - let db: Store = RocksDbStore::open(&config).await.unwrap().into(); - let fts_store = FtsStore::from(db.clone()); - //let fts_store = ElasticSearchStore::open(&config).await.unwrap().into(); + let config = Config::new(&CONFIG.replace("{TMP}", &temp_dir.path.to_string_lossy())).unwrap(); + let stores = config.parse_stores().await.unwrap(); - if insert { - db.destroy().await; + for (store_id, store) in stores.stores { + println!("Testing store {}...", store_id); + if insert { + store.destroy().await; + } + query::test(store.clone(), FtsStore::Store(store.clone()), insert).await; + assign_id::test(store).await; } - query::test(db.clone(), fts_store, insert).await; - assign_id::test(db).await; + if insert { temp_dir.delete(); }