From 262aed19af4224bfa68f63598d7570adf2316586 Mon Sep 17 00:00:00 2001 From: mdecimus <11444311+mdecimus@users.noreply.github.com> Date: Sun, 1 Feb 2026 19:02:09 +0100 Subject: [PATCH] Bootstrap from registry - part 6 --- Cargo.lock | 831 +++++++++--------- crates/common/src/addresses.rs | 65 +- crates/common/src/auth/access_token.rs | 4 +- crates/common/src/auth/mod.rs | 8 +- crates/common/src/auth/oauth/config.rs | 216 ++--- crates/common/src/auth/oauth/token.rs | 2 +- crates/common/src/auth/rate_limit.rs | 4 +- crates/common/src/config/inner.rs | 4 +- crates/common/src/config/mod.rs | 12 +- crates/common/src/config/smtp/auth.rs | 7 - crates/common/src/config/telemetry.rs | 2 - crates/common/src/core.rs | 36 +- crates/common/src/enterprise/config.rs | 2 +- crates/common/src/lib.rs | 67 +- crates/common/src/listener/blocked.rs | 282 +++--- crates/common/src/manager/boot.rs | 9 +- crates/common/src/manager/mod.rs | 37 - crates/common/src/manager/reload.rs | 22 +- crates/common/src/manager/webadmin.rs | 5 +- crates/http-proto/Cargo.toml | 1 + crates/http-proto/src/context.rs | 5 +- crates/nlp/src/language/mod.rs | 2 +- crates/registry/src/schema/mod.rs | 47 +- crates/registry/src/types/datetime.rs | 12 +- crates/registry/src/types/duration.rs | 10 + crates/registry/src/types/error.rs | 13 +- crates/registry/src/types/ipmask.rs | 28 + crates/store/src/backend/azure/mod.rs | 82 +- .../src/backend/composite/read_replica.rs | 117 +-- .../src/backend/composite/sharded_blob.rs | 124 +-- .../src/backend/composite/sharded_lookup.rs | 64 +- crates/store/src/backend/elastic/main.rs | 79 +- crates/store/src/backend/elastic/mod.rs | 3 + crates/store/src/backend/foundationdb/main.rs | 112 +-- crates/store/src/backend/fs/mod.rs | 34 +- crates/store/src/backend/http/config.rs | 116 +-- crates/store/src/backend/meili/main.rs | 48 +- crates/store/src/backend/memory/mod.rs | 118 +-- crates/store/src/backend/mysql/main.rs | 112 +-- crates/store/src/backend/postgres/main.rs | 110 ++- crates/store/src/backend/postgres/mod.rs | 3 - crates/store/src/backend/postgres/search.rs | 13 +- crates/store/src/backend/redis/mod.rs | 210 ++--- crates/store/src/backend/rocksdb/main.rs | 61 +- crates/store/src/backend/s3/mod.rs | 131 +-- crates/store/src/backend/sqlite/main.rs | 67 +- crates/store/src/backend/sqlite/mod.rs | 6 +- crates/store/src/bootstrap/blob.rs | 60 ++ crates/store/src/bootstrap/data.rs | 31 + crates/store/src/bootstrap/lookup.rs | 76 ++ crates/store/src/bootstrap/memory.rs | 42 + crates/store/src/bootstrap/mod.rs | 11 + crates/store/src/bootstrap/search.rs | 54 ++ crates/store/src/config.rs | 467 ---------- crates/store/src/dispatch/blob.rs | 171 ++-- crates/store/src/dispatch/lookup.rs | 27 +- crates/store/src/dispatch/registry.rs | 6 +- crates/store/src/dispatch/search.rs | 29 +- crates/store/src/dispatch/store.rs | 18 + crates/store/src/lib.rs | 89 +- .../src/registry}/bootstrap.rs | 49 +- crates/store/src/registry/mod.rs | 2 + crates/trc/src/event/enums.rs | 4 +- crates/trc/src/event/enums_impl.rs | 22 + crates/utils/src/glob.rs | 50 +- crates/utils/src/lib.rs | 1 + tests/Cargo.toml | 4 +- 67 files changed, 1976 insertions(+), 2580 deletions(-) create mode 100644 crates/store/src/bootstrap/blob.rs create mode 100644 crates/store/src/bootstrap/data.rs create mode 100644 crates/store/src/bootstrap/lookup.rs create mode 100644 crates/store/src/bootstrap/memory.rs create mode 100644 crates/store/src/bootstrap/mod.rs create mode 100644 crates/store/src/bootstrap/search.rs delete mode 100644 crates/store/src/config.rs rename crates/{common/src/manager => store/src/registry}/bootstrap.rs (68%) diff --git a/Cargo.lock b/Cargo.lock index 285c79f5..a455f4e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -193,7 +193,7 @@ dependencies = [ "nom", "num-traits", "rusticata-macros", - "thiserror 2.0.17", + "thiserror 2.0.18", "time", ] @@ -205,7 +205,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", "synstructure", ] @@ -217,7 +217,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -287,7 +287,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -309,7 +309,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -320,7 +320,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -337,7 +337,7 @@ checksum = "07a9b245ba0739fc90935094c29adbaee3f977218b5fb95e822e261cda7f56a3" dependencies = [ "http 1.4.0", "log", - "rustls 0.23.35", + "rustls 0.23.36", "serde", "serde_json", "url", @@ -387,7 +387,7 @@ dependencies = [ "bytes", "dyn-clone", "futures", - "getrandom 0.2.16", + "getrandom 0.2.17", "hmac 0.12.1", "http-types", "once_cell", @@ -509,9 +509,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "base64ct" -version = "1.8.1" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e050f626429857a27ddccb31e0aca21356bfa709c04041aefddac081a8f068a" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] name = "bincode" @@ -559,7 +559,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -577,7 +577,7 @@ dependencies = [ "regex", "rustc-hash 2.1.1", "shlex", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -637,9 +637,9 @@ dependencies = [ [[package]] name = "bitpacking" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c1d3e2bfd8d06048a179f7b17afc3188effa10385e7b00dc65af6aae732ea92" +checksum = "96a7139abd3d9cebf8cd6f920a389cf3dc9576172e32f4563f188cae3c3eb019" dependencies = [ "crunchy", ] @@ -677,15 +677,16 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.2" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" +checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", - "constant_time_eq", + "constant_time_eq 0.4.2", + "cpufeatures", ] [[package]] @@ -780,7 +781,7 @@ checksum = "89385e82b5d1821d2219e0b095efa2cc1f246cbf99080f3be46a1a85c0d392d9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -896,9 +897,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.50" +version = "1.2.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f50d563227a1c37cc0a263f64eca3334388c01c5e4c4861a9def205c614383c" +checksum = "6354c81bbfd62d9cfa9cb3c773c2b7b2a3a482d569de977fd0e961f6e7c00583" dependencies = [ "find-msvc-tools", "jobserver", @@ -964,9 +965,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.42" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" dependencies = [ "iana-time-zone", "js-sys", @@ -1077,7 +1078,7 @@ dependencies = [ "idna", "imagesize", "imap_proto", - "indexmap 2.12.1", + "indexmap 2.13.0", "infer 0.19.0", "jmap_proto", "libc", @@ -1109,7 +1110,7 @@ dependencies = [ "ring", "rkyv", "rsa", - "rustls 0.23.35", + "rustls 0.23.36", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", @@ -1178,7 +1179,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", "once_cell", "tiny-keccak", ] @@ -1218,6 +1219,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "coordinator" version = "0.15.4" @@ -1462,7 +1469,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -1496,7 +1503,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -1510,7 +1517,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -1521,7 +1528,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -1532,7 +1539,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core 0.21.3", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -1543,9 +1550,9 @@ checksum = "06d2e3287df1c007e74221c49ca10a95d557349e54b3a75dc2fb14712c751f04" [[package]] name = "data-encoding" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" +checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" [[package]] name = "dav" @@ -1628,7 +1635,7 @@ checksum = "3d697d376cbfa018c23eb4caab1fd1883dd9c906a8c034e8d9a3cb06a7e0bef9" dependencies = [ "async-trait", "deadpool 0.12.3", - "getrandom 0.2.16", + "getrandom 0.2.17", "tokio", "tokio-postgres", "tracing", @@ -1702,7 +1709,7 @@ checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -1759,7 +1766,7 @@ dependencies = [ "regex", "reqwest", "rkyv", - "rustls 0.23.35", + "rustls 0.23.36", "rustls-pki-types", "scrypt", "serde", @@ -1826,7 +1833,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -2056,7 +2063,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -2108,7 +2115,7 @@ version = "0.15.4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -2131,9 +2138,9 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] name = "fancy-regex" -version = "0.14.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298" +checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8" dependencies = [ "bit-set 0.8.0", "regex-automata", @@ -2154,9 +2161,9 @@ checksum = "95765f67b4b18863968b4a1bd5bb576f732b29a4a28c7cd84c09fa3e2875f33c" [[package]] name = "fastbloom" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18c1ddb9231d8554c2d6bdf4cfaabf0c59251658c68b6c95cd52dd0c513a912a" +checksum = "4e7f34442dbe69c60fe8eaf58a8cafff81a1f278816d8ab4db255b3bef4ac3c4" dependencies = [ "getrandom 0.3.4", "libm", @@ -2197,9 +2204,9 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "find-msvc-tools" -version = "0.1.5" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" +checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db" [[package]] name = "fixedbitset" @@ -2215,14 +2222,14 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flate2" -version = "1.1.5" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" +checksum = "b375d6465b98090a5f25b1c7703f3859783755aa9a80433b36e0379a3ec2f369" dependencies = [ "crc32fast", - "libz-rs-sys", "libz-sys", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -2282,7 +2289,7 @@ dependencies = [ "memchr", "mime", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", "tracing", ] @@ -2334,7 +2341,7 @@ checksum = "c4ca5370149145ec3741cd7e82832f17f893b9421ee4e484d9511c6702bd9911" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", "try_map", ] @@ -2435,7 +2442,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -2512,9 +2519,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", "js-sys", @@ -2570,7 +2577,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -2623,7 +2630,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.12.1", + "indexmap 2.13.0", "slab", "tokio", "tokio-util", @@ -2632,9 +2639,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" dependencies = [ "atomic-waker", "bytes", @@ -2642,7 +2649,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.4.0", - "indexmap 2.12.1", + "indexmap 2.13.0", "slab", "tokio", "tokio-util", @@ -2695,7 +2702,7 @@ checksum = "149e3ea90eb5a26ad354cfe3cb7f7401b9329032d0235f2687d03a35f30e5d4c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -2802,9 +2809,9 @@ dependencies = [ "once_cell", "rand 0.9.2", "ring", - "rustls 0.23.35", + "rustls 0.23.36", "rustls-pki-types", - "thiserror 2.0.17", + "thiserror 2.0.18", "time", "tinyvec", "tokio", @@ -2828,9 +2835,9 @@ dependencies = [ "parking_lot", "rand 0.9.2", "resolv-conf", - "rustls 0.23.35", + "rustls 0.23.36", "smallvec", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-rustls 0.26.4", "tracing", @@ -3016,6 +3023,7 @@ dependencies = [ "hyper 1.8.1", "hyper-util", "percent-encoding", + "registry", "serde", "serde_json", "trc", @@ -3072,7 +3080,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2 0.4.12", + "h2 0.4.13", "http 1.4.0", "http-body 1.0.1", "httparse", @@ -3108,12 +3116,12 @@ dependencies = [ "http 1.4.0", "hyper 1.8.1", "hyper-util", - "rustls 0.23.35", + "rustls 0.23.36", "rustls-pki-types", "tokio", "tokio-rustls 0.26.4", "tower-service", - "webpki-roots 1.0.4", + "webpki-roots 1.0.5", ] [[package]] @@ -3147,7 +3155,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.1", + "socket2 0.6.2", "tokio", "tower-service", "tracing", @@ -3155,9 +3163,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.64" +version = "0.1.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -3310,14 +3318,14 @@ dependencies = [ "directory", "email", "imap_proto", - "indexmap 2.12.1", + "indexmap 2.13.0", "mail-parser", "mail-send", "md5 0.8.0", "nlp", "parking_lot", "rand 0.9.2", - "rustls 0.23.35", + "rustls 0.23.36", "rustls-pemfile 2.2.0", "store", "tokio", @@ -3365,7 +3373,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", "zstd", ] @@ -3392,9 +3400,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.12.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ "equivalent", "hashbrown 0.16.1", @@ -3465,9 +3473,9 @@ dependencies = [ [[package]] name = "iri-string" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f867b9d1d896b67beb18518eda36fdb77a32ea590de864f1325b294a6d14397" +checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" dependencies = [ "memchr", "serde", @@ -3511,9 +3519,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ee5b5339afb4c41626dde77b7a611bd4f2c202b897852b4bcf5d03eddc61010" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" [[package]] name = "jemalloc-sys" @@ -3626,7 +3634,7 @@ dependencies = [ "maybe-async", "parking_lot", "reqwest", - "rustls 0.23.35", + "rustls 0.23.36", "rustls-pki-types", "serde", "serde_json", @@ -3699,9 +3707,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.83" +version = "0.3.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" dependencies = [ "once_cell", "wasm-bindgen", @@ -3742,7 +3750,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d" dependencies = [ - "indexmap 2.12.1", + "indexmap 2.13.0", ] [[package]] @@ -3828,9 +3836,9 @@ dependencies = [ "log", "nom", "percent-encoding", - "rustls 0.23.35", - "rustls-native-certs 0.8.2", - "thiserror 2.0.17", + "rustls 0.23.36", + "rustls-native-certs 0.8.3", + "thiserror 2.0.18", "tokio", "tokio-rustls 0.26.4", "tokio-stream", @@ -3847,9 +3855,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.178" +version = "0.2.180" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" +checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" [[package]] name = "libflate" @@ -3887,19 +3895,18 @@ dependencies = [ [[package]] name = "libm" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df15f6eac291ed1cf25865b1ee60399f57e7c227e7f51bdbd4c5270396a9ed50" +checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" dependencies = [ "bitflags", "libc", - "redox_syscall 0.6.0", ] [[package]] @@ -3928,15 +3935,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "libz-rs-sys" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c10501e7805cee23da17c7790e59df2870c0d4043ec6d03f67d31e2b53e77415" -dependencies = [ - "zlib-rs", -] - [[package]] name = "libz-sys" version = "1.1.23" @@ -4098,12 +4096,12 @@ dependencies = [ "base64 0.22.1", "gethostname", "md5 0.8.0", - "rustls 0.23.35", + "rustls 0.23.36", "rustls-pki-types", "smtp-proto", "tokio", "tokio-rustls 0.26.4", - "webpki-roots 1.0.4", + "webpki-roots 1.0.5", ] [[package]] @@ -4123,7 +4121,7 @@ dependencies = [ "md5 0.8.0", "parking_lot", "rkyv", - "rustls 0.23.35", + "rustls 0.23.36", "rustls-pemfile 2.2.0", "sieve-rs", "store", @@ -4157,7 +4155,7 @@ checksum = "5cf92c10c7e361d6b99666ec1c6f9805b0bea2c3bd8c78dc6fe98ac5bd78db11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -4294,9 +4292,9 @@ dependencies = [ [[package]] name = "moka" -version = "0.12.12" +version = "0.12.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3dec6bd31b08944e08b58fd99373893a6c17054d6f3ea5006cc894f4f4eee2a" +checksum = "b4ac832c50ced444ef6be0767a008b02c106a909ba79d1d830501e94b96f6b7e" dependencies = [ "crossbeam-channel", "crossbeam-epoch", @@ -4326,7 +4324,7 @@ checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -4342,9 +4340,9 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", "termcolor", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -4365,12 +4363,12 @@ dependencies = [ "pem", "percent-encoding", "rand 0.9.2", - "rustls 0.23.35", + "rustls 0.23.36", "rustls-pemfile 2.2.0", "serde", "serde_json", "socket2 0.5.10", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-rustls 0.26.4", "tokio-util", @@ -4402,7 +4400,7 @@ dependencies = [ "serde_json", "sha1", "sha2 0.10.9", - "thiserror 2.0.17", + "thiserror 2.0.18", "uuid", ] @@ -4412,7 +4410,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", ] [[package]] @@ -4451,7 +4449,7 @@ dependencies = [ "data-encoding", "ed25519", "ed25519-dalek", - "getrandom 0.2.16", + "getrandom 0.2.17", "log", "rand 0.8.5", "signatory", @@ -4563,9 +4561,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" [[package]] name = "num-integer" @@ -4626,7 +4624,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -4698,7 +4696,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -4707,6 +4705,12 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "openssl-sys" version = "0.9.111" @@ -4729,7 +4733,7 @@ dependencies = [ "futures-sink", "js-sys", "pin-project-lite", - "thiserror 2.0.17", + "thiserror 2.0.18", "tracing", ] @@ -4761,7 +4765,7 @@ dependencies = [ "opentelemetry_sdk", "prost", "reqwest", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tonic", "tracing", @@ -4799,7 +4803,7 @@ dependencies = [ "percent-encoding", "rand 0.9.2", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tracing", ] @@ -4881,7 +4885,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.18", + "redox_syscall", "smallvec", "windows-link", ] @@ -4942,9 +4946,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.4" +version = "2.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbcfd20a6d4eeba40179f05735784ad32bdaef05ce8e8af05f180d45bb3e7e22" +checksum = "2c9eb05c21a464ea704b53158d358a31e6425db2f63a1a7312268b05fe2b75f7" dependencies = [ "memchr", "ucd-trie", @@ -4952,9 +4956,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.8.4" +version = "2.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51f72981ade67b1ca6adc26ec221be9f463f2b5839c7508998daa17c23d94d7f" +checksum = "68f9dbced329c441fa79d80472764b1a2c7e57123553b8519b36663a2fb234ed" dependencies = [ "pest", "pest_generator", @@ -4962,22 +4966,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.8.4" +version = "2.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee9efd8cdb50d719a80088b76f81aec7c41ed6d522ee750178f83883d271625" +checksum = "3bb96d5051a78f44f43c8f712d8e810adb0ebf923fc9ed2655a7f66f63ba8ee5" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] name = "pest_meta" -version = "2.8.4" +version = "2.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf1d70880e76bdc13ba52eafa6239ce793d85c8e43896507e43dd8984ff05b82" +checksum = "602113b5b5e8621770cfd490cfd90b9f84ab29bd2b0e49ad83eb6d186cef2365" dependencies = [ "pest", "sha2 0.10.9", @@ -4990,7 +4994,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset 0.4.2", - "indexmap 2.12.1", + "indexmap 2.13.0", ] [[package]] @@ -5001,7 +5005,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset 0.5.7", "hashbrown 0.15.5", - "indexmap 2.12.1", + "indexmap 2.13.0", "serde", ] @@ -5055,7 +5059,7 @@ dependencies = [ "phf_shared 0.13.1", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -5102,7 +5106,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -5198,7 +5202,7 @@ dependencies = [ "imap", "mail-parser", "mail-send", - "rustls 0.23.35", + "rustls 0.23.36", "store", "tokio", "tokio-rustls 0.26.4", @@ -5209,15 +5213,15 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f59e70c4aef1e55797c2e8fd94a4f2a973fc972cfde0e0b05f683667b0cd39dd" +checksum = "f89776e4d69bb58bc6993e99ffa1d11f228b839984854c7daeb5d37f87cbe950" [[package]] name = "postgres-protocol" -version = "0.6.9" +version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbef655056b916eb868048276cfd5d6a7dea4f81560dfd047f97c8c6fe3fcfd4" +checksum = "3ee9dd5fe15055d2b6806f4736aa0c9637217074e224bbec46d4041b91bb9491" dependencies = [ "base64 0.22.1", "byteorder", @@ -5233,9 +5237,9 @@ dependencies = [ [[package]] name = "postgres-types" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef4605b7c057056dd35baeb6ac0c0338e4975b1f2bef0f65da953285eb007095" +checksum = "54b858f82211e84682fecd373f68e1ceae642d8d751a1ebd13f33de6257b3e20" dependencies = [ "bytes", "fallible-iterator 0.2.0", @@ -5261,9 +5265,9 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppmd-rust" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d558c559f0450f16f2a27a1f017ef38468c1090c9ce63c8e51366232d53717b4" +checksum = "efca4c95a19a79d1c98f791f10aebd5c1363b473244630bb7dbde1dc98455a24" [[package]] name = "ppv-lite86" @@ -5287,7 +5291,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -5361,14 +5365,14 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] name = "proc-macro2" -version = "1.0.103" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] @@ -5379,7 +5383,7 @@ version = "0.15.4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -5393,7 +5397,7 @@ dependencies = [ "lazy_static", "memchr", "parking_lot", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -5416,7 +5420,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -5431,9 +5435,9 @@ dependencies = [ [[package]] name = "psl" -version = "2.1.174" +version = "2.1.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a72fc75a5577d37dc9e161f7ed3b27cda8091f254367370c988a41fe2683696" +checksum = "f4171658a0c9db483caa2dbd02dd4ade6574a23d7e3ff3483646f9b5e9dbd5b0" dependencies = [ "psl-types", ] @@ -5461,7 +5465,7 @@ checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -5538,9 +5542,9 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.1", - "rustls 0.23.35", - "socket2 0.6.1", - "thiserror 2.0.17", + "rustls 0.23.36", + "socket2 0.6.2", + "thiserror 2.0.18", "tokio", "tracing", "web-time", @@ -5559,11 +5563,11 @@ dependencies = [ "rand 0.9.2", "ring", "rustc-hash 2.1.1", - "rustls 0.23.35", + "rustls 0.23.36", "rustls-pki-types", "rustls-platform-verifier", "slab", - "thiserror 2.0.17", + "thiserror 2.0.18", "tinyvec", "tracing", "web-time", @@ -5578,16 +5582,16 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.1", + "socket2 0.6.2", "tracing", "windows-sys 0.60.2", ] [[package]] name = "quote" -version = "1.0.42" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" dependencies = [ "proc-macro2", ] @@ -5665,7 +5669,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ "rand_chacha 0.9.0", - "rand_core 0.9.3", + "rand_core 0.9.5", ] [[package]] @@ -5695,7 +5699,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.3", + "rand_core 0.9.5", ] [[package]] @@ -5713,14 +5717,14 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", ] [[package]] name = "rand_core" -version = "0.9.3" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" dependencies = [ "getrandom 0.3.4", ] @@ -5824,9 +5828,9 @@ dependencies = [ [[package]] name = "rcgen" -version = "0.14.6" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ec0a99f2de91c3cddc84b37e7db80e4d96b743e05607f647eb236fc0455907f" +checksum = "10b99e0098aa4082912d4c649628623db6aba77335e4f4569ff5083a6448b32e" dependencies = [ "pem", "ring", @@ -5856,9 +5860,9 @@ dependencies = [ [[package]] name = "rdkafka-sys" -version = "4.9.0+2.10.0" +version = "4.10.0+2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5230dca48bc354d718269f3e4353280e188b610f7af7e2fcf54b7a79d5802872" +checksum = "e234cf318915c1059d4921ef7f75616b5219b10b46e9f3a511a15eb4b56a3f77" dependencies = [ "cmake", "libc", @@ -5885,16 +5889,16 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rand 0.9.2", - "rustls 0.23.35", - "rustls-native-certs 0.8.2", + "rustls 0.23.36", + "rustls-native-certs 0.8.3", "ryu", "sha1_smol", - "socket2 0.6.1", + "socket2 0.6.2", "tokio", "tokio-rustls 0.26.4", "tokio-util", "url", - "webpki-roots 1.0.4", + "webpki-roots 1.0.5", ] [[package]] @@ -5906,22 +5910,13 @@ dependencies = [ "bitflags", ] -[[package]] -name = "redox_syscall" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec96166dafa0886eb81fe1c0a388bece180fbef2135f97c1e2cf8302e74b43b5" -dependencies = [ - "bitflags", -] - [[package]] name = "redox_users" version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", "libredox", "thiserror 1.0.69", ] @@ -5932,9 +5927,9 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", "libredox", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -5954,7 +5949,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -6018,7 +6013,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.4.12", + "h2 0.4.13", "http 1.4.0", "http-body 1.0.1", "http-body-util", @@ -6031,7 +6026,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.35", + "rustls 0.23.36", "rustls-pki-types", "serde", "serde_json", @@ -6040,7 +6035,7 @@ dependencies = [ "tokio", "tokio-rustls 0.26.4", "tokio-util", - "tower 0.5.2", + "tower 0.5.3", "tower-http", "tower-service", "url", @@ -6048,7 +6043,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.4", + "webpki-roots 1.0.5", ] [[package]] @@ -6084,7 +6079,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.16", + "getrandom 0.2.17", "libc", "untrusted", "windows-sys 0.52.0", @@ -6111,14 +6106,14 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.8.12" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35a640b26f007713818e9a9b65d34da1cf58538207b052916a83d80e43f3ffa4" +checksum = "360b333c61ae24e5af3ae7c8660bd6b21ccd8200dbbc5d33c2454421e85b9c69" dependencies = [ "bytecheck", "bytes", - "hashbrown 0.15.5", - "indexmap 2.12.1", + "hashbrown 0.16.1", + "indexmap 2.13.0", "munge", "ptr_meta", "rancor", @@ -6130,13 +6125,13 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.8.12" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd83f5f173ff41e00337d97f6572e416d022ef8a19f371817259ae960324c482" +checksum = "7c02f8cdd12b307ab69fe0acf4cd2249c7460eb89dce64a0febadf934ebb6a9e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -6181,9 +6176,9 @@ dependencies = [ [[package]] name = "rsa" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40a0376c50d0358279d9d643e4bf7b7be212f1f4ff1da9070a7b54d22ef75c88" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ "const-oid", "digest 0.10.7", @@ -6278,9 +6273,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" [[package]] name = "rustc-hash" @@ -6339,15 +6334,15 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.35" +version = "0.23.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" +checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" dependencies = [ "log", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.8", + "rustls-webpki 0.103.9", "subtle", "zeroize", ] @@ -6358,7 +6353,7 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" dependencies = [ - "openssl-probe", + "openssl-probe 0.1.6", "rustls-pemfile 1.0.4", "schannel", "security-framework 2.11.1", @@ -6370,7 +6365,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" dependencies = [ - "openssl-probe", + "openssl-probe 0.1.6", "rustls-pemfile 2.2.0", "rustls-pki-types", "schannel", @@ -6379,11 +6374,11 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" dependencies = [ - "openssl-probe", + "openssl-probe 0.2.1", "rustls-pki-types", "schannel", "security-framework 3.5.1", @@ -6409,9 +6404,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.13.2" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21e6f2ab2928ca4291b86736a8bd920a277a399bba1589409d72154ff87c1282" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" dependencies = [ "web-time", "zeroize", @@ -6428,10 +6423,10 @@ dependencies = [ "jni", "log", "once_cell", - "rustls 0.23.35", - "rustls-native-certs 0.8.2", + "rustls 0.23.36", + "rustls-native-certs 0.8.3", "rustls-platform-verifier-android", - "rustls-webpki 0.103.8", + "rustls-webpki 0.103.9", "security-framework 3.5.1", "security-framework-sys", "webpki-root-certs", @@ -6466,9 +6461,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.8" +version = "0.103.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" +checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" dependencies = [ "ring", "rustls-pki-types", @@ -6483,9 +6478,9 @@ checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "ryu" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62049b2877bf12821e8f9ad256ee38fdc31db7387ec2d3b3f403024de2034aea" +checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" [[package]] name = "salsa20" @@ -6552,9 +6547,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289" +checksum = "54e910108742c57a770f492731f99be216a52fadd361b06c8fb59d74ccc267d2" dependencies = [ "dyn-clone", "either", @@ -6566,14 +6561,14 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301858a4023d78debd2353c7426dc486001bddc91ae31a76fb1f55132f7e2633" +checksum = "4908ad288c5035a8eb12cfdf0d49270def0a268ee162b75eeee0f85d155a7c45" dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -6705,7 +6700,7 @@ dependencies = [ "ecdsa", "ed25519", "ed25519-dalek", - "getrandom 0.2.16", + "getrandom 0.2.17", "hkdf", "idea", "idna", @@ -6728,7 +6723,7 @@ dependencies = [ "sha1collisiondetection", "sha2 0.10.9", "sha3", - "thiserror 2.0.17", + "thiserror 2.0.18", "twofish", "typenum", "x25519-dalek", @@ -6772,7 +6767,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -6783,16 +6778,16 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] name = "serde_json" -version = "1.0.147" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6af14725505314343e673e9ecb7cd7e8a36aa9791eb936235a3567cc31447ae4" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ - "indexmap 2.12.1", + "indexmap 2.13.0", "itoa", "memchr", "serde", @@ -6828,7 +6823,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -6853,9 +6848,9 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.12.1", + "indexmap 2.13.0", "schemars 0.9.0", - "schemars 1.1.0", + "schemars 1.2.0", "serde_core", "serde_json", "serde_with_macros", @@ -6871,7 +6866,7 @@ dependencies = [ "darling 0.21.3", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -6880,7 +6875,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.12.1", + "indexmap 2.13.0", "itoa", "ryu", "serde", @@ -6889,11 +6884,12 @@ dependencies = [ [[package]] name = "serial_test" -version = "3.2.0" +version = "3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b258109f244e1d6891bf1053a55d63a5cd4f8f4c30cf9a1280989f80e7a1fa9" +checksum = "0d0b343e184fc3b7bb44dff0705fffcf4b3756ba6aff420dddd8b24ca145e555" dependencies = [ - "futures", + "futures-executor", + "futures-util", "log", "once_cell", "parking_lot", @@ -6903,13 +6899,13 @@ dependencies = [ [[package]] name = "serial_test_derive" -version = "3.2.0" +version = "3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d69265a08751de7844521fd15003ae0a888e035773ba05695c5c759a6f89eef" +checksum = "6f50427f258fb77356e4cd4aa0e87e2bd2c66dbcee41dc405282cae2bfc26c83" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -7054,9 +7050,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "sieve-rs" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ada516ead6adf4f915e69fcb123640bcf23a9b29fc4b537145487e91af7a594" +checksum = "37baebe7a22af73f881c1856c620680444f6ea99482a7734a13f7fc570d17028" dependencies = [ "ahash", "arc-swap", @@ -7070,10 +7066,11 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.7" +version = "1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" dependencies = [ + "errno", "libc", ] @@ -7113,9 +7110,9 @@ checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "siphasher" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" [[package]] name = "slab" @@ -7158,7 +7155,7 @@ dependencies = [ "regex", "reqwest", "rkyv", - "rustls 0.23.35", + "rustls 0.23.36", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", @@ -7174,7 +7171,7 @@ dependencies = [ "trc", "types", "utils", - "webpki-roots 1.0.4", + "webpki-roots 1.0.5", "x509-parser", ] @@ -7223,9 +7220,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" dependencies = [ "libc", "windows-sys 0.60.2", @@ -7402,7 +7399,7 @@ dependencies = [ "rusqlite", "rust-s3", "rustls 0.21.12", - "rustls 0.23.35", + "rustls 0.23.36", "rustls-pki-types", "serde", "serde_json", @@ -7463,9 +7460,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.111" +version = "2.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" dependencies = [ "proc-macro2", "quote", @@ -7489,7 +7486,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -7573,7 +7570,7 @@ dependencies = [ "reqwest", "ring", "rkyv", - "rustls 0.23.35", + "rustls 0.23.36", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", @@ -7603,11 +7600,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl 2.0.17", + "thiserror-impl 2.0.18", ] [[package]] @@ -7618,18 +7615,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] name = "thiserror-impl" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -7643,31 +7640,31 @@ dependencies = [ [[package]] name = "time" -version = "0.3.44" +version = "0.3.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +checksum = "9da98b7d9b7dad93488a84b8248efc35352b0b2657397d4167e7ad67e5d535e5" dependencies = [ "deranged", "itoa", "js-sys", "num-conv", "powerfmt", - "serde", + "serde_core", "time-core", "time-macros", ] [[package]] name = "time-core" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] name = "time-macros" -version = "0.2.24" +version = "0.2.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +checksum = "78cc610bac2dcee56805c99642447d4c5dbde4d01f752ffea0199aee1f601dc4" dependencies = [ "num-conv", "time-core", @@ -7715,7 +7712,7 @@ checksum = "1461056cc1ef47003f7ee16e4cef3741068d4c7f6b627bfce49b7c00c120a530" dependencies = [ "futures-util", "pin-project-lite", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-rustls 0.26.4", ] @@ -7733,9 +7730,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.48.0" +version = "1.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" dependencies = [ "bytes", "libc", @@ -7743,7 +7740,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.1", + "socket2 0.6.2", "tokio-macros", "windows-sys 0.61.2", ] @@ -7756,14 +7753,14 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] name = "tokio-postgres" -version = "0.7.15" +version = "0.7.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b40d66d9b2cfe04b628173409368e58247e8eddbbd3b0e6c6ba1d09f20f6c9e" +checksum = "dcea47c8f71744367793f16c2db1f11cb859d28f436bdb4ca9193eb1f787ee42" dependencies = [ "async-trait", "byteorder", @@ -7779,7 +7776,7 @@ dependencies = [ "postgres-protocol", "postgres-types", "rand 0.9.2", - "socket2 0.6.1", + "socket2 0.6.2", "tokio", "tokio-util", "whoami", @@ -7801,15 +7798,15 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls 0.23.35", + "rustls 0.23.36", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" dependencies = [ "futures-core", "pin-project-lite", @@ -7824,7 +7821,7 @@ checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" dependencies = [ "futures-util", "log", - "rustls 0.23.35", + "rustls 0.23.36", "rustls-pki-types", "tokio", "tokio-rustls 0.26.4", @@ -7834,9 +7831,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.17" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ "bytes", "futures-core", @@ -7882,7 +7879,7 @@ version = "0.23.10+spec-1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269" dependencies = [ - "indexmap 2.12.1", + "indexmap 2.13.0", "toml_datetime", "toml_parser", "winnow", @@ -7933,7 +7930,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f124352108f58ef88299e909f6e9470f1cdc8d2a1397963901b4a6366206bf72" dependencies = [ "base32", - "constant_time_eq", + "constant_time_eq 0.3.1", "hmac 0.12.1", "sha1", "sha2 0.10.9", @@ -7963,9 +7960,9 @@ dependencies = [ [[package]] name = "tower" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", @@ -7989,7 +7986,7 @@ dependencies = [ "http-body 1.0.1", "iri-string", "pin-project-lite", - "tower 0.5.2", + "tower 0.5.3", "tower-layer", "tower-service", ] @@ -8026,7 +8023,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -8135,10 +8132,10 @@ dependencies = [ "httparse", "log", "rand 0.9.2", - "rustls 0.23.35", + "rustls 0.23.36", "rustls-pki-types", "sha1", - "thiserror 2.0.17", + "thiserror 2.0.18", "utf-8", ] @@ -8220,9 +8217,9 @@ dependencies = [ [[package]] name = "unicase" -version = "2.8.1" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" [[package]] name = "unicode-bidi" @@ -8315,19 +8312,20 @@ checksum = "3b5bb2756c16fb66f80cfbf5fb0e0c09a7001e739f453c9ec241b9c8b1556fda" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] name = "url" -version = "2.5.7" +version = "2.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" dependencies = [ "form_urlencoded", "idna", "percent-encoding", "serde", + "serde_derive", ] [[package]] @@ -8371,12 +8369,12 @@ dependencies = [ "psl", "quick_cache", "rand 0.9.2", - "rcgen 0.14.6", + "rcgen 0.14.7", "regex", "reqwest", "ring", "rkyv", - "rustls 0.23.35", + "rustls 0.23.36", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", @@ -8385,16 +8383,16 @@ dependencies = [ "tokio", "tokio-rustls 0.26.4", "trc", - "webpki-roots 1.0.4", + "webpki-roots 1.0.5", "x509-parser", "xxhash-rust", ] [[package]] name = "uuid" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" +checksum = "ee48d38b119b0cd71fe4141b30f5ba9c7c5d9f4e7a3a8b4a674e4b6ef789976f" dependencies = [ "getrandom 0.3.4", "js-sys", @@ -8423,7 +8421,7 @@ checksum = "8c44ce98e7227a04eeb4cf9c784109a5c9710e54849ceb4f09f8597247897f1e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", "unzip-n", ] @@ -8495,25 +8493,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] -name = "wasip2" -version = "1.0.1+wasi-0.2.4" +name = "wasi" +version = "0.14.7+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" dependencies = [ "wit-bindgen", ] [[package]] name = "wasite" -version = "0.1.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] [[package]] name = "wasm-bindgen" -version = "0.2.106" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" dependencies = [ "cfg-if", "once_cell", @@ -8524,11 +8534,12 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.56" +version = "0.4.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" +checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" dependencies = [ "cfg-if", + "futures-util", "js-sys", "once_cell", "wasm-bindgen", @@ -8537,9 +8548,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.106" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -8547,22 +8558,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.106" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.106" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" dependencies = [ "unicode-ident", ] @@ -8582,9 +8593,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.83" +version = "0.3.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" +checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" dependencies = [ "js-sys", "wasm-bindgen", @@ -8602,9 +8613,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee3e3b5f5e80bc89f30ce8d0343bf4e5f12341c51f3e26cbeecbc7c85443e85b" +checksum = "36a29fc0408b113f68cf32637857ab740edfafdf460c326cd2afaa2d84cc05dc" dependencies = [ "rustls-pki-types", ] @@ -8615,14 +8626,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.4", + "webpki-roots 1.0.5", ] [[package]] name = "webpki-roots" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e" +checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c" dependencies = [ "rustls-pki-types", ] @@ -8638,9 +8649,9 @@ dependencies = [ [[package]] name = "whoami" -version = "1.6.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +checksum = "8fae98cf96deed1b7572272dfc777713c249ae40aa1cf8862e091e8b745f5361" dependencies = [ "libredox", "wasite", @@ -8705,7 +8716,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -8716,7 +8727,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -9052,9 +9063,9 @@ dependencies = [ [[package]] name = "wit-bindgen" -version = "0.46.0" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" [[package]] name = "writeable" @@ -9096,7 +9107,7 @@ dependencies = [ "oid-registry", "ring", "rusticata-macros", - "thiserror 2.0.17", + "thiserror 2.0.18", "time", ] @@ -9140,15 +9151,15 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", "synstructure", ] [[package]] name = "zenoh" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e2df46d36ba8d173f18e9cce5011117fed81a90a22bc3cfe8bbcfaf66b54114" +checksum = "0f9ff8cb89f5267b8486a69466bc42f240f1ee2d5089e72395a23094e7b74f21" dependencies = [ "ahash", "arc-swap", @@ -9198,18 +9209,18 @@ dependencies = [ [[package]] name = "zenoh-buffers" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82d610eaa2099c16844b092c2b16605d115f354e332eb1d37e9af46e5430ecac" +checksum = "bd9216c3d6c84b56f3e3be634e52365022038e1ac1b9f662f10d425cbf6c0fa8" dependencies = [ "zenoh-collections", ] [[package]] name = "zenoh-codec" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f1cff28c30bca3d07e7b84f71f1c9b81d505449f6fec36a023cc8b601c01150" +checksum = "14bc6747664aa9ecf17becd6e9a29282e535a350cd7c6bd8de7bf2dc662fb93d" dependencies = [ "tracing", "uhlc", @@ -9219,18 +9230,18 @@ dependencies = [ [[package]] name = "zenoh-collections" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a72cac7d4bbe99e41ef46c175e0dd2933e883b17cef3e60151e4461b5a088ad3" +checksum = "0d642ecfe0d85f0cd846be9bc92805d926c092a6e6c7a575b6346752f8c3ae16" dependencies = [ "ahash", ] [[package]] name = "zenoh-config" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df65e4ab337079d897aaaa6d01474b4f8683daaa05a8671b18db34e71a872af7" +checksum = "39765a5f9975aba204c99f2f65308db4952dbea8e5ac79c78ac1eaf5711e970a" dependencies = [ "json5", "nonempty-collections", @@ -9253,9 +9264,9 @@ dependencies = [ [[package]] name = "zenoh-core" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "869deb22305e8f7a6aa324142ab6717f0cc979be8e4a2de646e8f31313224a1c" +checksum = "85c0c1388dccf287aec4e9d5e638630dc9d536db9f1da3522889b42697723b9b" dependencies = [ "lazy_static", "tokio", @@ -9265,9 +9276,9 @@ dependencies = [ [[package]] name = "zenoh-crypto" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f5b09559108c9e11128730d5a6f551dbcc226adfd82423439ad7eeed024c736" +checksum = "b433e08df3b03f2af2d23bd29a32aa5f5522c52e66d63e3d135bfa66373736dd" dependencies = [ "aes", "hmac 0.12.1", @@ -9279,15 +9290,15 @@ dependencies = [ [[package]] name = "zenoh-keyexpr" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8b754ea6460c4c3c5dd67e6a8c62bfc076c16f8b794898fb3ea6f0039a5939e" +checksum = "19a3c47c89cb55ea45a1b3fe7d1fe8682ea93530b1fc5245257812db14b55b3d" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", "hashbrown 0.16.1", "keyed-set", "rand 0.8.5", - "schemars 1.1.0", + "schemars 1.2.0", "serde", "token-cell", "zenoh-result", @@ -9295,9 +9306,9 @@ dependencies = [ [[package]] name = "zenoh-link" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d0c04d554b67600967cb520061ef9f635caf7704978357334e66cd7179a204a" +checksum = "6218cecab58435f31fb8b2e185f74f35af8aedd96e8bdd3557b333206b1acfda" dependencies = [ "zenoh-config", "zenoh-link-commons", @@ -9311,19 +9322,19 @@ dependencies = [ [[package]] name = "zenoh-link-commons" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52d1c652a706da1da46eae54a65c4b658153cdebfa213f061beea3bef71088f2" +checksum = "b98adc618f7edb570b9333ce583934a7c63e3a619cb49666515bfc06a000d7b6" dependencies = [ "async-trait", "base64 0.22.1", "flume", "futures", "quinn", - "rustls 0.23.35", + "rustls 0.23.36", "rustls-pemfile 2.2.0", "rustls-pki-types", - "rustls-webpki 0.103.8", + "rustls-webpki 0.103.9", "secrecy", "serde", "socket2 0.5.10", @@ -9331,7 +9342,7 @@ dependencies = [ "tokio", "tokio-util", "tracing", - "webpki-roots 1.0.4", + "webpki-roots 1.0.5", "x509-parser", "zenoh-buffers", "zenoh-codec", @@ -9345,22 +9356,22 @@ dependencies = [ [[package]] name = "zenoh-link-quic" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0069d889388d06e56eb86139021a1af12b9846222847aeabaa05eb92fc2ee108" +checksum = "2d0c25d681db958b7714370d5e1d72a523129633d9262b098e18d44824d39893" dependencies = [ "async-trait", "base64 0.22.1", "quinn", - "rustls 0.23.35", + "rustls 0.23.36", "rustls-pemfile 2.2.0", - "rustls-webpki 0.103.8", + "rustls-webpki 0.103.9", "secrecy", "time", "tokio", "tokio-util", "tracing", - "webpki-roots 1.0.4", + "webpki-roots 1.0.5", "zenoh-config", "zenoh-core", "zenoh-link-commons", @@ -9371,9 +9382,9 @@ dependencies = [ [[package]] name = "zenoh-link-tcp" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "279ff42ab4a65508069596045ab8a3c6fb03f94343c290a9b75f9568c49094e4" +checksum = "c0f23bd5d06a0014ce5a205961d6d47c8e8d792d9fd050ae9d0c9b609a187995" dependencies = [ "async-trait", "socket2 0.5.10", @@ -9389,16 +9400,16 @@ dependencies = [ [[package]] name = "zenoh-link-tls" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3172615126b64456beb0c537351eb9c715f20f641e37ae8f84cb209816d205a" +checksum = "17544cde682dbcd2a712114786feee14295c28a9f66fc1021637355511e32fa9" dependencies = [ "async-trait", "base64 0.22.1", - "rustls 0.23.35", + "rustls 0.23.36", "rustls-pemfile 2.2.0", "rustls-pki-types", - "rustls-webpki 0.103.8", + "rustls-webpki 0.103.9", "secrecy", "socket2 0.5.10", "time", @@ -9407,7 +9418,7 @@ dependencies = [ "tokio-rustls 0.26.4", "tokio-util", "tracing", - "webpki-roots 1.0.4", + "webpki-roots 1.0.5", "x509-parser", "zenoh-config", "zenoh-core", @@ -9419,9 +9430,9 @@ dependencies = [ [[package]] name = "zenoh-link-udp" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11006545a887167d431e8c3f2867ad386e7d678ee8babd3df4a7537b168e47de" +checksum = "587ca1de1caa0b444106d31c26801f4e87b354293d2d37afd8f70d9e793fe35e" dependencies = [ "async-trait", "libc", @@ -9441,21 +9452,21 @@ dependencies = [ [[package]] name = "zenoh-macros" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21df1681ca014a4bd69db91d356bcf699d346f854ff8a6b73123cee75f5073cd" +checksum = "b760a458cd906ac888b37fd1abdb21a0f58ecc64cc3882f83a976cb5ca8e0632" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", "zenoh-keyexpr", ] [[package]] name = "zenoh-plugin-trait" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4651bc51951eb004e71bba71583f9ea60240bdd11351b61dbd6956e906088567" +checksum = "7325b773c43a86a94f800cb971ab7e4b7e01ce76819c9c100ea783a47c3a25e4" dependencies = [ "git-version", "libloading", @@ -9471,9 +9482,9 @@ dependencies = [ [[package]] name = "zenoh-protocol" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab17d39b24f6fbd0a24e03a5ab96c236d3d5ebb5071978338c9bc3fe1ccff626" +checksum = "9b4d3dad7aeeea780495692b195cd56515569c32b76b9dd077cc408c3ebca03f" dependencies = [ "const_format", "rand 0.8.5", @@ -9486,18 +9497,18 @@ dependencies = [ [[package]] name = "zenoh-result" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b7d23a2f69c4b96ae95ac2f80808cce5f2ed9697ba37e769a75428fb6806a57" +checksum = "0b4dbfea68b947a790d5525bcf061e91e2fdc2798bce619851919b353a8580fa" dependencies = [ "anyhow", ] [[package]] name = "zenoh-runtime" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a71f356bb63c1c9d37cb4bc89112e83a9fd6079458d1440232d67ca41a52c687" +checksum = "760a1f7880f98427ad849d600257d1455a18afe981681f362684a3f91042537e" dependencies = [ "lazy_static", "ron", @@ -9510,9 +9521,9 @@ dependencies = [ [[package]] name = "zenoh-sync" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b1113111e247d6b1ebd5d406b43e23e78faca2c7525dca22a2a075c64787bd" +checksum = "98f132137bb003f10b7fff086cb18addf8e8273b9c0d2722a53b5074c8a79965" dependencies = [ "arc-swap", "event-listener 5.4.1", @@ -9525,9 +9536,9 @@ dependencies = [ [[package]] name = "zenoh-task" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a1f690f81704b1e934b9fb7a63ce544af73521ca9d11b5a439cdfc74108dd28" +checksum = "1b17d10136fdabec7e21a3fcef568c210ee6a2d71cde6adcde99e9236584f3a1" dependencies = [ "futures", "tokio", @@ -9539,9 +9550,9 @@ dependencies = [ [[package]] name = "zenoh-transport" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cdc4dcb4a68161afe25e44871a66a5c3c347d86a77628ab101d859a9518db9" +checksum = "50739b4c45e0963df8377abddb74701a4b708b178590eae92f27a604e25daf44" dependencies = [ "async-trait", "crossbeam-utils", @@ -9573,9 +9584,9 @@ dependencies = [ [[package]] name = "zenoh-util" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea9b52e8beceffc0deca01e10ca726c604286a1ac475f95acdd87ee55f18b120" +checksum = "9512987c13925d32d3331507c8807853d5b682ea8da94d0ba6534c7a8ace48aa" dependencies = [ "async-trait", "const_format", @@ -9586,7 +9597,7 @@ dependencies = [ "libc", "libloading", "pnet_datalink", - "schemars 1.1.0", + "schemars 1.2.0", "serde", "serde_json", "shellexpand", @@ -9600,22 +9611,22 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.31" +version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" +checksum = "fdea86ddd5568519879b8187e1cf04e24fce28f7fe046ceecbce472ff19a2572" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.31" +version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" +checksum = "0c15e1b46eff7c6c91195752e0eeed8ef040e391cdece7c25376957d5f15df22" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -9635,7 +9646,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", "synstructure", ] @@ -9650,13 +9661,13 @@ dependencies = [ [[package]] name = "zeroize_derive" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -9689,7 +9700,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -9701,13 +9712,13 @@ dependencies = [ "aes", "arbitrary", "bzip2", - "constant_time_eq", + "constant_time_eq 0.3.1", "crc32fast", "deflate64", "flate2", "getrandom 0.3.4", "hmac 0.12.1", - "indexmap 2.12.1", + "indexmap 2.13.0", "lzma-rust2", "memchr", "pbkdf2", @@ -9727,9 +9738,9 @@ checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" [[package]] name = "zmij" -version = "0.1.8" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1dccf46b25b205e4bebe1d5258a991df1cc17801017a845cb5b3fe0269781aa" +checksum = "02aae0f83f69aafc94776e879363e9771d7ecbffe2c7fbb6c14c5e00dfe88439" [[package]] name = "zopfli" diff --git a/crates/common/src/addresses.rs b/crates/common/src/addresses.rs index 8ef0fdb2..812515de 100644 --- a/crates/common/src/addresses.rs +++ b/crates/common/src/addresses.rs @@ -22,7 +22,9 @@ impl Server { email: &str, session_id: u64, ) -> trc::Result> { - let mut address = self + todo!() + + /*let mut address = self .core .smtp .session @@ -51,7 +53,7 @@ impl Server { } } - Ok(None) + Ok(None)*/ } pub async fn rcpt( @@ -60,8 +62,10 @@ impl Server { email: &str, session_id: u64, ) -> trc::Result { + todo!() + // Expand subaddress - let mut address = self + /*let mut address = self .core .smtp .session @@ -89,7 +93,7 @@ impl Server { } } - Ok(RcptType::Invalid) + Ok(RcptType::Invalid)*/ } pub async fn vrfy( @@ -98,18 +102,20 @@ impl Server { address: &str, session_id: u64, ) -> trc::Result> { - directory - .vrfy( - self.core - .smtp - .session - .rcpt - .subaddressing - .to_subaddress(self, address, session_id) - .await - .as_ref(), - ) - .await + todo!() + + /*directory + .vrfy( + self.core + .smtp + .session + .rcpt + .subaddressing + .to_subaddress(self, address, session_id) + .await + .as_ref(), + ) + .await* */ } pub async fn expn( @@ -118,21 +124,23 @@ impl Server { address: &str, session_id: u64, ) -> trc::Result> { - directory - .expn( - self.core - .smtp - .session - .rcpt - .subaddressing - .to_subaddress(self, address, session_id) - .await - .as_ref(), - ) - .await + todo!() + /*directory + .expn( + self.core + .smtp + .session + .rcpt + .subaddressing + .to_subaddress(self, address, session_id) + .await + .as_ref(), + ) + .await*/ } } +/* impl AddressMapping { pub fn parse(bp: &mut Bootstrap, key: impl AsKey) -> Self { let key = key.as_key(); @@ -224,3 +232,4 @@ impl AddressMapping { } } } +*/ diff --git a/crates/common/src/auth/access_token.rs b/crates/common/src/auth/access_token.rs index dd23790a..5f94cbcb 100644 --- a/crates/common/src/auth/access_token.rs +++ b/crates/common/src/auth/access_token.rs @@ -44,7 +44,7 @@ impl Server { let mut role_permissions = RolePermissions::default(); // Extract data - let mut object_quota = self.core.jmap.max_objects; + let mut object_quota = self.core.email.max_objects; let mut description = None; let mut tenant_id = None; let mut quota = None; @@ -235,7 +235,7 @@ impl Server { Err(err) => Err(err), }; - match &self.core.jmap.fallback_admin { + match &self.core.network.security.fallback_admin { Some((_, secret)) if account_id == u32::MAX => { self.build_access_token_from_principal(Principal::fallback_admin(secret), revision) .await diff --git a/crates/common/src/auth/mod.rs b/crates/common/src/auth/mod.rs index 5ab4b39e..9cf4d3d0 100644 --- a/crates/common/src/auth/mod.rs +++ b/crates/common/src/auth/mod.rs @@ -130,7 +130,9 @@ impl Server { match &req.credentials { Credentials::Plain { username, secret } => { // Then check if the credentials match the fallback admin or master user - match (&self.core.jmap.fallback_admin, &self.core.jmap.master_user) { + let master_user: Option<(String, String)> = None; + let todo = "implement master"; + match (&self.core.network.security.fallback_admin, &master_user) { (Some((fallback_admin, fallback_pass)), _) if username == fallback_admin => { if verify_secret_hash(fallback_pass, secret).await? { trc::event!( @@ -204,7 +206,9 @@ impl Server { ) .await .unwrap_or_default() - } else if let Some((_, fallback_pass)) = &self.core.jmap.fallback_admin { + } else if let Some((_, fallback_pass)) = + &self.core.network.security.fallback_admin + { Principal::fallback_admin(fallback_pass).into() } else { None diff --git a/crates/common/src/auth/oauth/config.rs b/crates/common/src/auth/oauth/config.rs index 50766225..af15bf2a 100644 --- a/crates/common/src/auth/oauth/config.rs +++ b/crates/common/src/auth/oauth/config.rs @@ -4,8 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::time::Duration; - +use crate::{ + config::{build_ecdsa_pem, build_rsa_keypair}, + manager::{bootstrap::Bootstrap, webadmin::Resource}, +}; use biscuit::{ jwa::{Algorithm, SignatureAlgorithm}, jwk::{ @@ -15,17 +17,12 @@ use biscuit::{ }, jws::Secret, }; +use registry::schema::{enums::JwtSignatureAlgorithm, prelude::Object, structs::Authentication}; use ring::signature::{self, KeyPair}; use rsa::{RsaPublicKey, pkcs1::DecodeRsaPublicKey, traits::PublicKeyParts}; use store::rand::{Rng, distr::Alphanumeric, rng}; -use utils::config::Config; use x509_parser::num_bigint::BigUint; -use crate::{ - config::{build_ecdsa_pem, build_rsa_keypair}, - manager::webadmin::Resource, -}; - #[derive(Clone)] pub struct OAuthConfig { pub oauth_key: String, @@ -46,32 +43,21 @@ pub struct OAuthConfig { } impl OAuthConfig { - pub fn parse(bp: &mut Bootstrap) -> Self { - let oidc_signature_algorithm = match config.value("oauth.oidc.signature-algorithm") { - Some(alg) => match alg.to_uppercase().as_str() { - "HS256" => SignatureAlgorithm::HS256, - "HS384" => SignatureAlgorithm::HS384, - "HS512" => SignatureAlgorithm::HS512, + pub async fn parse(bp: &mut Bootstrap) -> Self { + let auth = bp.setting_infallible::().await; - "RS256" => SignatureAlgorithm::RS256, - "RS384" => SignatureAlgorithm::RS384, - "RS512" => SignatureAlgorithm::RS512, - - "ES256" => SignatureAlgorithm::ES256, - "ES384" => SignatureAlgorithm::ES384, - - "PS256" => SignatureAlgorithm::PS256, - "PS384" => SignatureAlgorithm::PS384, - "PS512" => SignatureAlgorithm::PS512, - _ => { - config.new_parse_error( - "oauth.oidc.signature-algorithm", - format!("Invalid OIDC signature algorithm: {}", alg), - ); - SignatureAlgorithm::HS256 - } - }, - None => SignatureAlgorithm::HS256, + let oidc_signature_algorithm = match auth.signature_algorithm { + JwtSignatureAlgorithm::Es256 => SignatureAlgorithm::ES256, + JwtSignatureAlgorithm::Es384 => SignatureAlgorithm::ES384, + JwtSignatureAlgorithm::Ps256 => SignatureAlgorithm::PS256, + JwtSignatureAlgorithm::Ps384 => SignatureAlgorithm::PS384, + JwtSignatureAlgorithm::Ps512 => SignatureAlgorithm::PS512, + JwtSignatureAlgorithm::Rs256 => SignatureAlgorithm::RS256, + JwtSignatureAlgorithm::Rs384 => SignatureAlgorithm::RS384, + JwtSignatureAlgorithm::Rs512 => SignatureAlgorithm::RS512, + JwtSignatureAlgorithm::Hs256 => SignatureAlgorithm::HS256, + JwtSignatureAlgorithm::Hs384 => SignatureAlgorithm::HS384, + JwtSignatureAlgorithm::Hs512 => SignatureAlgorithm::HS512, }; let rand_key = rng() @@ -85,36 +71,23 @@ impl OAuthConfig { SignatureAlgorithm::None | SignatureAlgorithm::HS256 | SignatureAlgorithm::HS384 - | SignatureAlgorithm::HS512 => { - let key = config - .value("oauth.oidc.signature-key") - .map(|s| s.to_string().into_bytes()) - .unwrap_or(rand_key); - - ( - Secret::Bytes(key.clone()), - AlgorithmParameters::OctetKey(OctetKeyParameters { - key_type: OctetKeyType::Octet, - value: key, - }), - ) - } + | SignatureAlgorithm::HS512 => ( + Secret::Bytes(auth.signature_key.as_bytes().to_vec()), + AlgorithmParameters::OctetKey(OctetKeyParameters { + key_type: OctetKeyType::Octet, + value: auth.signature_key.as_bytes().to_vec(), + }), + ), SignatureAlgorithm::RS256 | SignatureAlgorithm::RS384 | SignatureAlgorithm::RS512 | SignatureAlgorithm::PS256 | SignatureAlgorithm::PS384 - | SignatureAlgorithm::PS512 => parse_rsa_key(bp).unwrap_or_else(|| { - ( - Secret::Bytes(rand_key.clone()), - AlgorithmParameters::OctetKey(OctetKeyParameters { - key_type: OctetKeyType::Octet, - value: rand_key, - }), - ) - }), - SignatureAlgorithm::ES256 | SignatureAlgorithm::ES384 | SignatureAlgorithm::ES512 => { - parse_ecdsa_key(config, oidc_signature_algorithm).unwrap_or_else(|| { + | SignatureAlgorithm::PS512 => parse_rsa_key(&auth) + .map_err(|err| { + bp.build_error(Object::Authentication.singleton(), err); + }) + .unwrap_or_else(|_| { ( Secret::Bytes(rand_key.clone()), AlgorithmParameters::OctetKey(OctetKeyParameters { @@ -122,7 +95,21 @@ impl OAuthConfig { value: rand_key, }), ) - }) + }), + SignatureAlgorithm::ES256 | SignatureAlgorithm::ES384 | SignatureAlgorithm::ES512 => { + parse_ecdsa_key(&auth, oidc_signature_algorithm) + .map_err(|err| { + bp.build_error(Object::Authentication.singleton(), err); + }) + .unwrap_or_else(|_| { + ( + Secret::Bytes(rand_key.clone()), + AlgorithmParameters::OctetKey(OctetKeyParameters { + key_type: OctetKeyType::Octet, + value: rand_key, + }), + ) + }) } }; @@ -145,49 +132,16 @@ impl OAuthConfig { }; OAuthConfig { - oauth_key: config - .value("oauth.key") - .map(|s| s.to_string()) - .unwrap_or_else(|| { - rng() - .sample_iter(Alphanumeric) - .take(64) - .map(char::from) - .collect::() - }), - oauth_expiry_user_code: config - .property_or_default::("oauth.expiry.user-code", "30m") - .unwrap_or_else(|| Duration::from_secs(30 * 60)) - .as_secs(), - oauth_expiry_auth_code: config - .property_or_default::("oauth.expiry.auth-code", "10m") - .unwrap_or_else(|| Duration::from_secs(10 * 60)) - .as_secs(), - oauth_expiry_token: config - .property_or_default::("oauth.expiry.token", "1h") - .unwrap_or_else(|| Duration::from_secs(60 * 60)) - .as_secs(), - oauth_expiry_refresh_token: config - .property_or_default::("oauth.expiry.refresh-token", "30d") - .unwrap_or_else(|| Duration::from_secs(30 * 24 * 60 * 60)) - .as_secs(), - oauth_expiry_refresh_token_renew: config - .property_or_default::("oauth.expiry.refresh-token-renew", "4d") - .unwrap_or_else(|| Duration::from_secs(4 * 24 * 60 * 60)) - .as_secs(), - oauth_max_auth_attempts: config - .property_or_default("oauth.auth.max-attempts", "3") - .unwrap_or(10), - oidc_expiry_id_token: config - .property_or_default::("oauth.oidc.expiry.id-token", "15m") - .unwrap_or_else(|| Duration::from_secs(15 * 60)) - .as_secs(), - allow_anonymous_client_registration: config - .property_or_default("oauth.client-registration.anonymous", "false") - .unwrap_or(false), - require_client_authentication: config - .property_or_default("oauth.client-registration.require", "false") - .unwrap_or(true), + oauth_key: auth.encryption_key, + oauth_expiry_user_code: auth.user_code_expiry.as_secs(), + oauth_expiry_auth_code: auth.auth_code_expiry.as_secs(), + oauth_expiry_token: auth.access_token_expiry.as_secs(), + oauth_expiry_refresh_token: auth.refresh_token_expiry.as_secs(), + oauth_expiry_refresh_token_renew: auth.refresh_token_renewal.as_secs(), + oauth_max_auth_attempts: auth.auth_code_max_attempts as u32, + oidc_expiry_id_token: auth.id_token_expiry.as_secs(), + allow_anonymous_client_registration: auth.anonymous_client_registration, + require_client_authentication: auth.require_client_registration, oidc_signing_secret, oidc_signature_algorithm, oidc_jwks, @@ -220,26 +174,13 @@ impl Default for OAuthConfig { } } -fn parse_rsa_key(bp: &mut Bootstrap) -> Option<(Secret, AlgorithmParameters)> { - let rsa_key_pair = match build_rsa_keypair(config.value_require("oauth.oidc.signature-key")?) { - Ok(key) => key, - Err(err) => { - config.new_build_error( - "oauth.oidc.signature-key", - format!("Failed to build RSA key: {}", err), - ); - return None; - } - }; +fn parse_rsa_key(auth: &Authentication) -> Result<(Secret, AlgorithmParameters), String> { + let rsa_key_pair = build_rsa_keypair(&auth.signature_key)?; let rsa_public_key = match RsaPublicKey::from_pkcs1_der(rsa_key_pair.public_key().as_ref()) { Ok(key) => key, Err(err) => { - config.new_build_error( - "oauth.oidc.signature-key", - format!("Failed to obtain RSA public key: {}", err), - ); - return None; + return Err(format!("Failed to obtain RSA public key: {}", err)); } }; @@ -250,17 +191,16 @@ fn parse_rsa_key(bp: &mut Bootstrap) -> Option<(Secret, AlgorithmParameters)> { ..Default::default() }; - ( + Ok(( Secret::RsaKeyPair(rsa_key_pair.into()), AlgorithmParameters::RSA(rsa_key_params), - ) - .into() + )) } fn parse_ecdsa_key( - bp: &mut Bootstrap, + auth: &Authentication, oidc_signature_algorithm: SignatureAlgorithm, -) -> Option<(Secret, AlgorithmParameters)> { +) -> Result<(Secret, AlgorithmParameters), String> { let (alg, curve) = match oidc_signature_algorithm { SignatureAlgorithm::ES256 => ( &signature::ECDSA_P256_SHA256_FIXED_SIGNING, @@ -273,18 +213,7 @@ fn parse_ecdsa_key( _ => unreachable!(), }; - let ecdsa_key_pair = - match build_ecdsa_pem(alg, config.value_require("oauth.oidc.signature-key")?) { - Ok(key) => key, - Err(err) => { - config.new_build_error( - "oauth.oidc.signature-key", - format!("Failed to build ECDSA key: {}", err), - ); - return None; - } - }; - + let ecdsa_key_pair = build_ecdsa_pem(alg, &auth.signature_key)?; let ecdsa_public_key = ecdsa_key_pair.public_key().as_ref(); let (x, y) = match oidc_signature_algorithm { @@ -292,11 +221,7 @@ fn parse_ecdsa_key( let points = match p256::EncodedPoint::from_bytes(ecdsa_public_key) { Ok(points) => points, Err(err) => { - config.new_build_error( - "oauth.oidc.signature-key", - format!("Failed to parse ECDSA key: {}", err), - ); - return None; + return Err(format!("Failed to parse ECDSA key: {}", err)); } }; @@ -309,11 +234,7 @@ fn parse_ecdsa_key( let points = match p384::EncodedPoint::from_bytes(ecdsa_public_key) { Ok(points) => points, Err(err) => { - config.new_build_error( - "oauth.oidc.signature-key", - format!("Failed to parse ECDSA key: {}", err), - ); - return None; + return Err(format!("Failed to parse ECDSA key: {}", err)); } }; @@ -333,9 +254,8 @@ fn parse_ecdsa_key( d: None, }; - ( + Ok(( Secret::EcdsaKeyPair(ecdsa_key_pair.into()), AlgorithmParameters::EllipticCurve(ecdsa_key_params), - ) - .into() + )) } diff --git a/crates/common/src/auth/oauth/token.rs b/crates/common/src/auth/oauth/token.rs index 70ac1cee..8e984805 100644 --- a/crates/common/src/auth/oauth/token.rs +++ b/crates/common/src/auth/oauth/token.rs @@ -244,7 +244,7 @@ impl Server { .details("Account does not contain secrets") .caused_by(trc::location!()), ) - } else if let Some((_, secret)) = &self.core.jmap.fallback_admin { + } else if let Some((_, secret)) = &self.core.network.security.fallback_admin { Ok(secret.into()) } else { Err(trc::AuthEvent::Error diff --git a/crates/common/src/auth/rate_limit.rs b/crates/common/src/auth/rate_limit.rs index 59bd9f16..6fb9ed31 100644 --- a/crates/common/src/auth/rate_limit.rs +++ b/crates/common/src/auth/rate_limit.rs @@ -20,7 +20,7 @@ impl Server { &self, access_token: &AccessToken, ) -> trc::Result> { - let is_rate_allowed = if let Some(rate) = &self.core.jmap.rate_authenticated { + let is_rate_allowed = if let Some(rate) = &self.core.network.http.rate_authenticated { self.core .storage .lookup @@ -57,7 +57,7 @@ impl Server { } pub async fn is_http_anonymous_request_allowed(&self, addr: &IpAddr) -> trc::Result<()> { - if let Some(rate) = &self.core.jmap.rate_anonymous + if let Some(rate) = &self.core.network.http.rate_anonymous && !self.is_ip_allowed(addr) && self .core diff --git a/crates/common/src/config/inner.rs b/crates/common/src/config/inner.rs index 69765697..0ef690b8 100644 --- a/crates/common/src/config/inner.rs +++ b/crates/common/src/config/inner.rs @@ -32,7 +32,7 @@ use utils::{ }; impl Data { - pub fn parse(bp: &mut Bootstrap) -> Self { + pub async fn parse(bp: &mut Bootstrap) -> Self { // Parse certificates let mut certificates = AHashMap::new(); let mut subject_names = AHashSet::new(); @@ -65,7 +65,7 @@ impl Data { }) .ok() .map(Arc::new), - blocked_ips: RwLock::new(BlockedIps::parse(bp).blocked_ip_addresses), + blocked_ips: RwLock::new(BlockedIps::parse(bp).await), jmap_id_gen: id_generator.clone(), queue_id_gen: id_generator.clone(), span_id_gen: id_generator, diff --git a/crates/common/src/config/mod.rs b/crates/common/src/config/mod.rs index 464c256c..2df0c265 100644 --- a/crates/common/src/config/mod.rs +++ b/crates/common/src/config/mod.rs @@ -10,6 +10,7 @@ use crate::{ auth::oauth::config::OAuthConfig, config::mailstore::{imap::ImapConfig, scripts::Scripting, spamfilter::SpamFilterConfig}, expr::*, + manager::bootstrap::Bootstrap, }; use arc_swap::ArcSwap; use coordinator::Coordinator; @@ -31,12 +32,9 @@ pub mod storage; pub mod telemetry; impl Core { - pub async fn parse( - bp: &mut Bootstrap, - mut stores: Stores, - config_manager: ConfigManager, - ) -> Self { - let mut data = config + pub async fn parse(bp: &mut Bootstrap, mut stores: Stores) -> Self { + todo!() + /*let mut data = config .value_require("storage.data") .map(|id| id.to_string()) .and_then(|id| { @@ -197,7 +195,7 @@ impl Core { blobs: stores.blob_stores, ftss: stores.search_stores, }, - } + }*/ } pub fn into_shared(self) -> ArcSwap { diff --git a/crates/common/src/config/smtp/auth.rs b/crates/common/src/config/smtp/auth.rs index 65c5d616..7e2eac82 100644 --- a/crates/common/src/config/smtp/auth.rs +++ b/crates/common/src/config/smtp/auth.rs @@ -22,7 +22,6 @@ use registry::{ types::ObjectType, }; use rustls_pki_types::{PrivateKeyDer, PrivatePkcs1KeyDer, PrivatePkcs8KeyDer, pem::PemObject}; -use std::sync::Arc; use utils::config::utils::ParseValue; #[derive(Clone)] @@ -34,12 +33,6 @@ pub struct MailAuthConfig { pub iprev: IpRevAuthConfig, } -#[derive(Clone)] -pub struct ResolvedSignature { - pub signer: Arc, - pub sealer: Arc, -} - #[derive(Clone)] pub struct DkimAuthConfig { pub verify: IfBlock, diff --git a/crates/common/src/config/telemetry.rs b/crates/common/src/config/telemetry.rs index c24a78b1..bcb62fa3 100644 --- a/crates/common/src/config/telemetry.rs +++ b/crates/common/src/config/telemetry.rs @@ -93,7 +93,6 @@ pub struct WebhookTracer { pub discard_after: Duration, pub tls_allow_invalid_certs: bool, pub headers: HeaderMap, - pub properties: HashSet, } // SPDX-SnippetBegin @@ -491,7 +490,6 @@ impl Tracers { key: hook.signature_key.unwrap_or_default(), throttle: hook.throttle.into_inner(), discard_after: hook.discard_after.into_inner(), - properties: Default::default(), }), }; diff --git a/crates/common/src/core.rs b/crates/common/src/core.rs index 0c20d498..667d7f4e 100644 --- a/crates/common/src/core.rs +++ b/crates/common/src/core.rs @@ -10,7 +10,7 @@ use crate::{ config::{ mailstore::spamfilter::SpamClassifier, smtp::{ - auth::{ArcSealer, DkimSigner, LazySignature, ResolvedSignature, build_signature}, + auth::{ArcSealer, DkimSigner}, queue::{ ConnectionStrategy, DEFAULT_QUEUE_NAME, MxConfig, QueueExpiry, QueueName, QueueStrategy, RequireOptional, RoutingStrategy, TlsStrategy, VirtualQueue, @@ -130,7 +130,8 @@ impl Server { } pub fn get_arc_sealer(&self, name: &str, session_id: u64) -> Option> { - self.resolve_signature(name).map(|s| s.sealer).or_else(|| { + todo!() + /*self.resolve_signature(name).map(|s| s.sealer).or_else(|| { trc::event!( Arc(trc::ArcEvent::SealerNotFound), Id = name.to_string(), @@ -138,11 +139,12 @@ impl Server { ); None - }) + })*/ } pub fn get_dkim_signer(&self, name: &str, session_id: u64) -> Option> { - self.resolve_signature(name).map(|s| s.signer).or_else(|| { + todo!() + /*self.resolve_signature(name).map(|s| s.signer).or_else(|| { trc::event!( Dkim(trc::DkimEvent::SignerNotFound), Id = name.to_string(), @@ -150,30 +152,7 @@ impl Server { ); None - }) - } - - fn resolve_signature(&self, name: &str) -> Option { - let lazy_resolver_ = self.core.smtp.mail_auth.signatures.get(name)?; - match lazy_resolver_.load().as_ref() { - LazySignature::Resolved(resolved_signature) => Some(resolved_signature.clone()), - LazySignature::Pending(bp) => { - let mut config = config.clone(); - if let Some((signer, sealer)) = build_signature(&mut config, name) { - let resolved = ResolvedSignature { - signer: Arc::new(signer), - sealer: Arc::new(sealer), - }; - lazy_resolver_.store(Arc::new(LazySignature::Resolved(resolved.clone()))); - Some(resolved) - } else { - config.log_errors(); - lazy_resolver_.store(Arc::new(LazySignature::Failed)); - None - } - } - LazySignature::Failed => None, - } + })*/ } pub fn get_trusted_sieve_script(&self, name: &str, session_id: u64) -> Option<&Arc> { @@ -895,6 +874,7 @@ impl Server { } pub async fn cluster_broadcast(&self, event: BroadcastEvent) { + let todo = "refactor event names"; if let Some(broadcast_tx) = &self.inner.ipc.broadcast_tx.clone() && broadcast_tx.send(event).await.is_err() { diff --git a/crates/common/src/enterprise/config.rs b/crates/common/src/enterprise/config.rs index 46661e63..600f5227 100644 --- a/crates/common/src/enterprise/config.rs +++ b/crates/common/src/enterprise/config.rs @@ -87,7 +87,7 @@ impl Enterprise { enterprise.license_key = Some(license); if let Err(err) = bp .registry - .put(Object::Enterprise.singleton(), &enterprise) + .update(Object::Enterprise.singleton(), &enterprise) .await { trc::error!( diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 8ba7a159..9277bc67 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -14,6 +14,7 @@ use crate::{ spamfilter::{IpResolver, SpamClassifier, SpamFilterConfig}, }, ipc::TrainTaskController, + listener::blocked::BlockedIps, }; use ahash::{AHashMap, AHashSet}; use arc_swap::ArcSwap; @@ -143,7 +144,7 @@ pub struct Data { pub tls_certificates: ArcSwap>>, pub tls_self_signed_cert: Option>, - pub blocked_ips: RwLock>, + pub blocked_ips: RwLock, pub asn_geo_data: AsnGeoLookupData, @@ -451,70 +452,6 @@ impl BuildHasher for ThrottleKeyHasherBuilder { } } -#[cfg(feature = "test_mode")] -#[allow(clippy::derivable_impls)] -impl Default for Server { - fn default() -> Self { - Self { - inner: Default::default(), - core: Default::default(), - } - } -} - -#[cfg(feature = "test_mode")] -#[allow(clippy::derivable_impls)] -impl Default for Inner { - fn default() -> Self { - Self { - shared_core: Default::default(), - data: Default::default(), - ipc: Default::default(), - cache: Default::default(), - } - } -} - -#[cfg(feature = "test_mode")] -#[allow(clippy::derivable_impls)] -impl Default for Caches { - fn default() -> Self { - Self { - access_tokens: Cache::new(1024, 10 * 1024 * 1024), - http_auth: Cache::new(1024, 10 * 1024 * 1024), - permissions: Cache::new(1024, 10 * 1024 * 1024), - messages: Cache::new(1024, 25 * 1024 * 1024), - files: Cache::new(1024, 10 * 1024 * 1024), - contacts: Cache::new(1024, 10 * 1024 * 1024), - events: Cache::new(1024, 10 * 1024 * 1024), - scheduling: Cache::new(1024, 10 * 1024 * 1024), - dns_rbl: CacheWithTtl::new(1024, 10 * 1024 * 1024), - dns_txt: CacheWithTtl::new(1024, 10 * 1024 * 1024), - dns_mx: CacheWithTtl::new(1024, 10 * 1024 * 1024), - dns_ptr: CacheWithTtl::new(1024, 10 * 1024 * 1024), - dns_ipv4: CacheWithTtl::new(1024, 10 * 1024 * 1024), - dns_ipv6: CacheWithTtl::new(1024, 10 * 1024 * 1024), - dns_tlsa: CacheWithTtl::new(1024, 10 * 1024 * 1024), - dns_mta_sts: CacheWithTtl::new(1024, 10 * 1024 * 1024), - } - } -} - -#[cfg(feature = "test_mode")] -impl Default for Ipc { - fn default() -> Self { - Self { - push_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0, - housekeeper_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0, - task_tx: Default::default(), - queue_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0, - report_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0, - broadcast_tx: None, - train_task_controller: Arc::new(TrainTaskController::default()), - } - } -} - pub fn ip_to_bytes(ip: &IpAddr) -> Vec { match ip { IpAddr::V4(ip) => ip.octets().to_vec(), diff --git a/crates/common/src/listener/blocked.rs b/crates/common/src/listener/blocked.rs index 615ac3fd..54216d78 100644 --- a/crates/common/src/listener/blocked.rs +++ b/crates/common/src/listener/blocked.rs @@ -9,66 +9,81 @@ use crate::{ ip_to_bytes, ipc::BroadcastEvent, manager::bootstrap::Bootstrap, }; use ahash::AHashSet; -use std::{fmt::Debug, net::IpAddr}; -use utils::{ - config::{ - ConfigKey, Rate, - ipmask::{IpAddrMask, IpAddrOrMask}, - utils::ParseValue, +use registry::{ + schema::{ + enums::BlockReason, + structs::{self, AllowedIp, BlockedIp, Rate}, }, - glob::GlobPattern, + types::{datetime::UTCDateTime, ipmask::IpAddrOrMask}, }; +use std::{fmt::Debug, net::IpAddr}; +use store::write::now; +use trc::AddContext; +use utils::glob::{GlobPattern, MatchType}; #[derive(Debug, Clone)] pub struct Security { - fallback_admin: Option<(String, String)>, + pub fallback_admin: Option<(String, String)>, - blocked_ip_networks: Vec, - has_blocked_networks: bool, + pub allowed_ip_addresses: AHashSet, + pub allowed_ip_networks: Vec, + pub has_allowed_networks: bool, + pub blocked_ip_expiration: Option, - allowed_ip_addresses: AHashSet, - allowed_ip_networks: Vec, - has_allowed_networks: bool, + pub http_banned_paths: Vec, + pub scanner_fail_rate: Option, - http_banned_paths: Vec, - scanner_fail_rate: Option, - - auth_fail_rate: Option, - rcpt_fail_rate: Option, - loiter_fail_rate: Option, + pub auth_fail_rate: Option, + pub rcpt_fail_rate: Option, + pub loiter_fail_rate: Option, } -pub const BLOCKED_IP_KEY: &str = "server.blocked-ip"; -pub const BLOCKED_IP_PREFIX: &str = "server.blocked-ip."; -pub const ALLOWED_IP_KEY: &str = "server.allowed-ip"; -pub const ALLOWED_IP_PREFIX: &str = "server.allowed-ip."; - +#[derive(Default)] pub struct BlockedIps { pub blocked_ip_addresses: AHashSet, - pub blocked_ip_networks: Vec, + pub blocked_ip_networks: Vec, + pub has_blocked_networks: bool, } impl Security { pub async fn parse(bp: &mut Bootstrap) -> Self { let mut allowed_ip_addresses = AHashSet::new(); let mut allowed_ip_networks = Vec::new(); + let mut expired_allows = Vec::new(); + let now = now() as i64; - for ip in config - .set_values(ALLOWED_IP_KEY) - .map(IpAddrOrMask::parse_value) - .collect::>() - { - match ip { - Ok(IpAddrOrMask::Ip(ip)) => { + for ip in bp.list_infallible::().await { + let id = ip.id; + let ip = ip.object; + + if ip.expires_at.is_none_or(|ip| ip.timestamp() > now) { + if let Some(ip) = ip.address.try_to_ip() { allowed_ip_addresses.insert(ip); + } else { + allowed_ip_networks.push(ip.address); } - Ok(IpAddrOrMask::Mask(ip)) => { - allowed_ip_networks.push(ip); - } - Err(err) => { - config.new_parse_error(ALLOWED_IP_KEY, err); + } else { + expired_allows.push((id, ip.address)); + } + } + + if !expired_allows.is_empty() { + for (id, _) in &expired_allows { + if let Err(err) = bp.registry.delete(*id).await { + trc::error!( + err.details("Failed to delete expired allowed IP from registry.") + .caused_by(trc::location!()) + ); } } + + trc::event!( + Security(trc::SecurityEvent::IpAllowExpired), + Details = expired_allows + .into_iter() + .map(|(_, ip)| trc::Value::from(ip.into_inner().0)) + .collect::>() + ); } #[cfg(not(feature = "test_mode"))] @@ -78,60 +93,27 @@ impl Security { allowed_ip_addresses.insert(IpAddr::V6(std::net::Ipv6Addr::LOCALHOST)); } - let blocked = BlockedIps::parse(bp); - - // Parse blocked HTTP paths - let mut http_banned_paths = config - .values("server.auto-ban.scan.paths") - .filter_map(|(_, v)| { - let v = v.trim(); - if !v.is_empty() { - MatchType::parse(v).into() - } else { - None - } - }) - .collect::>(); - if http_banned_paths.is_empty() { - for pattern in [ - "*.php*", - "*.cgi*", - "*.asp*", - "*/wp-*", - "*/php*", - "*/cgi-bin*", - "*xmlrpc*", - "*../*", - "*/..*", - "*joomla*", - "*wordpress*", - "*drupal*", - ] - .iter() - { - http_banned_paths.push(MatchType::Matches(GlobPattern::compile(pattern, true))); - } - } - + let security = bp.setting_infallible::().await; Security { - has_blocked_networks: !blocked.blocked_ip_networks.is_empty(), - blocked_ip_networks: blocked.blocked_ip_networks, + fallback_admin: bp.local.fallback_admin_user.as_ref().and_then(|user| { + bp.local + .fallback_admin_secret + .as_ref() + .map(|secret| (user.to_string(), secret.to_string())) + }), has_allowed_networks: !allowed_ip_networks.is_empty(), allowed_ip_addresses, allowed_ip_networks, - auth_fail_rate: config - .property_or_default::>("server.auto-ban.auth.rate", "100/1d") - .unwrap_or_default(), - rcpt_fail_rate: config - .property_or_default::>("server.auto-ban.abuse.rate", "35/1d") - .unwrap_or_default(), - loiter_fail_rate: config - .property_or_default::>("server.auto-ban.loiter.rate", "150/1d") - .unwrap_or_default(), - http_banned_paths, - scanner_fail_rate: config - .property_or_default::>("server.auto-ban.scan.rate", "30/1d") - .unwrap_or_default(), + blocked_ip_expiration: security.auth_ban_period.map(|v| v.as_secs()), + auth_fail_rate: security.auth_ban_rate, + rcpt_fail_rate: security.abuse_ban_rate, + loiter_fail_rate: security.loiter_ban_rate, + http_banned_paths: security + .scan_ban_paths + .iter() + .map(|pattern| MatchType::Matches(GlobPattern::compile(pattern, true))) + .collect(), + scanner_fail_rate: security.scan_ban_rate, } } } @@ -152,7 +134,10 @@ impl Server { .is_none()); if !is_allowed { - return self.block_ip(ip).await.map(|_| true); + return self + .block_ip(ip, BlockReason::RcptToFailure) + .await + .map(|_| true); } } @@ -169,7 +154,10 @@ impl Server { .is_none(); if !is_allowed { - return self.block_ip(ip).await.map(|_| true); + return self + .block_ip(ip, BlockReason::PortScanning) + .await + .map(|_| true); } } @@ -180,7 +168,9 @@ impl Server { let paths = &self.core.network.security.http_banned_paths; if !paths.is_empty() && paths.iter().any(|p| p.matches(path)) && !self.is_ip_allowed(&ip) { - self.block_ip(ip).await.map(|_| true) + self.block_ip(ip, BlockReason::PortScanning) + .await + .map(|_| true) } else { Ok(false) } @@ -196,7 +186,10 @@ impl Server { .is_none(); if !is_allowed { - return self.block_ip(ip).await.map(|_| true); + return self + .block_ip(ip, BlockReason::Loitering) + .await + .map(|_| true); } } @@ -219,29 +212,41 @@ impl Server { .await? .is_none())); if !is_allowed { - return self.block_ip(ip).await.map(|_| true); + return self + .block_ip(ip, BlockReason::AuthFailure) + .await + .map(|_| true); } } Ok(false) } - pub async fn block_ip(&self, ip: IpAddr) -> trc::Result<()> { + pub async fn block_ip(&self, ip: IpAddr, reason: BlockReason) -> trc::Result<()> { // Add IP to blocked list - self.inner.data.blocked_ips.write().insert(ip); + self.inner + .data + .blocked_ips + .write() + .blocked_ip_addresses + .insert(ip); // Write blocked IP to config - self.core - .storage - .config - .set( - [ConfigKey { - key: format!("{}.{}", BLOCKED_IP_KEY, ip), - value: String::new(), - }], - true, - ) - .await?; + let now = now() as i64; + self.registry() + .insert(&BlockedIp { + address: IpAddrOrMask::from_ip(ip), + created_at: UTCDateTime::from_timestamp(now), + expires_at: self + .core + .network + .security + .blocked_ip_expiration + .map(|v| UTCDateTime::from_timestamp(now + v as i64)), + reason, + }) + .await + .caused_by(trc::location!())?; // Increment version self.cluster_broadcast(BroadcastEvent::ReloadBlockedIps) @@ -255,15 +260,14 @@ impl Server { } pub fn is_ip_blocked(&self, ip: &IpAddr) -> bool { - self.inner.data.blocked_ips.read().contains(ip) - || (self.core.network.security.has_blocked_networks - && self - .core - .network - .security + let blocked_ips = self.inner.data.blocked_ips.read(); + (blocked_ips.blocked_ip_addresses.contains(ip) + || (blocked_ips.has_blocked_networks + && blocked_ips .blocked_ip_networks .iter() - .any(|network| network.matches(ip))) + .any(|network| network.matches(ip)))) + && !self.is_ip_allowed(ip) } pub fn is_ip_allowed(&self, ip: &IpAddr) -> bool { @@ -280,31 +284,45 @@ impl Server { } impl BlockedIps { - pub fn parse(bp: &mut Bootstrap) -> Self { - let mut blocked_ip_addresses = AHashSet::new(); - let mut blocked_ip_networks = Vec::new(); + pub async fn parse(bp: &mut Bootstrap) -> Self { + let mut ips = Self::default(); + let mut expired_blocks = Vec::new(); + let now = now() as i64; - for ip in config - .set_values(BLOCKED_IP_KEY) - .map(IpAddrOrMask::parse_value) - .collect::>() - { - match ip { - Ok(IpAddrOrMask::Ip(ip)) => { - blocked_ip_addresses.insert(ip); - } - Ok(IpAddrOrMask::Mask(ip)) => { - blocked_ip_networks.push(ip); - } - Err(err) => { - config.new_parse_error(BLOCKED_IP_KEY, err); + for ip in bp.list_infallible::().await { + let id = ip.id; + let ip = ip.object; + + if ip.expires_at.is_none_or(|ip| ip.timestamp() > now) { + if let Some(ip) = ip.address.try_to_ip() { + ips.blocked_ip_addresses.insert(ip); + } else { + ips.blocked_ip_networks.push(ip.address); } + } else { + expired_blocks.push((id, ip.address)); } } - Self { - blocked_ip_addresses, - blocked_ip_networks, + if !expired_blocks.is_empty() { + for (id, _) in &expired_blocks { + if let Err(err) = bp.registry.delete(*id).await { + trc::error!( + err.details("Failed to delete expired blocked IP from registry.") + .caused_by(trc::location!()) + ); + } + } + trc::event!( + Security(trc::SecurityEvent::IpBlockExpired), + Details = expired_blocks + .into_iter() + .map(|(_, ip)| trc::Value::from(ip.into_inner().0)) + .collect::>() + ); } + + ips.has_blocked_networks = !ips.blocked_ip_networks.is_empty(); + ips } } diff --git a/crates/common/src/manager/boot.rs b/crates/common/src/manager/boot.rs index f459758e..20a9add1 100644 --- a/crates/common/src/manager/boot.rs +++ b/crates/common/src/manager/boot.rs @@ -175,7 +175,8 @@ pub const DEFAULT_SETTINGS: &[(&str, &str)] = &[ impl BootManager { pub async fn init() -> Self { - let mut config_path = std::env::var("CONFIG_PATH").ok(); + todo!() + /*let mut config_path = std::env::var("CONFIG_PATH").ok(); let mut import_export = StoreOp::None; if config_path.is_none() { @@ -253,7 +254,7 @@ impl BootManager { config.resolve_macros(&["env"]).await; // Parser servers - let mut servers = Listeners::parse(&mut config); + let mut servers = Listeners::parse(&mut config).await; // Bind ports and drop privileges servers.bind_and_drop_priv(&mut config); @@ -318,7 +319,7 @@ impl BootManager { } // Parse telemetry - let telemetry = Telemetry::parse(&mut config, &stores); + let telemetry = Telemetry::parse(&mut config); match import_export { StoreOp::None => { @@ -541,7 +542,7 @@ impl BootManager { .await; std::process::exit(0); } - } + }*/ } } diff --git a/crates/common/src/manager/mod.rs b/crates/common/src/manager/mod.rs index b18a755e..42007262 100644 --- a/crates/common/src/manager/mod.rs +++ b/crates/common/src/manager/mod.rs @@ -11,7 +11,6 @@ use utils::HttpLimitResponse; pub mod backup; pub mod boot; -pub mod bootstrap; pub mod console; pub mod reload; pub mod restore; @@ -35,42 +34,6 @@ const DEFAULT_WEBADMIN_URL: &str = const DEFAULT_WEBADMIN_URL: &str = "https://github.com/stalwartlabs/webadmin/releases/latest/download/webadmin-oss.zip"; -impl ConfigManager { - pub async fn fetch_resource(&self, resource_id: &str) -> Result, String> { - if let Some(url) = self - .get(&format!("{resource_id}.resource")) - .await - .map_err(|err| { - format!("Failed to fetch configuration key '{resource_id}.resource': {err}",) - })? - { - fetch_resource(&url, None, Duration::from_secs(60), MAX_SIZE).await - } else { - match resource_id { - "spam-filter" => { - fetch_resource( - DEFAULT_SPAMFILTER_URL, - None, - Duration::from_secs(60), - MAX_SIZE, - ) - .await - } - "webadmin" => { - fetch_resource( - DEFAULT_WEBADMIN_URL, - None, - Duration::from_secs(60), - MAX_SIZE, - ) - .await - } - _ => Err(format!("Unknown resource: {resource_id}")), - } - } - } -} - const MAX_SIZE: usize = 100 * 1024 * 1024; pub async fn fetch_resource( diff --git a/crates/common/src/manager/reload.rs b/crates/common/src/manager/reload.rs index 46044198..ec08cdbb 100644 --- a/crates/common/src/manager/reload.rs +++ b/crates/common/src/manager/reload.rs @@ -7,7 +7,7 @@ use crate::{ Core, Server, config::{server::Listeners, telemetry::Telemetry}, - listener::blocked::{BLOCKED_IP_KEY, BlockedIps}, + listener::blocked::BlockedIps, }; use ahash::AHashMap; use arc_swap::ArcSwap; @@ -22,7 +22,8 @@ pub struct ReloadResult { impl Server { pub async fn reload_blocked_ips(&self) -> trc::Result { - let mut config = self + todo!() + /*let mut config = self .core .storage .config @@ -30,22 +31,24 @@ impl Server { .await?; *self.inner.data.blocked_ips.write() = BlockedIps::parse(&mut config).blocked_ip_addresses; - Ok(config.into()) + Ok(config.into())*/ } pub async fn reload_certificates(&self) -> trc::Result { - let mut config = self.core.storage.config.build_config("certificate").await?; + todo!() + /*let mut config = self.core.storage.config.build_config("certificate").await?; let mut certificates = self.inner.data.tls_certificates.load().as_ref().clone(); parse_certificates(&mut config, &mut certificates, &mut Default::default()); self.inner.data.tls_certificates.store(certificates.into()); - Ok(config.into()) + Ok(config.into())*/ } pub async fn reload_lookups(&self) -> trc::Result { - let mut config = self.core.storage.config.build_config("lookup").await?; + todo!() + /*let mut config = self.core.storage.config.build_config("lookup").await?; let mut stores = Stores::default(); stores.parse_static_stores(&mut config, true); @@ -58,11 +61,12 @@ impl Server { config, new_core: core.into(), tracers: None, - }) + })*/ } pub async fn reload(&self) -> trc::Result { - let mut config = self.core.storage.config.build_config("").await?; + todo!() + /*let mut config = self.core.storage.config.build_config("").await?; // Load stores let mut stores = Stores { @@ -129,7 +133,7 @@ impl Server { } } else { config.into() - }) + })*/ } } diff --git a/crates/common/src/manager/webadmin.rs b/crates/common/src/manager/webadmin.rs index 516cb478..5bf5d328 100644 --- a/crates/common/src/manager/webadmin.rs +++ b/crates/common/src/manager/webadmin.rs @@ -141,7 +141,8 @@ impl WebAdminManager { } pub async fn update(&self, core: &Core) -> trc::Result<()> { - let bytes = core + todo!() + /* let bytes = core .storage .config .fetch_resource("webadmin") @@ -152,7 +153,7 @@ impl WebAdminManager { .reason(err) .details("Failed to download webadmin") })?; - core.storage.blob.put_blob(WEBADMIN_KEY, &bytes).await + core.storage.blob.put_blob(WEBADMIN_KEY, &bytes).await*/ } pub async fn update_and_unpack(&self, core: &Core) -> trc::Result<()> { diff --git a/crates/http-proto/Cargo.toml b/crates/http-proto/Cargo.toml index 4363fdff..ab1dbcc9 100644 --- a/crates/http-proto/Cargo.toml +++ b/crates/http-proto/Cargo.toml @@ -6,6 +6,7 @@ edition = "2024" [dependencies] common = { path = "../common" } trc = { path = "../trc" } +registry = { path = "../registry" } serde = { version = "1.0", features = ["derive"]} serde_json = "1.0" hyper = { version = "1.0.1", features = ["server", "http1", "http2"] } diff --git a/crates/http-proto/src/context.rs b/crates/http-proto/src/context.rs index 4f6335dc..65cc2044 100644 --- a/crates/http-proto/src/context.rs +++ b/crates/http-proto/src/context.rs @@ -10,6 +10,7 @@ use common::{ }; use compact_str::{ToCompactString, format_compact}; use hyper::StatusCode; +use registry::schema::enums::ExpressionVariable; use crate::{HttpContext, HttpRequest, HttpSessionData}; @@ -21,7 +22,7 @@ impl<'x> HttpContext<'x> { pub async fn resolve_response_url(&self, server: &Server) -> String { server .eval_if( - &server.core.network.http_response_url, + &server.core.network.http.response_url, self, self.session.session_id, ) @@ -39,7 +40,7 @@ impl<'x> HttpContext<'x> { pub async fn has_endpoint_access(&self, server: &Server) -> StatusCode { server .eval_if( - &server.core.network.http_allowed_endpoint, + &server.core.network.http.allowed_endpoint, self, self.session.session_id, ) diff --git a/crates/nlp/src/language/mod.rs b/crates/nlp/src/language/mod.rs index 1bb3a2ac..47aad013 100644 --- a/crates/nlp/src/language/mod.rs +++ b/crates/nlp/src/language/mod.rs @@ -133,7 +133,7 @@ impl Language { pub fn from_iso_639(code: &str) -> Option { hashify::map!( - code.split_once('-').map(|c| c.0).unwrap_or(code).as_bytes(), + code.split_once(['-', '_']).map(|c| c.0).unwrap_or(code).as_bytes(), Language, "en" => Language::English, "es" => Language::Spanish, diff --git a/crates/registry/src/schema/mod.rs b/crates/registry/src/schema/mod.rs index 0d204254..5c163bac 100644 --- a/crates/registry/src/schema/mod.rs +++ b/crates/registry/src/schema/mod.rs @@ -7,14 +7,17 @@ use crate::{ schema::{ enums::{TracingLevel, TracingLevelOpt}, - prelude::{HttpAuth, NodeRange, Property}, + prelude::{Duration, HttpAuth, NodeRange, Property}, }, types::EnumType, }; use std::{collections::HashMap, fmt::Display}; use utils::{ - HeaderMap, - config::{cron::SimpleCron, http::build_http_headers}, + Client, HeaderMap, + config::{ + cron::SimpleCron, + http::{build_http_client, build_http_headers}, + }, }; #[allow(clippy::derivable_impls)] @@ -79,6 +82,44 @@ impl HttpAuth { ), } } + + pub fn build_http_client( + &self, + extra_headers: HashMap, + content_type: Option<&str>, + timeout: Duration, + allow_invalid_certs: bool, + ) -> Result { + match self { + HttpAuth::Unauthenticated => build_http_client( + extra_headers, + None, + None, + None, + content_type, + timeout.into_inner(), + allow_invalid_certs, + ), + HttpAuth::Basic(auth) => build_http_client( + extra_headers, + auth.username.as_str().into(), + auth.secret.as_str().into(), + None, + content_type, + timeout.into_inner(), + allow_invalid_certs, + ), + HttpAuth::Bearer(auth) => build_http_client( + extra_headers, + None, + None, + auth.bearer_token.as_str().into(), + content_type, + timeout.into_inner(), + allow_invalid_certs, + ), + } + } } impl Display for Property { diff --git a/crates/registry/src/types/datetime.rs b/crates/registry/src/types/datetime.rs index 38ce92fc..9b7131ea 100644 --- a/crates/registry/src/types/datetime.rs +++ b/crates/registry/src/types/datetime.rs @@ -5,7 +5,7 @@ */ use crate::pickle::{Pickle, PickledStream}; -use std::{fmt::Display, str::FromStr}; +use std::{fmt::Display, str::FromStr, time::SystemTime}; #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] #[repr(transparent)] @@ -126,14 +126,24 @@ impl FromStr for UTCDateTime { } impl UTCDateTime { + pub fn now() -> Self { + UTCDateTime( + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()) as i64, + ) + } + pub fn from_timestamp(timestamp: i64) -> Self { UTCDateTime(timestamp) } + #[inline(always)] pub fn timestamp(&self) -> i64 { self.0 } + #[inline(always)] pub fn is_valid(&self) -> bool { self.0 != i64::MAX } diff --git a/crates/registry/src/types/duration.rs b/crates/registry/src/types/duration.rs index 1c038dca..569be6be 100644 --- a/crates/registry/src/types/duration.rs +++ b/crates/registry/src/types/duration.rs @@ -23,6 +23,16 @@ impl Duration { pub fn is_valid(&self) -> bool { self.0.as_millis() > 0 } + + #[inline(always)] + pub fn as_secs(&self) -> u64 { + self.0.as_secs() + } + + #[inline(always)] + pub fn as_millis(&self) -> u64 { + self.0.as_millis() as u64 + } } impl Default for Duration { diff --git a/crates/registry/src/types/error.rs b/crates/registry/src/types/error.rs index 6c21badb..5f3786db 100644 --- a/crates/registry/src/types/error.rs +++ b/crates/registry/src/types/error.rs @@ -4,7 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{schema::prelude::Property, types::id::Id}; +use crate::{ + schema::prelude::{Object, Property}, + types::id::Id, +}; use std::fmt::Display; #[derive(Debug, Clone, PartialEq, Eq)] @@ -31,6 +34,14 @@ pub enum Error { object_id: Option, error: trc::Error, }, + TypeMismatch { + object_id: Id, + object_type: Object, + expected_type: Object, + }, + NotFound { + object_id: Id, + }, } #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/crates/registry/src/types/ipmask.rs b/crates/registry/src/types/ipmask.rs index c67c5a98..4279cd59 100644 --- a/crates/registry/src/types/ipmask.rs +++ b/crates/registry/src/types/ipmask.rs @@ -19,6 +19,19 @@ pub enum IpAddrOrMask { } impl IpAddrOrMask { + pub fn from_ip(ip: IpAddr) -> Self { + match ip { + IpAddr::V4(addr) => IpAddrOrMask::V4 { + addr, + mask: u32::MAX, + }, + IpAddr::V6(addr) => IpAddrOrMask::V6 { + addr, + mask: u128::MAX, + }, + } + } + pub fn is_valid(&self) -> bool { !matches!( self, @@ -26,6 +39,21 @@ impl IpAddrOrMask { ) } + pub fn try_to_ip(&self) -> Option { + match self { + IpAddrOrMask::V4 { addr, mask } if *mask == u32::MAX => Some(IpAddr::V4(*addr)), + IpAddrOrMask::V6 { addr, mask } if *mask == u128::MAX => Some(IpAddr::V6(*addr)), + _ => None, + } + } + + pub fn into_inner(self) -> (IpAddr, u128) { + match self { + IpAddrOrMask::V4 { addr, mask } => (IpAddr::V4(addr), mask as u128), + IpAddrOrMask::V6 { addr, mask } => (IpAddr::V6(addr), mask), + } + } + pub fn matches(&self, remote: &IpAddr) -> bool { match self { IpAddrOrMask::V4 { addr, mask } => match *mask { diff --git a/crates/store/src/backend/azure/mod.rs b/crates/store/src/backend/azure/mod.rs index 950b10dd..ae1b4686 100644 --- a/crates/store/src/backend/azure/mod.rs +++ b/crates/store/src/backend/azure/mod.rs @@ -4,18 +4,17 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{fmt::Display, io::Write, ops::Range, time::Duration}; - use azure_core::error::ErrorKind; use azure_core::{ExponentialRetryOptions, RetryOptions, StatusCode, TransportOptions}; use azure_storage::StorageCredentials; use azure_storage_blobs::prelude::{ClientBuilder, ContainerClient}; use futures::stream::StreamExt; +use registry::schema::structs::{self}; use std::sync::Arc; -use utils::{ - codec::base32_custom::Base32Writer, - config::{Config, utils::AsKey}, -}; +use std::{fmt::Display, io::Write, ops::Range}; +use utils::codec::base32_custom::Base32Writer; + +use crate::BlobStore; pub struct AzureStore { client: ContainerClient, @@ -23,76 +22,45 @@ pub struct AzureStore { } impl AzureStore { - pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { - let prefix = prefix.as_key(); - - let storage_account = config - .value_require((&prefix, "storage-account"))? - .to_string(); - let container = config.value_require((&prefix, "container"))?.to_string(); - - let credentials = match ( - config.value((&prefix, "azure-access-key")), - config.value((&prefix, "sas-token")), - ) { + pub async fn open(config: structs::AzureStore) -> Result { + let credentials = match (config.access_key, config.sas_token) { (Some(access_key), None) => { - StorageCredentials::access_key(storage_account.clone(), access_key.to_string()) + StorageCredentials::access_key(config.storage_account.clone(), access_key) } (None, Some(sas_token)) => match StorageCredentials::sas_token(sas_token) { Ok(cred) => cred, Err(err) => { - config.new_build_error( - prefix.as_str(), - format!("Failed to create credentials: {err:?}"), - ); - return None; + return Err(format!("Failed to create credentials: {err:?}")); } }, _ => { - config.new_build_error( - prefix.as_str(), - concat!( - "Failed to create credentials: exactly one of ", - "'azure-access-key' and 'sas-token' must be specified" - ), - ); - return None; + return Err(concat!( + "Failed to create credentials: exactly one of ", + "'azure-access-key' and 'sas-token' must be specified" + ) + .to_string()); } }; - let timeout = config - .property_or_default::((&prefix, "timeout"), "30s") - .unwrap_or_else(|| Duration::from_secs(30)); - let transport = match reqwest::Client::builder().timeout(timeout).build() { + let transport = match reqwest::Client::builder() + .timeout(config.timeout.into_inner()) + .build() + { Ok(client) => Arc::new(client), Err(err) => { - config.new_build_error( - prefix.as_str(), - format!("Failed to create HTTP client: {err:?}"), - ); - return None; + return Err(format!("Failed to create HTTP client: {err:?}")); } }; - // Take the configured number of retries and multiply by 2. This is intended to match the - // precedent set by the S3 back end, where we do the indicated number of retries, - // ourselves, but internally the rust-s3 crate is also retrying each of our requests up to - // one additional time, itself. So our retries, and the S3 backend's retries, are - // comparable to each other. - let max_retries: u32 = config - .property_or_default((&prefix, "max-retries"), "3") - .unwrap_or(3) - * 2; - - Some(AzureStore { - client: ClientBuilder::new(storage_account, credentials) + Ok(BlobStore::Azure(Arc::new(AzureStore { + client: ClientBuilder::new(config.storage_account, credentials) .transport(TransportOptions::new(transport)) .retry(RetryOptions::exponential( - ExponentialRetryOptions::default().max_retries(max_retries), + ExponentialRetryOptions::default().max_retries(config.max_retries as u32 * 2), )) - .container_client(container), - prefix: config.value((&prefix, "key-prefix")).map(|s| s.to_string()), - }) + .container_client(config.container), + prefix: config.key_prefix, + }))) } pub(crate) async fn get_blob( diff --git a/crates/store/src/backend/composite/read_replica.rs b/crates/store/src/backend/composite/read_replica.rs index 26cdef75..0f14bd9d 100644 --- a/crates/store/src/backend/composite/read_replica.rs +++ b/crates/store/src/backend/composite/read_replica.rs @@ -9,16 +9,18 @@ */ use crate::{ - Deserialize, IterateParams, Key, Store, Stores, ValueKey, + Deserialize, IterateParams, Key, Store, ValueKey, search::{IndexDocument, SearchComparator, SearchDocumentId, SearchFilter, SearchQuery}, write::{AssignedIds, Batch, SearchIndex, ValueClass}, }; use std::{ future::Future, ops::Range, - sync::atomic::{AtomicUsize, Ordering}, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, }; -use utils::config::{Config, utils::AsKey}; pub struct SQLReadReplica { primary: Store, @@ -27,101 +29,12 @@ pub struct SQLReadReplica { } impl SQLReadReplica { - pub async fn open( - config: &mut Config, - prefix: impl AsKey, - stores: &Stores, - create_store_tables: bool, - create_search_tables: bool, - ) -> Option { - let prefix = prefix.as_key(); - let primary_id = config.value_require((&prefix, "primary"))?.to_string(); - let replica_ids = config - .values((&prefix, "replicas")) - .map(|(_, v)| v.to_string()) - .collect::>(); - - let primary = if let Some(store) = stores.stores.get(&primary_id) { - if store.is_pg_or_mysql() { - store.clone() - } else { - config.new_build_error( - (&prefix, "primary"), - "Primary store must be a PostgreSQL or MySQL store", - ); - return None; - } - } else { - config.new_build_error( - (&prefix, "primary"), - format!("Primary store {primary_id} not found"), - ); - return None; - }; - let mut replicas = Vec::with_capacity(replica_ids.len()); - for replica_id in replica_ids { - if let Some(store) = stores.stores.get(&replica_id) { - if store.is_pg_or_mysql() { - replicas.push(store.clone()); - } else { - config.new_build_error( - (&prefix, "replicas"), - "Replica store must be a PostgreSQL or MySQL store", - ); - return None; - } - } else { - config.new_build_error( - (&prefix, "replicas"), - format!("Replica store {replica_id} not found"), - ); - return None; - } - } - if !replicas.is_empty() { - if create_store_tables { - let result = match &primary { - #[cfg(feature = "postgres")] - Store::PostgreSQL(store) => store.create_storage_tables().await, - #[cfg(feature = "mysql")] - Store::MySQL(store) => store.create_storage_tables().await, - _ => panic!("Invalid store type"), - }; - - if let Err(err) = result { - config.new_build_error( - (&prefix, "primary"), - format!("Failed to create tables: {err}"), - ); - } - } - - if create_search_tables { - let result = match &primary { - #[cfg(feature = "postgres")] - Store::PostgreSQL(store) => store.create_search_tables().await, - #[cfg(feature = "mysql")] - Store::MySQL(store) => store.create_search_tables().await, - _ => panic!("Invalid store type"), - }; - - if let Err(err) = result { - config.new_build_warning( - (&prefix, "primary"), - format!("Failed to create search tables: {err}"), - ); - } - } - - Some(Self { - primary, - replicas, - last_used_replica: AtomicUsize::new(0), - }) - } else { - config.new_build_error((&prefix, "replicas"), "No replica stores specified"); - None - } + pub fn open(primary: Store, replicas: Vec) -> Result { + Ok(Store::SQLReadReplica(Arc::new(Self { + primary, + replicas, + last_used_replica: AtomicUsize::new(0), + }))) } async fn run_op<'x, F, T, R>(&'x self, f: F) -> trc::Result @@ -321,4 +234,12 @@ impl SQLReadReplica { _ => panic!("Invalid store type"), } } + + pub fn primary_store(&self) -> &Store { + &self.primary + } + + pub fn into_primary(self) -> Store { + self.primary + } } diff --git a/crates/store/src/backend/composite/sharded_blob.rs b/crates/store/src/backend/composite/sharded_blob.rs index 99f20114..a54b2c68 100644 --- a/crates/store/src/backend/composite/sharded_blob.rs +++ b/crates/store/src/backend/composite/sharded_blob.rs @@ -8,48 +8,66 @@ * */ -use std::ops::Range; - -use utils::config::{Config, utils::AsKey}; - -use crate::{BlobBackend, Store, Stores}; +use crate::{BlobStore, Store, backend::fs::FsStore}; +use registry::schema::structs::{BlobStoreBase, ShardedBlobStore}; +use std::{ops::Range, sync::Arc}; pub struct ShardedBlob { - pub stores: Vec, + pub stores: Vec, } +#[allow(unreachable_patterns)] impl ShardedBlob { - pub fn open(config: &mut Config, prefix: impl AsKey, stores: &Stores) -> Option { - let prefix = prefix.as_key(); - let store_ids = config - .values((&prefix, "stores")) - .map(|(_, v)| v.to_string()) - .collect::>(); + pub async fn open(config: ShardedBlobStore) -> Result { + if config.stores.len() >= 2 { + let mut stores = Vec::new(); - let mut blob_stores = Vec::with_capacity(store_ids.len()); - for store_id in store_ids { - if let Some(store) = stores.blob_stores.get(&store_id) { - blob_stores.push(store.backend.clone()); - } else { - config.new_build_error( - (&prefix, "stores"), - format!("Blob store {store_id} not found"), - ); - return None; + for store in config.stores { + let result = match store { + #[cfg(feature = "s3")] + BlobStoreBase::S3(s3_store) => { + crate::backend::s3::S3Store::open(s3_store).await + } + #[cfg(feature = "azure")] + BlobStoreBase::Azure(azure_store) => { + crate::backend::azure::AzureStore::open(azure_store).await + } + BlobStoreBase::FileSystem(file_system_store) => { + FsStore::open(file_system_store).await + } + #[cfg(feature = "foundation")] + BlobStoreBase::FoundationDb(foundation_db_store) => { + crate::backend::foundationdb::FdbStore::open(foundation_db_store) + .await + .map(BlobStore::Store) + } + #[cfg(feature = "postgres")] + BlobStoreBase::PostgreSql(postgre_sql_store) => { + crate::backend::postgres::PostgresStore::open(postgre_sql_store) + .await + .map(BlobStore::Store) + } + #[cfg(feature = "mysql")] + BlobStoreBase::MySql(my_sql_store) => { + crate::backend::mysql::MysqlStore::open(my_sql_store) + .await + .map(BlobStore::Store) + } + _ => Err( + "Binary was not compiled with the selected blob store backend".to_string(), + ), + }; + + stores.push(result?); } - } - if !blob_stores.is_empty() { - Some(Self { - stores: blob_stores, - }) + Ok(BlobStore::Sharded(Arc::new(ShardedBlob { stores }))) } else { - config.new_build_error((&prefix, "stores"), "No blob stores specified"); - None + Err("At least two blob stores are required for sharded blob store".to_string()) } } #[inline(always)] - fn get_store(&self, key: &[u8]) -> &BlobBackend { + fn get_store(&self, key: &[u8]) -> &BlobStore { &self.stores[xxhash_rust::xxh3::xxh3_64(key) as usize % self.stores.len()] } @@ -58,9 +76,9 @@ impl ShardedBlob { key: &[u8], read_range: Range, ) -> trc::Result>> { - Box::pin(async move { + async move { match self.get_store(key) { - BlobBackend::Store(store) => match store { + BlobStore::Store(store) => match store { #[cfg(feature = "sqlite")] Store::SQLite(store) => store.get_blob(key, read_range).await, #[cfg(feature = "foundation")] @@ -82,21 +100,21 @@ impl ShardedBlob { // SPDX-SnippetEnd Store::None => Err(trc::StoreEvent::NotConfigured.into()), }, - BlobBackend::Fs(store) => store.get_blob(key, read_range).await, + BlobStore::Fs(store) => store.get_blob(key, read_range).await, #[cfg(feature = "s3")] - BlobBackend::S3(store) => store.get_blob(key, read_range).await, + BlobStore::S3(store) => store.get_blob(key, read_range).await, #[cfg(feature = "azure")] - BlobBackend::Azure(store) => store.get_blob(key, read_range).await, - BlobBackend::Sharded(_) => unimplemented!(), + BlobStore::Azure(store) => store.get_blob(key, read_range).await, + BlobStore::Sharded(_) => unimplemented!(), } - }) + } .await } pub async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> { - Box::pin(async move { + async move { match self.get_store(key) { - BlobBackend::Store(store) => match store { + BlobStore::Store(store) => match store { #[cfg(feature = "sqlite")] Store::SQLite(store) => store.put_blob(key, data).await, #[cfg(feature = "foundation")] @@ -118,21 +136,21 @@ impl ShardedBlob { Store::SQLReadReplica(store) => store.put_blob(key, data).await, Store::None => Err(trc::StoreEvent::NotConfigured.into()), }, - BlobBackend::Fs(store) => store.put_blob(key, data).await, + BlobStore::Fs(store) => store.put_blob(key, data).await, #[cfg(feature = "s3")] - BlobBackend::S3(store) => store.put_blob(key, data).await, + BlobStore::S3(store) => store.put_blob(key, data).await, #[cfg(feature = "azure")] - BlobBackend::Azure(store) => store.put_blob(key, data).await, - BlobBackend::Sharded(_) => unimplemented!(), + BlobStore::Azure(store) => store.put_blob(key, data).await, + BlobStore::Sharded(_) => unimplemented!(), } - }) + } .await } pub async fn delete_blob(&self, key: &[u8]) -> trc::Result { - Box::pin(async move { + async move { match self.get_store(key) { - BlobBackend::Store(store) => match store { + BlobStore::Store(store) => match store { #[cfg(feature = "sqlite")] Store::SQLite(store) => store.delete_blob(key).await, #[cfg(feature = "foundation")] @@ -154,14 +172,18 @@ impl ShardedBlob { // SPDX-SnippetEnd Store::None => Err(trc::StoreEvent::NotConfigured.into()), }, - BlobBackend::Fs(store) => store.delete_blob(key).await, + BlobStore::Fs(store) => store.delete_blob(key).await, #[cfg(feature = "s3")] - BlobBackend::S3(store) => store.delete_blob(key).await, + BlobStore::S3(store) => store.delete_blob(key).await, #[cfg(feature = "azure")] - BlobBackend::Azure(store) => store.delete_blob(key).await, - BlobBackend::Sharded(_) => unimplemented!(), + BlobStore::Azure(store) => store.delete_blob(key).await, + BlobStore::Sharded(_) => unimplemented!(), } - }) + } .await } + + pub fn into_single(self) -> BlobStore { + self.stores.into_iter().next().unwrap() + } } diff --git a/crates/store/src/backend/composite/sharded_lookup.rs b/crates/store/src/backend/composite/sharded_lookup.rs index 6e95c694..bfc7ff1e 100644 --- a/crates/store/src/backend/composite/sharded_lookup.rs +++ b/crates/store/src/backend/composite/sharded_lookup.rs @@ -8,49 +8,51 @@ * */ -use utils::config::{Config, utils::AsKey}; +#[cfg(feature = "redis")] +use registry::schema::structs::InMemoryStoreBase; +use registry::schema::structs::ShardedInMemoryStore; use crate::{ - Deserialize, InMemoryStore, Stores, Value, + Deserialize, InMemoryStore, Value, dispatch::lookup::{KeyValue, LookupKey}, }; +use std::sync::Arc; #[derive(Debug)] pub struct ShardedInMemory { pub stores: Vec, } +#[allow(unreachable_patterns)] impl ShardedInMemory { - pub fn open(config: &mut Config, prefix: impl AsKey, stores: &Stores) -> Option { - let prefix = prefix.as_key(); - let store_ids = config - .values((&prefix, "stores")) - .map(|(_, v)| v.to_string()) - .collect::>(); + pub async fn open(config: ShardedInMemoryStore) -> Result { + if config.stores.len() >= 2 { + let mut stores = Vec::new(); - let mut in_memory_stores = Vec::with_capacity(store_ids.len()); - for store_id in store_ids { - if let Some(store) = stores - .in_memory_stores - .get(&store_id) - .filter(|store| store.is_redis()) - { - in_memory_stores.push(store.clone()); - } else { - config.new_build_error( - (&prefix, "stores"), - format!("In-memory store {store_id} not found"), - ); - return None; + for store in config.stores { + let result = match store { + #[cfg(feature = "redis")] + InMemoryStoreBase::Redis(redis_store) => { + crate::backend::redis::RedisStore::open_single(redis_store).await + } + #[cfg(feature = "redis")] + InMemoryStoreBase::RedisCluster(redis_cluster_store) => { + crate::backend::redis::RedisStore::open_cluster(redis_cluster_store).await + } + _ => Err( + "Binary was not compiled with the selected in-memory backend".to_string(), + ), + }; + + stores.push(result?); } - } - if !in_memory_stores.is_empty() { - Some(Self { - stores: in_memory_stores, - }) + + Ok(InMemoryStore::Sharded(Arc::new(ShardedInMemory { stores }))) } else { - config.new_build_error((&prefix, "stores"), "No in-memory stores specified"); - None + Err( + "At least two in-memory stores are required for sharded in-memory store" + .to_string(), + ) } } @@ -174,4 +176,8 @@ impl ShardedInMemory { }) .await } + + pub fn into_single(self) -> InMemoryStore { + self.stores.into_iter().next().unwrap() + } } diff --git a/crates/store/src/backend/elastic/main.rs b/crates/store/src/backend/elastic/main.rs index cf916ff2..10ffef56 100644 --- a/crates/store/src/backend/elastic/main.rs +++ b/crates/store/src/backend/elastic/main.rs @@ -4,72 +4,47 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use std::sync::Arc; + use crate::{ + SearchStore, backend::elastic::ElasticSearchStore, search::{ CalendarSearchField, ContactSearchField, EmailSearchField, SearchableField, TracingSearchField, }, }; +use registry::schema::structs; use reqwest::{Error, Response, Url}; use serde_json::{Value, json}; -use utils::config::{Config, http::build_http_client, utils::AsKey}; impl ElasticSearchStore { - pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { - todo!() - /*let client = build_http_client(config, prefix.clone(), "application/json".into())?; - let prefix = prefix.as_key(); - let url = config - .value_require((&prefix, "url"))? - .trim_end_matches("/") - .to_string(); - Url::parse(&url) - .map_err(|e| config.new_parse_error((&prefix, "url"), format!("Invalid URL: {e}",))) - .ok()?; + pub async fn open(config: structs::ElasticSearchStore) -> Result { + Url::parse(&config.url).map_err(|e| format!("Invalid URL: {e}",))?; - let es = Self { client, url }; - - let shards = config - .property_or_default((&prefix, "index.shards"), "3") - .unwrap_or(3); - let replicas = config - .property_or_default((&prefix, "index.replicas"), "0") - .unwrap_or(0); - let with_source = config - .property_or_default((&prefix, "index.include-source"), "false") - .unwrap_or(false); - - if let Err(err) = es.create_indexes(shards, replicas, with_source).await { - config.new_build_error(prefix.as_str(), err.to_string()); - } - - Some(es)*/ + Ok(SearchStore::ElasticSearch(Arc::new(Self { + client: config.http_auth.build_http_client( + config.http_headers, + "application/json".into(), + config.timeout, + config.allow_invalid_certs, + )?, + url: config.url, + num_replicas: config.num_replicas as usize, + num_shards: config.num_shards as usize, + include_source: config.include_source, + }))) } - pub async fn create_indexes( - &self, - shards: usize, - replicas: usize, - with_source: bool, - ) -> trc::Result<()> { - self.create_index::(shards, replicas, with_source) - .await?; - self.create_index::(shards, replicas, with_source) - .await?; - self.create_index::(shards, replicas, with_source) - .await?; - self.create_index::(shards, replicas, with_source) - .await?; + pub async fn create_indexes(&self) -> trc::Result<()> { + self.create_index::().await?; + self.create_index::().await?; + self.create_index::().await?; + self.create_index::().await?; Ok(()) } - async fn create_index( - &self, - shards: usize, - replicas: usize, - with_source: bool, - ) -> trc::Result<()> { + async fn create_index(&self) -> trc::Result<()> { let mut mappings = serde_json::Map::new(); mappings.insert( "properties".to_string(), @@ -81,14 +56,14 @@ impl ElasticSearchStore { .collect::>(), ), ); - if !with_source { + if !self.include_source { mappings.insert("_source".to_string(), json!({ "enabled": false })); } let body = json!({ "mappings": mappings, "settings": { - "index.number_of_shards": shards, - "index.number_of_replicas": replicas, + "index.number_of_shards": self.num_shards, + "index.number_of_replicas": self.num_replicas, "analysis": { "analyzer": { "default": { diff --git a/crates/store/src/backend/elastic/mod.rs b/crates/store/src/backend/elastic/mod.rs index bb9b0fa4..d0936f93 100644 --- a/crates/store/src/backend/elastic/mod.rs +++ b/crates/store/src/backend/elastic/mod.rs @@ -15,6 +15,9 @@ pub mod search; pub struct ElasticSearchStore { client: Client, url: String, + num_shards: usize, + num_replicas: usize, + include_source: bool, } #[derive(Debug, Deserialize)] diff --git a/crates/store/src/backend/foundationdb/main.rs b/crates/store/src/backend/foundationdb/main.rs index a1afdc36..1c53d818 100644 --- a/crates/store/src/backend/foundationdb/main.rs +++ b/crates/store/src/backend/foundationdb/main.rs @@ -4,108 +4,54 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::time::Duration; - -use foundationdb::{Database, api, options::DatabaseOption}; -use utils::config::{Config, utils::AsKey}; - use super::FdbStore; +use crate::Store; +use foundationdb::{Database, api, options::DatabaseOption}; +use registry::schema::structs; +use std::sync::Arc; impl FdbStore { - pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { - let prefix = prefix.as_key(); + pub async fn open(config: structs::FoundationDbStore) -> Result { let guard = unsafe { api::FdbApiBuilder::default() .build() - .map_err(|err| { - config.new_build_error( - prefix.as_str(), - format!("Failed to boot FoundationDB: {err:?}"), - ) - }) - .ok()? + .map_err(|err| format!("Failed to boot FoundationDB: {err:?}"))? .boot() - .map_err(|err| { - config.new_build_error( - prefix.as_str(), - format!("Failed to boot FoundationDB: {err:?}"), - ) - }) - .ok()? + .map_err(|err| format!("Failed to boot FoundationDB: {err:?}"))? }; - let db = Database::new(config.value((&prefix, "cluster-file"))) - .map_err(|err| { - config.new_build_error( - prefix.as_str(), - format!("Failed to create FoundationDB database: {err:?}"), - ) - }) - .ok()?; + let db = Database::new(config.cluster_file.as_deref()) + .map_err(|err| format!("Failed to create FoundationDB database: {err:?}"))?; - if let Some(value) = config - .property::>((&prefix, "transaction.timeout")) - .unwrap_or_default() - { - db.set_option(DatabaseOption::TransactionTimeout(value.as_millis() as i32)) - .map_err(|err| { - config.new_build_error( - (&prefix, "transaction.timeout"), - format!("Failed to set option: {err:?}"), - ) - }) - .ok()?; - } - if let Some(value) = config.property((&prefix, "transaction.retry-limit")) { - db.set_option(DatabaseOption::TransactionRetryLimit(value)) - .map_err(|err| { - config.new_build_error( - (&prefix, "transaction.retry-limit"), - format!("Failed to set option: {err:?}"), - ) - }) - .ok()?; - } - if let Some(value) = config - .property::>((&prefix, "transaction.max-retry-delay")) - .unwrap_or_default() - { - db.set_option(DatabaseOption::TransactionMaxRetryDelay( - value.as_millis() as i32 + if let Some(value) = config.transaction_timeout { + db.set_option(DatabaseOption::TransactionTimeout( + value.into_inner().as_millis() as i32, )) - .map_err(|err| { - config.new_build_error( - (&prefix, "transaction.max-retry-delay"), - format!("Failed to set option: {err:?}"), - ) - }) - .ok()?; + .map_err(|err| format!("Failed to set option: {err:?}"))?; } - if let Some(value) = config.property((&prefix, "ids.machine")) { + if let Some(value) = config.transaction_retry_limit { + db.set_option(DatabaseOption::TransactionRetryLimit(value as i32)) + .map_err(|err| format!("Failed to set option: {err:?}"))?; + } + if let Some(value) = config.transaction_retry_delay { + db.set_option(DatabaseOption::TransactionMaxRetryDelay( + value.into_inner().as_millis() as i32, + )) + .map_err(|err| format!("Failed to set option: {err:?}"))?; + } + if let Some(value) = config.machine_id { db.set_option(DatabaseOption::MachineId(value)) - .map_err(|err| { - config.new_build_error( - (&prefix, "ids.machine"), - format!("Failed to set option: {err:?}"), - ) - }) - .ok()?; + .map_err(|err| format!("Failed to set option: {err:?}"))?; } - if let Some(value) = config.property((&prefix, "ids.datacenter")) { + if let Some(value) = config.datacenter_id { db.set_option(DatabaseOption::DatacenterId(value)) - .map_err(|err| { - config.new_build_error( - (&prefix, "ids.datacenter"), - format!("Failed to set option: {err:?}"), - ) - }) - .ok()?; + .map_err(|err| format!("Failed to set option: {err:?}"))?; } - Some(Self { + Ok(Store::FoundationDb(Arc::new(Self { guard, db, version: Default::default(), - }) + }))) } } diff --git a/crates/store/src/backend/fs/mod.rs b/crates/store/src/backend/fs/mod.rs index 4d891326..d9a4b5d7 100644 --- a/crates/store/src/backend/fs/mod.rs +++ b/crates/store/src/backend/fs/mod.rs @@ -4,16 +4,14 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{io::SeekFrom, ops::Range, path::PathBuf}; - +use crate::BlobStore; +use registry::schema::structs; +use std::{io::SeekFrom, ops::Range, path::PathBuf, sync::Arc}; use tokio::{ fs::{self, File}, io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}, }; -use utils::{ - codec::base32_custom::Base32Writer, - config::{Config, utils::AsKey}, -}; +use utils::codec::base32_custom::Base32Writer; pub struct FsStore { path: PathBuf, @@ -21,30 +19,18 @@ pub struct FsStore { } impl FsStore { - pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { - let prefix = prefix.as_key(); - let path = PathBuf::from(config.value_require((&prefix, "path"))?); + pub async fn open(config: structs::FileSystemStore) -> Result { + let path = PathBuf::from(&config.path); if !path.exists() { fs::create_dir_all(&path) .await - .map_err(|e| { - config.new_build_error( - (&prefix, "path"), - format!("Failed to create directory: {e}"), - ) - }) - .ok()?; + .map_err(|e| format!("Failed to create directory: {e}"))?; } - Some(FsStore { + Ok(BlobStore::Fs(Arc::new(FsStore { path, - hash_levels: std::cmp::min( - config - .property_or_default((&prefix, "depth"), "2") - .unwrap_or(2), - 5, - ), - }) + hash_levels: std::cmp::min(config.depth as usize, 5), + }))) } pub(crate) async fn get_blob( diff --git a/crates/store/src/backend/http/config.rs b/crates/store/src/backend/http/config.rs index b93efb12..6182c75a 100644 --- a/crates/store/src/backend/http/config.rs +++ b/crates/store/src/backend/http/config.rs @@ -4,92 +4,48 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::{HttpStore, HttpStoreConfig, HttpStoreFormat}; +use crate::{InMemoryStore, LookupStores, registry::bootstrap::Bootstrap}; +use ahash::AHashMap; +use arc_swap::ArcSwap; +use registry::schema::structs::{self, HttpLookupFormat}; use std::{ collections::hash_map::Entry, sync::atomic::{AtomicBool, AtomicU64}, - time::Duration, }; -use ahash::AHashMap; -use arc_swap::ArcSwap; -use utils::config::Config; - -use crate::{InMemoryStore, Stores}; - -use super::{HttpStore, HttpStoreConfig, HttpStoreFormat}; - -impl Stores { - pub fn parse_http_stores(&mut self, config: &mut Config, is_reload: bool) { +impl LookupStores { + pub async fn parse_http(&mut self, bp: &mut Bootstrap) { // Parse remote lists - for id in config.sub_keys("http-lookup", ".url") { - let id_ = id.as_str(); - if !config - .property_or_default(("http-lookup", id_, "enable"), "true") - .unwrap_or(true) - { + for http in bp.list_infallible::().await { + let id = http.id; + let http = http.object; + if !http.enable { continue; } - let format = match config - .value_require(("http-lookup", id_, "format")) - .unwrap_or_default() - { - "list" => HttpStoreFormat::List, - "csv" => HttpStoreFormat::Csv { - index_key: config - .property_require(("http-lookup", id_, "index.key")) - .unwrap_or(0), - index_value: config.property(("http-lookup", id_, "index.value")), - separator: config - .property_or_default::(("http-lookup", id_, "separator"), ",") - .unwrap_or_default() - .chars() - .next() - .unwrap_or(','), - skip_first: config - .property_or_default::(("http-lookup", id_, "skip-first"), "false") - .unwrap_or(false), - }, - other => { - let message = format!("Invalid format: {other:?}"); - config.new_build_error(("http-lookup", id_, "format"), message); - continue; - } - }; - let http_config = HttpStoreConfig { - url: config - .value_require(("http-lookup", id_, "url")) - .unwrap_or_default() - .to_string(), - retry: config - .property_or_default::(("http-lookup", id_, "retry"), "1h") - .unwrap_or(Duration::from_secs(3600)) - .as_secs(), - refresh: config - .property_or_default::(("http-lookup", id_, "refresh"), "12h") - .unwrap_or(Duration::from_secs(43200)) - .as_secs(), - timeout: config - .property_or_default::(("http-lookup", id_, "timeout"), "30s") - .unwrap_or(Duration::from_secs(30)), - gzipped: config - .property_or_default::(("http-lookup", id_, "gzipped"), "false") - .unwrap_or_default(), - max_size: config - .property_or_default::(("http-lookup", id_, "limits.size"), "104857600") - .unwrap_or(104857600), - max_entries: config - .property_or_default::(("http-lookup", id_, "limits.entries"), "100000") - .unwrap_or(100000), - max_entry_size: config - .property_or_default::(("http-lookup", id_, "limits.entry-size"), "512") - .unwrap_or(512), - format, - id, + url: http.url, + retry: http.retry.as_secs(), + refresh: http.refresh.as_secs(), + timeout: http.timeout.into_inner(), + gzipped: http.is_gzipped, + max_size: http.max_size as usize, + max_entries: http.max_entries as usize, + max_entry_size: http.max_entry_size as usize, + format: match http.format { + HttpLookupFormat::List => HttpStoreFormat::List, + HttpLookupFormat::Csv(csv) => HttpStoreFormat::Csv { + index_key: csv.index_key as u32, + index_value: csv.index_value.map(|v| v as u32), + separator: csv.separator.chars().next().unwrap_or(','), + skip_first: csv.skip_first, + }, + }, + id: http.namespace, }; - match self.in_memory_stores.entry(http_config.id.clone()) { + match self.stores.entry(http_config.id.clone()) { Entry::Vacant(entry) => { let store = HttpStore { entries: ArcSwap::from_pointee(AHashMap::new()), @@ -100,13 +56,15 @@ impl Stores { entry.insert(InMemoryStore::Http(store.into())); } - Entry::Occupied(e) if !is_reload => { - config.new_build_error( - ("http-lookup", e.key().as_str()), - "An in-memory store with this id already exists", + Entry::Occupied(_) => { + bp.build_error( + id, + format!( + "An lookup store with the {} namespace already exists", + http_config.id + ), ); } - _ => {} } } } diff --git a/crates/store/src/backend/meili/main.rs b/crates/store/src/backend/meili/main.rs index 9e026ce5..e263cdc2 100644 --- a/crates/store/src/backend/meili/main.rs +++ b/crates/store/src/backend/meili/main.rs @@ -5,58 +5,48 @@ */ use crate::{ + SearchStore, backend::meili::{MeiliSearchStore, Task, TaskStatus, TaskUid}, search::{ CalendarSearchField, ContactSearchField, EmailSearchField, SearchableField, TracingSearchField, }, }; +use registry::schema::structs; use reqwest::{Error, Response, Url}; use serde_json::{Value, json}; -use std::time::Duration; -use utils::config::{Config, http::build_http_client, utils::AsKey}; +use std::{sync::Arc, time::Duration}; impl MeiliSearchStore { - pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { - todo!() - /*let client = build_http_client(config, prefix.clone(), "application/json".into())?; - let prefix = prefix.as_key(); - let url = config - .value_require((&prefix, "url"))? - .trim_end_matches("/") - .to_string(); - Url::parse(&url) - .map_err(|e| config.new_parse_error((&prefix, "url"), format!("Invalid URL: {e}",))) - .ok()?; - let task_poll_interval = config - .property_or_default::((&prefix, "task.poll-interval"), "500ms") - .unwrap_or(Duration::from_millis(500)); - let task_poll_retries = config - .property_or_default::((&prefix, "task.poll-retries"), "120") - .unwrap_or(120); - let task_fail_on_timeout = config - .property_or_default::((&prefix, "task.fail-on-timeout"), "true") - .unwrap_or(true); + pub async fn open(config: structs::MeilisearchStore) -> Result { + let client = config.http_auth.build_http_client( + config.http_headers, + "application/json".into(), + config.timeout, + config.allow_invalid_certs, + )?; + + Url::parse(&config.url).map_err(|e| format!("Invalid URL: {e}",))?; let ms = Self { client, - url, + url: config.url, task_poll_interval: Duration::from_millis(500), task_poll_retries: 120, task_fail_on_timeout: true, }; if let Err(err) = ms.create_indexes().await { - config.new_build_error(prefix.as_str(), err.to_string()); + return Err(format!("Failed to create indexes: {err}")); } - Some(Self { + Ok(SearchStore::MeiliSearch(Arc::new(MeiliSearchStore { client: ms.client, url: ms.url, - task_poll_interval, - task_poll_retries, - task_fail_on_timeout, - })*/ + task_poll_interval: config.poll_interval.into_inner(), + task_poll_retries: config.max_retries as usize, + task_fail_on_timeout: config.fail_on_timeout, + }))) } pub async fn create_indexes(&self) -> trc::Result<()> { diff --git a/crates/store/src/backend/memory/mod.rs b/crates/store/src/backend/memory/mod.rs index 821bd8a5..05e1f9c0 100644 --- a/crates/store/src/backend/memory/mod.rs +++ b/crates/store/src/backend/memory/mod.rs @@ -4,96 +4,60 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::collections::hash_map::Entry; - +use crate::{InMemoryStore, LookupStores, Value, registry::bootstrap::Bootstrap}; use ahash::AHashMap; -use utils::{config::Config, glob::GlobMap}; +use registry::schema::structs; +use utils::glob::{GlobMap, GlobSet}; -use crate::{InMemoryStore, Stores, Value}; +#[derive(Debug)] +pub enum StaticMemoryStore { + Map(GlobMap>), + Set(GlobSet), +} -pub type StaticMemoryStore = GlobMap>; - -impl Stores { - pub fn parse_static_stores(&mut self, config: &mut Config, is_reload: bool) { +impl LookupStores { + pub async fn parse_static(&mut self, bp: &mut Bootstrap) { let mut lookups = AHashMap::new(); - let mut errors = Vec::new(); - for (key, value) in config.iterate_prefix("lookup") { - if let Some((id, key)) = key - .split_once('.') - .filter(|(id, key)| !id.is_empty() && !key.is_empty()) + for lookup in bp.list_infallible::().await { + if let StaticMemoryStore::Map(map) = lookups + .entry(lookup.object.namespace) + .or_insert_with(|| StaticMemoryStore::Map(Default::default())) { - // Detect value type - let value = if !value.is_empty() { - let mut has_integers = false; - let mut has_floats = false; - let mut has_others = false; - - for (pos, ch) in value.as_bytes().iter().enumerate() { - match ch { - b'.' if !has_floats && has_integers => { - has_floats = true; - } - b'0'..=b'9' => { - has_integers = true; - } - b'-' if pos == 0 && value.len() > 1 => {} - _ => { - has_others = true; - } - } - } - - if has_others { - if value == "true" { - Value::Integer(1.into()) - } else if value == "false" { - Value::Integer(0.into()) - } else { - Value::Text(value.to_string().into()) - } - } else if has_floats { - value - .parse() - .map(Value::Float) - .unwrap_or_else(|_| Value::Text(value.to_string().into())) - } else { - value - .parse() - .map(Value::Integer) - .unwrap_or_else(|_| Value::Text(value.to_string().into())) - } + if lookup.object.is_glob_pattern { + map.insert_pattern(&lookup.object.key, Value::from(lookup.object.value)); } else { - Value::Text("".into()) - }; - - // Add entry - lookups - .entry(id.to_string()) - .or_insert_with(StaticMemoryStore::default) - .insert(key, value); + map.insert_entry(lookup.object.key, Value::from(lookup.object.value)); + } } else { - errors.push(key.to_string()); + bp.build_warning( + lookup.id, + "Memory lookup has mixed types (key-value and set)", + ); } } - for error in errors { - config.new_parse_error(error, "Invalid lookup key format"); + for lookup in bp.list_infallible::().await { + if let StaticMemoryStore::Set(set) = lookups + .entry(lookup.object.namespace) + .or_insert_with(|| StaticMemoryStore::Set(Default::default())) + { + if lookup.object.is_glob_pattern { + set.insert_pattern(&lookup.object.key); + } else { + set.insert_entry(lookup.object.key); + } + } else { + bp.build_warning( + lookup.id, + "Memory lookup has mixed types (key-value and set)", + ); + } } - for (id, store) in lookups { - match self.in_memory_stores.entry(id) { - Entry::Vacant(entry) => { - entry.insert(InMemoryStore::Static(store.into())); - } - Entry::Occupied(e) if !is_reload => { - config.new_build_error( - ("lookup", e.key().as_str()), - "An in-memory store with this id already exists", - ); - } - _ => {} - } + for (namespace, store) in lookups { + self.stores + .insert(namespace, InMemoryStore::Static(store.into())); } } } diff --git a/crates/store/src/backend/mysql/main.rs b/crates/store/src/backend/mysql/main.rs index a8ad6a34..99d16864 100644 --- a/crates/store/src/backend/mysql/main.rs +++ b/crates/store/src/backend/mysql/main.rs @@ -4,13 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::time::Duration; - -use mysql_async::{ - Conn, OptsBuilder, Pool, PoolConstraints, PoolOpts, SslOpts, prelude::Queryable, -}; -use utils::config::{Config, utils::AsKey}; - +use super::{MysqlStore, into_error}; use crate::{ backend::mysql::MysqlSearchField, search::{ @@ -19,88 +13,72 @@ use crate::{ }, *, }; - -use super::{MysqlStore, into_error}; +use ::registry::schema::structs; +use mysql_async::{ + Conn, OptsBuilder, Pool, PoolConstraints, PoolOpts, SslOpts, prelude::Queryable, +}; impl MysqlStore { - pub async fn open( - config: &mut Config, - prefix: impl AsKey, - create_store_tables: bool, - create_search_tables: bool, - ) -> Option { - let prefix = prefix.as_key(); + pub async fn open(config: structs::MySqlStore) -> Result { let mut opts = OptsBuilder::default() - .ip_or_hostname(config.value_require((&prefix, "host"))?.to_string()) - .user(config.value((&prefix, "user")).map(|s| s.to_string())) - .pass(config.value((&prefix, "password")).map(|s| s.to_string())) - .db_name( - config - .value_require((&prefix, "database"))? - .to_string() - .into(), - ) - .max_allowed_packet(config.property((&prefix, "max-allowed-packet"))) - .wait_timeout( - config - .property::>((&prefix, "timeout")) - .unwrap_or_default() - .map(|t| t.as_secs() as usize), - ) - .client_found_rows(true); - if let Some(port) = config.property((&prefix, "port")) { - opts = opts.tcp_port(port); - } + .ip_or_hostname(config.host) + .user(config.auth_username) + .pass(config.auth_secret) + .db_name(Some(config.database)) + .max_allowed_packet(config.max_allowed_packet.map(|v| v as usize)) + .wait_timeout(config.timeout.map(|t| t.as_secs() as usize)) + .client_found_rows(true) + .tcp_port(config.port as u16); - if config - .property_or_default::((&prefix, "tls.enable"), "false") - .unwrap_or_default() - { - let allow_invalid = config - .property_or_default::((&prefix, "tls.allow-invalid-certs"), "false") - .unwrap_or_default(); + if config.enable_tls { opts = opts.ssl_opts(Some( SslOpts::default() - .with_danger_accept_invalid_certs(allow_invalid) - .with_danger_skip_domain_validation(allow_invalid), + .with_danger_accept_invalid_certs(config.allow_invalid_certs) + .with_danger_skip_domain_validation(config.allow_invalid_certs), )); } // Configure connection pool let mut pool_min = PoolConstraints::default().min(); let mut pool_max = PoolConstraints::default().max(); - if let Some(n_size) = config - .property::((&prefix, "pool.min-connections")) - .filter(|&n| n > 0) - { - pool_min = n_size; + if let Some(n_size) = config.pool_min_connections { + pool_min = n_size as usize; } - if let Some(n_size) = config - .property::((&prefix, "pool.max-connections")) - .filter(|&n| n > 0) - { - pool_max = n_size; + if let Some(n_size) = config.pool_max_connections { + pool_max = n_size as usize; } opts = opts.pool_opts( PoolOpts::default().with_constraints(PoolConstraints::new(pool_min, pool_max).unwrap()), ); - let db = Self { + let mut replicas = vec![]; + for replica in config.read_replicas { + replicas.push(Store::MySQL(Arc::new(MysqlStore { + conn_pool: Pool::new( + opts.clone() + .ip_or_hostname(replica.host) + .user(replica.auth_username) + .pass(replica.auth_secret) + .db_name(Some(replica.database)) + .tcp_port(replica.port as u16), + ), + }))) + } + + let primary = Store::MySQL(Arc::new(MysqlStore { conn_pool: Pool::new(opts), - }; + })); - if create_store_tables && let Err(err) = db.create_storage_tables().await { - config.new_build_error(prefix.as_str(), format!("Failed to create tables: {err}")); + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + if !replicas.is_empty() { + return backend::composite::read_replica::SQLReadReplica::open(primary, replicas); } + // SPDX-SnippetEnd - if create_search_tables && let Err(err) = db.create_search_tables().await { - config.new_build_warning( - prefix.as_str(), - format!("Failed to create search tables: {err}"), - ); - } - - Some(db) + Ok(primary) } pub(crate) async fn create_storage_tables(&self) -> trc::Result<()> { diff --git a/crates/store/src/backend/postgres/main.rs b/crates/store/src/backend/postgres/main.rs index 9968cff7..fca6dbd8 100644 --- a/crates/store/src/backend/postgres/main.rs +++ b/crates/store/src/backend/postgres/main.rs @@ -13,86 +13,74 @@ use crate::{ }, *, }; +use ::registry::schema::structs; use deadpool::managed::Object; use deadpool_postgres::{Config, Manager, ManagerConfig, PoolConfig, RecyclingMethod, Runtime}; -use nlp::language::Language; -use std::time::Duration; use tokio_postgres::NoTls; -use utils::{config::utils::AsKey, rustls_client_config}; +use utils::rustls_client_config; impl PostgresStore { - pub async fn open( - config: &mut utils::config::Config, - prefix: impl AsKey, - create_store_tables: bool, - create_search_tables: bool, - ) -> Option { - let prefix = prefix.as_key(); + pub async fn open(config: structs::PostgreSqlStore) -> Result { let mut cfg = Config::new(); - cfg.dbname = config - .value_require((&prefix, "database"))? - .to_string() - .into(); - cfg.host = config.value((&prefix, "host")).map(|s| s.to_string()); - cfg.user = config.value((&prefix, "user")).map(|s| s.to_string()); - cfg.password = config.value((&prefix, "password")).map(|s| s.to_string()); - cfg.port = config.property((&prefix, "port")); - cfg.connect_timeout = config - .property::>((&prefix, "timeout")) - .unwrap_or_default(); - cfg.options = config.value((&prefix, "options")).map(|s| s.to_string()); + cfg.dbname = config.database.into(); + cfg.host = config.host.into(); + cfg.user = config.auth_username; + cfg.password = config.auth_secret; + cfg.port = (config.port as u16).into(); + cfg.connect_timeout = config.timeout.map(|t| t.into_inner()); + cfg.options = config.options; cfg.manager = Some(ManagerConfig { recycling_method: RecyclingMethod::Clean, }); - if let Some(max_conn) = config.property::((&prefix, "pool.max-connections")) { - cfg.pool = PoolConfig::new(max_conn).into(); + if let Some(max_conn) = config.pool_max_connections { + cfg.pool = PoolConfig::new(max_conn as usize).into(); } - let mut db = Self { - conn_pool: if config - .property_or_default::((&prefix, "tls.enable"), "false") - .unwrap_or_default() - { + let todo = "implement disabled languages properly"; + + let mut replicas = vec![]; + for replica in config.read_replicas { + let mut cfg = cfg.clone(); + cfg.dbname = replica.database.into(); + cfg.host = replica.host.into(); + cfg.user = replica.auth_username; + cfg.password = replica.auth_secret; + cfg.port = (replica.port as u16).into(); + cfg.options = replica.options; + replicas.push(Store::PostgreSQL(Arc::new(PostgresStore { + conn_pool: if config.enable_tls { + cfg.create_pool( + Some(Runtime::Tokio1), + MakeRustlsConnect::new(rustls_client_config(config.allow_invalid_certs)), + ) + } else { + cfg.create_pool(Some(Runtime::Tokio1), NoTls) + } + .map_err(|e| format!("Failed to create connection pool: {e}"))?, + }))); + } + + let primary = Store::PostgreSQL(Arc::new(PostgresStore { + conn_pool: if config.enable_tls { cfg.create_pool( Some(Runtime::Tokio1), - MakeRustlsConnect::new(rustls_client_config( - config - .property_or_default((&prefix, "tls.allow-invalid-certs"), "false") - .unwrap_or_default(), - )), + MakeRustlsConnect::new(rustls_client_config(config.allow_invalid_certs)), ) } else { cfg.create_pool(Some(Runtime::Tokio1), NoTls) } - .map_err(|e| { - config.new_build_error( - prefix.as_str(), - format!("Failed to create connection pool: {e}"), - ) - }) - .ok()?, - languages: config - .properties::((&prefix, "languages")) - .into_iter() - .map(|(_, v)| v) - .collect(), - }; + .map_err(|e| format!("Failed to create connection pool: {e}"))?, + })); - if db.languages.is_empty() { - db.languages.insert(Language::English); + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + if !replicas.is_empty() { + return backend::composite::read_replica::SQLReadReplica::open(primary, replicas); } + // SPDX-SnippetEnd - if create_store_tables && let Err(err) = db.create_storage_tables().await { - config.new_build_error(prefix.as_str(), format!("Failed to create tables: {err}")); - } - - if create_search_tables && let Err(err) = db.create_search_tables().await { - config.new_build_warning( - prefix.as_str(), - format!("Failed to create search tables: {err}"), - ); - } - - Some(db) + Ok(primary) } pub(crate) async fn create_storage_tables(&self) -> trc::Result<()> { diff --git a/crates/store/src/backend/postgres/mod.rs b/crates/store/src/backend/postgres/mod.rs index 6f08df3b..5152aeb0 100644 --- a/crates/store/src/backend/postgres/mod.rs +++ b/crates/store/src/backend/postgres/mod.rs @@ -11,9 +11,7 @@ use crate::{ }, write::SearchIndex, }; -use ahash::AHashSet; use deadpool_postgres::Pool; -use nlp::language::Language; pub mod blob; pub mod lookup; @@ -25,7 +23,6 @@ pub mod write; pub struct PostgresStore { pub(crate) conn_pool: Pool, - pub(crate) languages: AHashSet, } #[inline(always)] diff --git a/crates/store/src/backend/postgres/search.rs b/crates/store/src/backend/postgres/search.rs index c06c7e96..866f254d 100644 --- a/crates/store/src/backend/postgres/search.rs +++ b/crates/store/src/backend/postgres/search.rs @@ -60,14 +60,7 @@ impl PostgresStore { let value_ref = format!("${}", values.len() + 1); let (text_len, language) = if let SearchValue::Text { value, language } = value { - ( - value.len(), - if self.languages.contains(language) { - pg_lang(language).unwrap_or("simple") - } else { - "simple" - }, - ) + (value.len(), pg_lang(language).unwrap_or("simple")) } else { (0, "simple") }; @@ -194,9 +187,7 @@ impl PostgresStore { query.push(' '); let language = match &value { - SearchValue::Text { language, .. } - if self.languages.contains(language) => - { + SearchValue::Text { language, .. } => { pg_lang(language).unwrap_or("simple") } _ => "simple", diff --git a/crates/store/src/backend/redis/mod.rs b/crates/store/src/backend/redis/mod.rs index 29d451bc..7c4dc7cb 100644 --- a/crates/store/src/backend/redis/mod.rs +++ b/crates/store/src/backend/redis/mod.rs @@ -4,6 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::InMemoryStore; use deadpool::{ Runtime, managed::{Manager, Pool}, @@ -12,8 +13,11 @@ use redis::{ Client, ProtocolVersion, cluster::{ClusterClient, ClusterClientBuilder}, }; -use std::{fmt::Display, time::Duration}; -use utils::config::{Config, utils::AsKey}; +use registry::{ + schema::{enums::RedisProtocol, structs}, + types::duration::Duration, +}; +use std::{fmt::Display, sync::Arc}; pub mod lookup; pub mod pool; @@ -25,12 +29,12 @@ pub struct RedisStore { pub struct RedisConnectionManager { pub client: Client, - timeout: Duration, + timeout: std::time::Duration, } pub struct RedisClusterConnectionManager { pub client: ClusterClient, - timeout: Duration, + timeout: std::time::Duration, } pub enum RedisPool { @@ -39,154 +43,80 @@ pub enum RedisPool { } impl RedisStore { - pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { - let prefix = prefix.as_key(); - let urls = config - .values((&prefix, "urls")) - .map(|(_, v)| v.to_string()) - .collect::>(); - if urls.is_empty() { - config.new_build_error((&prefix, "urls"), "No Redis URLs specified"); - return None; + pub async fn open_single(config: structs::RedisStore) -> Result { + Ok(InMemoryStore::Redis(Arc::new(RedisStore { + pool: RedisPool::Single(build_pool( + RedisConnectionManager { + client: Client::open(config.url) + .map_err(|err| format!("Failed to open Redis client: {err:?}"))?, + timeout: config.timeout.into_inner(), + }, + config.pool_max_connections, + config.pool_timeout_create, + config.pool_timeout_wait, + config.pool_timeout_recycle, + )?), + }))) + } + + pub async fn open_cluster(config: structs::RedisClusterStore) -> Result { + let mut builder = ClusterClientBuilder::new(config.urls.into_iter()); + if let Some(value) = config.auth_username { + builder = builder.username(value); + } + if let Some(value) = config.auth_secret { + builder = builder.password(value); + } + if let Some(value) = config.max_retries { + builder = builder.retries(value as u32); + } + if let Some(value) = config.max_retry_wait { + builder = builder.max_retry_wait(value.as_millis()); + } + if let Some(value) = config.min_retry_wait { + builder = builder.min_retry_wait(value.as_millis()); + } + if config.read_from_replicas { + builder = builder.read_from_replicas(); + } + if matches!(config.protocol_version, RedisProtocol::Resp3) { + builder = builder.use_protocol(ProtocolVersion::RESP3); } - Some( - match config.value((&prefix, "redis-type")).unwrap_or("single") { - "single" => { - let client = Client::open(urls.into_iter().next().unwrap()) - .map_err(|err| { - config.new_build_error( - prefix.as_str(), - format!("Failed to open Redis client: {err:?}"), - ) - }) - .ok()?; - let timeout = config - .property_or_default((&prefix, "timeout"), "10s") - .unwrap_or_default(); + let client = builder + .build() + .map_err(|err| format!("Failed to open Redis client: {err:?}"))?; - Self { - pool: RedisPool::Single( - build_pool(config, &prefix, RedisConnectionManager { client, timeout }) - .map_err(|err| { - config.new_build_error( - prefix.as_str(), - format!("Failed to build Redis pool: {err:?}"), - ) - }) - .ok()?, - ), - } - } - "cluster" => { - let mut builder = ClusterClientBuilder::new(urls.into_iter()); - if let Some(value) = config.property((&prefix, "user")) { - builder = builder.username(value); - } - if let Some(value) = config.property((&prefix, "password")) { - builder = builder.password(value); - } - if let Some(value) = config.property((&prefix, "retry.total")) { - builder = builder.retries(value); - } - if let Some(value) = config - .property::>((&prefix, "retry.max-wait")) - .unwrap_or_default() - { - builder = builder.max_retry_wait(value.as_millis() as u64); - } - if let Some(value) = config - .property::>((&prefix, "retry.min-wait")) - .unwrap_or_default() - { - builder = builder.min_retry_wait(value.as_millis() as u64); - } - if let Some(true) = config.property::((&prefix, "read-from-replicas")) { - builder = builder.read_from_replicas(); - } - if config - .value((&prefix, "protocol-version")) - .unwrap_or("resp2") - == "resp3" - { - builder = builder.use_protocol(ProtocolVersion::RESP3); - } - - let client = builder - .build() - .map_err(|err| { - config.new_build_error( - prefix.as_str(), - format!("Failed to open Redis client: {err:?}"), - ) - }) - .ok()?; - let timeout = config - .property_or_default::((&prefix, "timeout"), "10s") - .unwrap_or_else(|| Duration::from_secs(10)); - - Self { - pool: RedisPool::Cluster( - build_pool( - config, - &prefix, - RedisClusterConnectionManager { client, timeout }, - ) - .map_err(|err| { - config.new_build_error( - prefix.as_str(), - format!("Failed to build Redis pool: {err:?}"), - ) - }) - .ok()?, - ), - } - } - invalid => { - let err = format!("Invalid Redis type {invalid:?}"); - config.new_parse_error((&prefix, "redis-type"), err); - return None; - } - }, - ) + Ok(InMemoryStore::Redis(Arc::new(RedisStore { + pool: RedisPool::Cluster(build_pool( + RedisClusterConnectionManager { + client, + timeout: config.timeout.into_inner(), + }, + config.pool_max_connections, + config.pool_timeout_create, + config.pool_timeout_wait, + config.pool_timeout_recycle, + )?), + }))) } } fn build_pool( - config: &mut Config, - prefix: &str, manager: M, + max_size: u64, + create_timeout: Option, + wait_timeout: Option, + recycle_timeout: Option, ) -> Result, String> { Pool::builder(manager) .runtime(Runtime::Tokio1) - .max_size( - config - .property_or_default((prefix, "pool.max-connections"), "10") - .unwrap_or(10), - ) - .create_timeout( - config - .property_or_default::>((prefix, "pool.create-timeout"), "30s") - .unwrap_or_default(), - ) - .wait_timeout( - config - .property_or_default::>((prefix, "pool.wait-timeout"), "30s") - .unwrap_or_default(), - ) - .recycle_timeout( - config - .property_or_default::>((prefix, "pool.recycle-timeout"), "30s") - .unwrap_or_default(), - ) + .max_size(max_size as usize) + .create_timeout(create_timeout.map(|v| v.into_inner())) + .wait_timeout(wait_timeout.map(|v| v.into_inner())) + .recycle_timeout(recycle_timeout.map(|v| v.into_inner())) .build() - .map_err(|err| { - format!( - "Failed to build pool for {prefix:?}: {err}", - prefix = prefix, - err = err - ) - }) + .map_err(|err| format!("Failed to build pool: {err}")) } #[inline(always)] diff --git a/crates/store/src/backend/rocksdb/main.rs b/crates/store/src/backend/rocksdb/main.rs index b16d6053..d2b246f7 100644 --- a/crates/store/src/backend/rocksdb/main.rs +++ b/crates/store/src/backend/rocksdb/main.rs @@ -6,28 +6,22 @@ use super::{CF_BLOBS, RocksDbStore}; use crate::*; +use ::registry::schema::structs; use rocksdb::{ColumnFamilyDescriptor, MergeOperands, OptimisticTransactionDB, Options}; use std::path::PathBuf; use tokio::sync::oneshot; -use utils::config::{Config, utils::AsKey}; impl RocksDbStore { - pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { - let prefix = prefix.as_key(); + pub async fn open(config: structs::RocksDbStore) -> Result { // Create the database directory if it doesn't exist - let idx_path: PathBuf = PathBuf::from(config.value_require((&prefix, "path"))?); - std::fs::create_dir_all(&idx_path) - .map_err(|err| { - config.new_build_error( - (&prefix, "path"), - format!( - "Failed to create database directory {}: {:?}", - idx_path.display(), - err - ), - ) - }) - .ok()?; + let idx_path: PathBuf = PathBuf::from(config.path); + std::fs::create_dir_all(&idx_path).map_err(|err| { + format!( + "Failed to create database directory {}: {:?}", + idx_path.display(), + err + ) + })?; let mut cfs = Vec::new(); @@ -44,11 +38,7 @@ impl RocksDbStore { // Blobs let mut cf_opts = Options::default(); cf_opts.set_enable_blob_files(true); - cf_opts.set_min_blob_size( - config - .property_or_default((&prefix, "min-blob-size"), "16834") - .unwrap_or(16834), - ); + cf_opts.set_min_blob_size(config.blob_size); cfs.push(ColumnFamilyDescriptor::new(CF_BLOBS, cf_opts)); // Other cfs @@ -93,39 +83,24 @@ impl RocksDbStore { db_opts.set_level_compaction_dynamic_level_bytes(true); //db_opts.set_keep_log_file_num(100); //db_opts.set_max_successive_merges(100); - db_opts.set_write_buffer_size( - config - .property_or_default((&prefix, "write-buffer-size"), "134217728") - .unwrap_or(134217728), - ); + db_opts.set_write_buffer_size(config.buffer_size as usize); - Some(RocksDbStore { + Ok(Store::RocksDb(Arc::new(RocksDbStore { db: OptimisticTransactionDB::open_cf_descriptors(&db_opts, idx_path, cfs) - .map_err(|err| { - config.new_build_error( - prefix.as_str(), - format!("Failed to open database: {:?}", err), - ) - }) - .ok()? + .map_err(|err| format!("Failed to open database: {:?}", err))? .into(), worker_pool: rayon::ThreadPoolBuilder::new() .num_threads(std::cmp::max( config - .property::((&prefix, "pool.workers")) + .pool_workers .filter(|v| *v > 0) + .map(|v| v as usize) .unwrap_or_else(num_cpus::get), 4, )) .build() - .map_err(|err| { - config.new_build_error( - (&prefix, "pool.workers"), - format!("Failed to build worker pool: {:?}", err), - ) - }) - .ok()?, - }) + .map_err(|err| format!("Failed to build worker pool: {:?}", err))?, + }))) } pub async fn spawn_worker(&self, mut f: U) -> trc::Result diff --git a/crates/store/src/backend/s3/mod.rs b/crates/store/src/backend/s3/mod.rs index 1ea9aeef..7a9e18c9 100644 --- a/crates/store/src/backend/s3/mod.rs +++ b/crates/store/src/backend/s3/mod.rs @@ -4,12 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::BlobStore; +use registry::schema::structs; use s3::{Bucket, Region, creds::Credentials}; -use std::{fmt::Display, io::Write, ops::Range, time::Duration}; -use utils::{ - codec::base32_custom::Base32Writer, - config::{Config, utils::AsKey}, -}; +use std::{fmt::Display, io::Write, ops::Range, sync::Arc, time::Duration}; +use utils::codec::base32_custom::Base32Writer; pub struct S3Store { bucket: Box, @@ -18,65 +17,79 @@ pub struct S3Store { } impl S3Store { - pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { + pub async fn open(config: structs::S3Store) -> Result { // Obtain region and endpoint from config - let prefix = prefix.as_key(); - let region = config.value_require((&prefix, "region"))?.to_string(); - let region = if let Some(endpoint) = config.value((&prefix, "endpoint")) { - Region::Custom { - region: region.to_string(), - endpoint: endpoint.to_string(), - } - } else { - region.parse().unwrap() + let region = match config.region { + structs::S3StoreRegion::UsEast1 => Region::UsEast1, + structs::S3StoreRegion::UsEast2 => Region::UsEast2, + structs::S3StoreRegion::UsWest1 => Region::UsWest1, + structs::S3StoreRegion::UsWest2 => Region::UsWest2, + structs::S3StoreRegion::CaCentral1 => Region::CaCentral1, + structs::S3StoreRegion::AfSouth1 => Region::AfSouth1, + structs::S3StoreRegion::ApEast1 => Region::ApEast1, + structs::S3StoreRegion::ApSouth1 => Region::ApSouth1, + structs::S3StoreRegion::ApNortheast1 => Region::ApNortheast1, + structs::S3StoreRegion::ApNortheast2 => Region::ApNortheast2, + structs::S3StoreRegion::ApNortheast3 => Region::ApNortheast3, + structs::S3StoreRegion::ApSoutheast1 => Region::ApSoutheast1, + structs::S3StoreRegion::ApSoutheast2 => Region::ApSoutheast2, + structs::S3StoreRegion::CnNorth1 => Region::CnNorth1, + structs::S3StoreRegion::CnNorthwest1 => Region::CnNorthwest1, + structs::S3StoreRegion::EuNorth1 => Region::EuNorth1, + structs::S3StoreRegion::EuCentral1 => Region::EuCentral1, + structs::S3StoreRegion::EuCentral2 => Region::EuCentral2, + structs::S3StoreRegion::EuWest1 => Region::EuWest1, + structs::S3StoreRegion::EuWest2 => Region::EuWest2, + structs::S3StoreRegion::EuWest3 => Region::EuWest3, + structs::S3StoreRegion::IlCentral1 => Region::IlCentral1, + structs::S3StoreRegion::MeSouth1 => Region::MeSouth1, + structs::S3StoreRegion::SaEast1 => Region::SaEast1, + structs::S3StoreRegion::DoNyc3 => Region::DoNyc3, + structs::S3StoreRegion::DoAms3 => Region::DoAms3, + structs::S3StoreRegion::DoSgp1 => Region::DoSgp1, + structs::S3StoreRegion::DoFra1 => Region::DoFra1, + structs::S3StoreRegion::Yandex => Region::Yandex, + structs::S3StoreRegion::WaUsEast1 => Region::WaUsEast1, + structs::S3StoreRegion::WaUsEast2 => Region::WaUsEast2, + structs::S3StoreRegion::WaUsCentral1 => Region::WaUsCentral1, + structs::S3StoreRegion::WaUsWest1 => Region::WaUsWest1, + structs::S3StoreRegion::WaCaCentral1 => Region::WaCaCentral1, + structs::S3StoreRegion::WaEuCentral1 => Region::WaEuCentral1, + structs::S3StoreRegion::WaEuCentral2 => Region::WaEuCentral2, + structs::S3StoreRegion::WaEuWest1 => Region::WaEuWest1, + structs::S3StoreRegion::WaEuWest2 => Region::WaEuWest2, + structs::S3StoreRegion::WaApNortheast1 => Region::WaApNortheast1, + structs::S3StoreRegion::WaApNortheast2 => Region::WaApNortheast2, + structs::S3StoreRegion::WaApSoutheast1 => Region::WaApSoutheast1, + structs::S3StoreRegion::WaApSoutheast2 => Region::WaApSoutheast2, + structs::S3StoreRegion::Custom(custom) => Region::Custom { + region: custom.custom_region, + endpoint: custom.custom_endpoint, + }, }; let credentials = Credentials::new( - config.value((&prefix, "access-key")), - config.value((&prefix, "secret-key")), - config.value((&prefix, "security-token")), - config.value((&prefix, "session-token")), - config.value((&prefix, "profile")), + config.access_key.as_deref(), + config.secret_key.as_deref(), + config.security_token.as_deref(), + config.session_token.as_deref(), + config.profile.as_deref(), ) - .map_err(|err| { - config.new_build_error( - prefix.as_str(), - format!("Failed to create credentials: {err:?}"), - ) - }) - .ok()?; - let timeout = config - .property_or_default::((&prefix, "timeout"), "30s") - .unwrap_or_else(|| Duration::from_secs(30)); - /*let allow_invalid = config - .property_or_default::((&prefix, "tls.allow-invalid"), "false") - .unwrap_or_default();*/ + .map_err(|err| format!("Failed to create credentials: {err:?}"))?; - Some(S3Store { - bucket: Bucket::new( - config.value_require((&prefix, "bucket"))?, - region, - credentials, - ) - .map_err(|err| { - config.new_build_error(prefix.as_str(), format!("Failed to create bucket: {err:?}")) - }) - .ok()? - .with_path_style() - /*.set_dangereous_config(allow_invalid, allow_invalid) - .map_err(|err| { - config.new_build_error(prefix.as_str(), format!("Failed to create bucket: {err:?}")) - }) - .ok()?*/ - .with_request_timeout(timeout) - .map_err(|err| { - config.new_build_error(prefix.as_str(), format!("Failed to create bucket: {err:?}")) - }) - .ok()?, - max_retries: config - .property_or_default((&prefix, "max-retries"), "3") - .unwrap_or(3), - prefix: config.value((&prefix, "key-prefix")).map(|s| s.to_string()), - }) + Ok(BlobStore::S3(Arc::new(S3Store { + bucket: Bucket::new(&config.bucket, region, credentials) + .map_err(|err| format!("Failed to create bucket: {err:?}"))? + .with_path_style() + /*.set_dangereous_config(allow_invalid, allow_invalid) + .map_err(|err| { + format!("Failed to create bucket: {err:?}") + }) + ?*/ + .with_request_timeout(config.timeout.into_inner()) + .map_err(|err| format!("Failed to create bucket: {err:?}"))?, + max_retries: config.max_retries as u32, + prefix: config.key_prefix, + }))) } pub(crate) async fn get_blob( diff --git a/crates/store/src/backend/sqlite/main.rs b/crates/store/src/backend/sqlite/main.rs index 612f26fc..58e41619 100644 --- a/crates/store/src/backend/sqlite/main.rs +++ b/crates/store/src/backend/sqlite/main.rs @@ -4,65 +4,38 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::{SqliteStore, into_error, pool::SqliteConnectionManager}; +use crate::*; +use ::registry::schema::structs; use r2d2::Pool; use tokio::sync::oneshot; -use utils::config::{Config, utils::AsKey}; - -use crate::*; - -use super::{SqliteStore, into_error, pool::SqliteConnectionManager}; impl SqliteStore { - pub fn open(config: &mut Config, prefix: impl AsKey) -> Option { - let prefix = prefix.as_key(); - let db = Self { + pub fn open(config: structs::SqliteStore) -> Result { + Ok(Store::SQLite(Arc::new(SqliteStore { conn_pool: Pool::builder() - .max_size( - config - .property((&prefix, "pool.max-connections")) - .unwrap_or_else(|| (num_cpus::get() * 4) as u32), - ) - .build( - SqliteConnectionManager::file(config.value_require((&prefix, "path"))?) - .with_init(|c| { - c.execute_batch(concat!( - "PRAGMA journal_mode = WAL; ", - "PRAGMA synchronous = NORMAL; ", - "PRAGMA temp_store = memory;", - "PRAGMA busy_timeout = 30000;" - )) - }), - ) - .map_err(|err| { - config.new_build_error( - prefix.as_str(), - format!("Failed to build connection pool: {err}"), - ) - }) - .ok()?, + .max_size(config.pool_max_connections as u32) + .build(SqliteConnectionManager::file(&config.path).with_init(|c| { + c.execute_batch(concat!( + "PRAGMA journal_mode = WAL; ", + "PRAGMA synchronous = NORMAL; ", + "PRAGMA temp_store = memory;", + "PRAGMA busy_timeout = 30000;" + )) + })) + .map_err(|err| format!("Failed to build connection pool: {err}"))?, worker_pool: rayon::ThreadPoolBuilder::new() .num_threads(std::cmp::max( config - .property::((&prefix, "pool.workers")) + .pool_workers .filter(|v| *v > 0) + .map(|v| v as usize) .unwrap_or_else(num_cpus::get), 4, )) .build() - .map_err(|err| { - config.new_build_error( - prefix.as_str(), - format!("Failed to build worker pool: {err}"), - ) - }) - .ok()?, - }; - - if let Err(err) = db.create_tables() { - config.new_build_error(prefix.as_str(), format!("Failed to create tables: {err}")); - } - - Some(db) + .map_err(|err| format!("Failed to build worker pool: {err}"))?, + }))) } #[cfg(feature = "test_mode")] @@ -85,7 +58,7 @@ impl SqliteStore { Ok(db) } - pub(super) fn create_tables(&self) -> trc::Result<()> { + pub(crate) fn create_tables(&self) -> trc::Result<()> { let conn = self.conn_pool.get().map_err(into_error)?; for table in [ diff --git a/crates/store/src/backend/sqlite/mod.rs b/crates/store/src/backend/sqlite/mod.rs index aa7212f4..4346f68d 100644 --- a/crates/store/src/backend/sqlite/mod.rs +++ b/crates/store/src/backend/sqlite/mod.rs @@ -4,11 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::fmt::Display; - -use r2d2::Pool; - use self::pool::SqliteConnectionManager; +use r2d2::Pool; +use std::fmt::Display; pub mod blob; pub mod lookup; diff --git a/crates/store/src/bootstrap/blob.rs b/crates/store/src/bootstrap/blob.rs new file mode 100644 index 00000000..fd4d1bae --- /dev/null +++ b/crates/store/src/bootstrap/blob.rs @@ -0,0 +1,60 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{BlobStore, backend::fs::FsStore, registry::bootstrap::Bootstrap}; +use registry::schema::{prelude::Object, structs}; + +#[allow(unreachable_patterns)] +impl BlobStore { + pub async fn build(bp: &mut Bootstrap) -> Option { + let result = match bp.setting_infallible::().await { + structs::BlobStore::Default => return None, + #[cfg(feature = "foundation")] + structs::BlobStore::FoundationDb(foundation_db_store) => { + crate::backend::foundationdb::FdbStore::open(foundation_db_store) + .await + .map(BlobStore::Store) + } + #[cfg(feature = "postgres")] + structs::BlobStore::PostgreSql(postgre_sql_store) => { + crate::backend::postgres::PostgresStore::open(postgre_sql_store) + .await + .map(BlobStore::Store) + } + #[cfg(feature = "mysql")] + structs::BlobStore::MySql(my_sql_store) => { + crate::backend::mysql::MysqlStore::open(my_sql_store) + .await + .map(BlobStore::Store) + } + #[cfg(feature = "s3")] + structs::BlobStore::S3(s3_store) => crate::backend::s3::S3Store::open(s3_store).await, + #[cfg(feature = "azure")] + structs::BlobStore::Azure(azure_store) => { + crate::backend::azure::AzureStore::open(azure_store).await + } + structs::BlobStore::FileSystem(file_system_store) => { + FsStore::open(file_system_store).await + } + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + structs::BlobStore::Sharded(store) => { + crate::backend::composite::sharded_blob::ShardedBlob::open(store).await + } // SPDX-SnippetEnd + _ => Err("Binary was not compiled with the selected blob store backend".to_string()), + }; + + match result { + Ok(store) => Some(store), + Err(err) => { + bp.build_error(Object::BlobStore.singleton(), err); + None + } + } + } +} diff --git a/crates/store/src/bootstrap/data.rs b/crates/store/src/bootstrap/data.rs new file mode 100644 index 00000000..a13a023f --- /dev/null +++ b/crates/store/src/bootstrap/data.rs @@ -0,0 +1,31 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::Store; +use registry::schema::structs::DataStore; + +impl Store { + pub async fn build(config: DataStore) -> Result { + #[allow(unreachable_patterns)] + match config { + #[cfg(feature = "rocks")] + DataStore::RocksDb(store) => crate::backend::rocksdb::RocksDbStore::open(store).await, + #[cfg(feature = "foundation")] + DataStore::FoundationDb(store) => { + crate::backend::foundationdb::FdbStore::open(store).await + } + #[cfg(feature = "postgres")] + DataStore::PostgreSql(store) => { + crate::backend::postgres::PostgresStore::open(store).await + } + #[cfg(feature = "mysql")] + DataStore::MySql(store) => crate::backend::mysql::MysqlStore::open(store).await, + #[cfg(feature = "sqlite")] + DataStore::Sqlite(store) => crate::backend::sqlite::SqliteStore::open(store), + _ => Err("Binary was not compiled with the selected data store backend".to_string()), + } + } +} diff --git a/crates/store/src/bootstrap/lookup.rs b/crates/store/src/bootstrap/lookup.rs new file mode 100644 index 00000000..72ebef6a --- /dev/null +++ b/crates/store/src/bootstrap/lookup.rs @@ -0,0 +1,76 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{InMemoryStore, LookupStores, registry::bootstrap::Bootstrap}; +use registry::schema::structs::{LookupStore, StoreLookup}; +use std::collections::hash_map::Entry; + +#[allow(unreachable_patterns)] +impl LookupStores { + pub async fn parse_stores(&mut self, bp: &mut Bootstrap) { + for store in bp.list_infallible::().await { + let id = store.id; + let store = store.object; + + let result = match store.store { + #[cfg(feature = "postgres")] + LookupStore::PostgreSql(postgre_sql_store) => { + crate::backend::postgres::PostgresStore::open(postgre_sql_store) + .await + .map(InMemoryStore::Store) + } + #[cfg(feature = "mysql")] + LookupStore::MySql(my_sql_store) => { + crate::backend::mysql::MysqlStore::open(my_sql_store) + .await + .map(InMemoryStore::Store) + } + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + LookupStore::Sharded(sharded_in_memory_store) => { + crate::backend::composite::sharded_lookup::ShardedInMemory::open( + sharded_in_memory_store, + ) + .await + } + // SPDX-SnippetEnd + #[cfg(feature = "redis")] + LookupStore::Redis(redis_store) => { + crate::backend::redis::RedisStore::open_single(redis_store).await + } + #[cfg(feature = "redis")] + LookupStore::RedisCluster(redis_cluster_store) => { + crate::backend::redis::RedisStore::open_cluster(redis_cluster_store).await + } + _ => Err( + "Binary was not compiled with the selected lookup store backend".to_string(), + ), + }; + + match result { + Ok(lookup) => match self.stores.entry(store.namespace.clone()) { + Entry::Vacant(entry) => { + entry.insert(lookup); + } + Entry::Occupied(_) => { + bp.build_error( + id, + format!( + "An lookup store with the {} namespace already exists", + store.namespace + ), + ); + } + }, + Err(err) => { + bp.build_error(id, err); + } + } + } + } +} diff --git a/crates/store/src/bootstrap/memory.rs b/crates/store/src/bootstrap/memory.rs new file mode 100644 index 00000000..98e04890 --- /dev/null +++ b/crates/store/src/bootstrap/memory.rs @@ -0,0 +1,42 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{InMemoryStore, registry::bootstrap::Bootstrap}; +use registry::schema::{prelude::Object, structs}; + +#[allow(unreachable_patterns)] +impl InMemoryStore { + pub async fn build(bp: &mut Bootstrap) -> Option { + let result = match bp.setting_infallible::().await { + structs::InMemoryStore::Default => return None, + #[cfg(feature = "redis")] + structs::InMemoryStore::Redis(redis_store) => { + crate::backend::redis::RedisStore::open_single(redis_store).await + } + #[cfg(feature = "redis")] + structs::InMemoryStore::RedisCluster(redis_cluster_store) => { + crate::backend::redis::RedisStore::open_cluster(redis_cluster_store).await + } + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + structs::InMemoryStore::Sharded(store) => { + crate::backend::composite::sharded_lookup::ShardedInMemory::open(store).await + } + // SPDX-SnippetEnd + _ => Err("Binary was not compiled with the selected in-memory backend".to_string()), + }; + + match result { + Ok(store) => Some(store), + Err(err) => { + bp.build_error(Object::InMemoryStore.singleton(), err); + None + } + } + } +} diff --git a/crates/store/src/bootstrap/mod.rs b/crates/store/src/bootstrap/mod.rs new file mode 100644 index 00000000..80cda2a0 --- /dev/null +++ b/crates/store/src/bootstrap/mod.rs @@ -0,0 +1,11 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod blob; +pub mod data; +pub mod lookup; +pub mod memory; +pub mod search; diff --git a/crates/store/src/bootstrap/search.rs b/crates/store/src/bootstrap/search.rs new file mode 100644 index 00000000..c4345429 --- /dev/null +++ b/crates/store/src/bootstrap/search.rs @@ -0,0 +1,54 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + SearchStore, + backend::{elastic::ElasticSearchStore, meili::MeiliSearchStore}, + registry::bootstrap::Bootstrap, +}; +use registry::schema::{prelude::Object, structs}; + +#[allow(unreachable_patterns)] +impl SearchStore { + pub async fn build(bp: &mut Bootstrap) -> Option { + let result = match bp.setting_infallible::().await { + structs::SearchStore::Default => return None, + structs::SearchStore::ElasticSearch(elastic_search_store) => { + ElasticSearchStore::open(elastic_search_store).await + } + structs::SearchStore::Meilisearch(meilisearch_store) => { + MeiliSearchStore::open(meilisearch_store).await + } + #[cfg(feature = "foundation")] + structs::SearchStore::FoundationDb(foundation_db_store) => { + crate::backend::foundationdb::FdbStore::open(foundation_db_store) + .await + .map(SearchStore::Store) + } + #[cfg(feature = "postgres")] + structs::SearchStore::PostgreSql(postgre_sql_store) => { + crate::backend::postgres::PostgresStore::open(postgre_sql_store) + .await + .map(SearchStore::Store) + } + #[cfg(feature = "mysql")] + structs::SearchStore::MySql(my_sql_store) => { + crate::backend::mysql::MysqlStore::open(my_sql_store) + .await + .map(SearchStore::Store) + } + _ => Err("Binary was not compiled with the selected search store backend".to_string()), + }; + + match result { + Ok(store) => Some(store), + Err(err) => { + bp.build_error(Object::SearchStore.singleton(), err); + None + } + } + } +} diff --git a/crates/store/src/config.rs b/crates/store/src/config.rs deleted file mode 100644 index e9f378d2..00000000 --- a/crates/store/src/config.rs +++ /dev/null @@ -1,467 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use crate::{ - BlobStore, CompressionAlgo, InMemoryStore, PurgeSchedule, PurgeStore, Store, Stores, - backend::{elastic::ElasticSearchStore, fs::FsStore, meili::MeiliSearchStore}, -}; -use utils::config::{Config, cron::SimpleCron, utils::ParseValue}; - -// SPDX-SnippetBegin -// SPDX-FileCopyrightText: 2020 Stalwart Labs LLC -// SPDX-License-Identifier: LicenseRef-SEL -#[cfg(feature = "enterprise")] -enum CompositeStore { - #[cfg(any(feature = "postgres", feature = "mysql"))] - SQLReadReplica(String), - ShardedBlob(String), - ShardedInMemory(String), -} -// SPDX-SnippetEnd - -impl Stores { - pub async fn parse_all(config: &mut Config, is_reload: bool) -> Self { - let mut stores = Self::parse(config).await; - stores.parse_in_memory(config, is_reload).await; - stores - } - - pub async fn parse(config: &mut Config) -> Self { - let mut stores = Self::default(); - stores.parse_stores(config).await; - stores - } - - pub async fn parse_stores(&mut self, config: &mut Config) { - let is_reload = !self.stores.is_empty(); - - // SPDX-SnippetBegin - // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - // SPDX-License-Identifier: LicenseRef-SEL - #[cfg(feature = "enterprise")] - let mut composite_stores = Vec::new(); - // SPDX-SnippetEnd - - for store_id in config.sub_keys("store", ".type") { - let id = store_id.as_str(); - // Parse store - #[cfg(feature = "test_mode")] - { - if config - .property_or_default::(("store", id, "disable"), "false") - .unwrap_or(false) - { - continue; - } - } - let protocol = if let Some(protocol) = config.value_require(("store", id, "type")) { - protocol.to_ascii_lowercase() - } else { - continue; - }; - let prefix = ("store", id); - let compression_algo = config - .property_or_default::(("store", id, "compression"), "none") - .unwrap_or(CompressionAlgo::None); - - match protocol.as_str() { - #[cfg(feature = "rocks")] - "rocksdb" => { - // Avoid opening the same store twice - if is_reload - && self - .stores - .values() - .any(|store| matches!(store, Store::RocksDb(_))) - { - continue; - } - - if let Some(db) = crate::backend::rocksdb::RocksDbStore::open(config, prefix) - .await - .map(Store::from) - { - self.stores.insert(store_id.clone(), db.clone()); - self.search_stores - .insert(store_id.clone(), db.clone().into()); - self.blob_stores.insert( - store_id.clone(), - BlobStore::from(db.clone()).with_compression(compression_algo), - ); - self.in_memory_stores.insert(store_id, db.into()); - } - } - #[cfg(feature = "foundation")] - "foundationdb" => { - // Avoid opening the same store twice - if is_reload - && self - .stores - .values() - .any(|store| matches!(store, Store::FoundationDb(_))) - { - continue; - } - - if let Some(db) = crate::backend::foundationdb::FdbStore::open(config, prefix) - .await - .map(Store::from) - { - self.stores.insert(store_id.clone(), db.clone()); - self.search_stores - .insert(store_id.clone(), db.clone().into()); - self.blob_stores.insert( - store_id.clone(), - BlobStore::from(db.clone()).with_compression(compression_algo), - ); - self.in_memory_stores.insert(store_id, db.into()); - } - } - #[cfg(feature = "postgres")] - "postgresql" => { - if let Some(db) = crate::backend::postgres::PostgresStore::open( - config, - prefix, - config.is_active_store(id), - config.is_active_search_store(id), - ) - .await - .map(Store::from) - { - self.stores.insert(store_id.clone(), db.clone()); - self.search_stores - .insert(store_id.clone(), db.clone().into()); - self.blob_stores.insert( - store_id.clone(), - BlobStore::from(db.clone()).with_compression(compression_algo), - ); - self.in_memory_stores.insert(store_id.clone(), db.into()); - } - } - #[cfg(feature = "mysql")] - "mysql" => { - if let Some(db) = crate::backend::mysql::MysqlStore::open( - config, - prefix, - config.is_active_store(id), - config.is_active_search_store(id), - ) - .await - .map(Store::from) - { - self.stores.insert(store_id.clone(), db.clone()); - self.search_stores - .insert(store_id.clone(), db.clone().into()); - self.blob_stores.insert( - store_id.clone(), - BlobStore::from(db.clone()).with_compression(compression_algo), - ); - self.in_memory_stores.insert(store_id.clone(), db.into()); - } - } - #[cfg(feature = "sqlite")] - "sqlite" => { - // Avoid opening the same store twice - if is_reload - && self - .stores - .values() - .any(|store| matches!(store, Store::SQLite(_))) - { - continue; - } - - if let Some(db) = - crate::backend::sqlite::SqliteStore::open(config, prefix).map(Store::from) - { - self.stores.insert(store_id.clone(), db.clone()); - self.search_stores - .insert(store_id.clone(), db.clone().into()); - self.blob_stores.insert( - store_id.clone(), - BlobStore::from(db.clone()).with_compression(compression_algo), - ); - self.in_memory_stores.insert(store_id.clone(), db.into()); - } - } - "fs" => { - if let Some(db) = FsStore::open(config, prefix).await.map(BlobStore::from) { - self.blob_stores - .insert(store_id, db.with_compression(compression_algo)); - } - } - #[cfg(feature = "s3")] - "s3" => { - if let Some(db) = crate::backend::s3::S3Store::open(config, prefix) - .await - .map(BlobStore::from) - { - self.blob_stores - .insert(store_id, db.with_compression(compression_algo)); - } - } - "elasticsearch" => { - if let Some(db) = ElasticSearchStore::open(config, prefix) - .await - .map(crate::SearchStore::from) - { - self.search_stores.insert(store_id, db); - } - } - "meilisearch" => { - if let Some(db) = MeiliSearchStore::open(config, prefix) - .await - .map(crate::SearchStore::from) - { - self.search_stores.insert(store_id, db); - } - } - #[cfg(feature = "redis")] - "redis" => { - if let Some(db) = crate::backend::redis::RedisStore::open(config, prefix) - .await - .map(std::sync::Arc::new) - { - self.in_memory_stores - .insert(store_id.clone(), InMemoryStore::Redis(db.clone())); - //self.pubsub_stores - // .insert(store_id, crate::PubSubStore::Redis(db)); - } - } - /*#[cfg(feature = "nats")] - "nats" => { - if let Some(db) = crate::backend::nats::NatsPubSub::open(config, prefix) - .await - .map(std::sync::Arc::new) - { - self.pubsub_stores - .insert(store_id, crate::PubSubStore::Nats(db)); - } - } - #[cfg(feature = "zenoh")] - "zenoh" => { - if let Some(db) = crate::backend::zenoh::ZenohPubSub::open(config, prefix) - .await - .map(std::sync::Arc::new) - { - self.pubsub_stores - .insert(store_id, crate::PubSubStore::Zenoh(db)); - } - } - #[cfg(feature = "kafka")] - "kafka" => { - if let Some(db) = crate::backend::kafka::KafkaPubSub::open(config, prefix) - .await - .map(std::sync::Arc::new) - { - self.pubsub_stores - .insert(store_id, crate::PubSubStore::Kafka(db)); - } - }*/ - // SPDX-SnippetBegin - // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - // SPDX-License-Identifier: LicenseRef-SEL - #[cfg(feature = "enterprise")] - "sql-read-replica" => { - #[cfg(any(feature = "postgres", feature = "mysql"))] - composite_stores.push(CompositeStore::SQLReadReplica(store_id)); - } - #[cfg(feature = "enterprise")] - "distributed-blob" | "sharded-blob" => { - composite_stores.push(CompositeStore::ShardedBlob(store_id)); - } - #[cfg(feature = "enterprise")] - "sharded-in-memory" => { - composite_stores.push(CompositeStore::ShardedInMemory(store_id)); - } - // SPDX-SnippetEnd - #[cfg(feature = "azure")] - "azure" => { - if let Some(db) = crate::backend::azure::AzureStore::open(config, prefix) - .await - .map(BlobStore::from) - { - self.blob_stores - .insert(store_id, db.with_compression(compression_algo)); - } - } - unknown => { - config.new_parse_warning( - ("store", id, "type"), - format!("Unknown directory type: {unknown:?}"), - ); - } - } - } - - // SPDX-SnippetBegin - // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - // SPDX-License-Identifier: LicenseRef-SEL - #[cfg(feature = "enterprise")] - for composite_store in composite_stores { - match composite_store { - #[cfg(any(feature = "postgres", feature = "mysql"))] - CompositeStore::SQLReadReplica(id) => { - let prefix = ("store", id.as_str()); - if let Some(db) = crate::backend::composite::read_replica::SQLReadReplica::open( - config, - prefix, - self, - config.is_active_store(&id), - config.is_active_search_store(&id), - ) - .await - { - let db = Store::SQLReadReplica(db.into()); - self.stores.insert(id.to_string(), db.clone()); - self.search_stores.insert(id.to_string(), db.clone().into()); - self.blob_stores.insert( - id.to_string(), - BlobStore::from(db.clone()).with_compression( - config - .property_or_default::( - ("store", id.as_str(), "compression"), - "none", - ) - .unwrap_or(CompressionAlgo::None), - ), - ); - self.in_memory_stores.insert(id, db.into()); - } - } - CompositeStore::ShardedBlob(id) => { - let prefix = ("store", id.as_str()); - if let Some(db) = crate::backend::composite::sharded_blob::ShardedBlob::open( - config, prefix, self, - ) { - let store = BlobStore { - backend: crate::BlobBackend::Sharded(db.into()), - compression: config - .property_or_default::( - ("store", id.as_str(), "compression"), - "none", - ) - .unwrap_or(CompressionAlgo::None), - }; - self.blob_stores.insert(id, store); - } - } - CompositeStore::ShardedInMemory(id) => { - let prefix = ("store", id.as_str()); - if let Some(db) = - crate::backend::composite::sharded_lookup::ShardedInMemory::open( - config, prefix, self, - ) - { - self.in_memory_stores - .insert(id, InMemoryStore::Sharded(db.into())); - } - } - } - } - - // SPDX-SnippetEnd - } - - pub async fn parse_in_memory(&mut self, config: &mut Config, is_reload: bool) { - // Parse memory stores - self.parse_static_stores(config, is_reload); - - // Parse http stores - self.parse_http_stores(config, is_reload); - - // Parse purge schedules - if let Some(store) = config - .value("storage.data") - .and_then(|store_id| self.stores.get(store_id)) - { - let store_id = config.value("storage.data").unwrap().to_string(); - self.purge_schedules.push(PurgeSchedule { - cron: config - .property_or_default::( - ("store", store_id.as_str(), "purge.frequency"), - "0 3 *", - ) - .unwrap_or_else(|| SimpleCron::parse_value("0 3 *").unwrap()), - store_id, - store: PurgeStore::Data(store.clone()), - }); - - if let Some(blob_store) = config - .value("storage.blob") - .and_then(|blob_store_id| self.blob_stores.get(blob_store_id)) - { - let store_id = config.value("storage.blob").unwrap().to_string(); - self.purge_schedules.push(PurgeSchedule { - cron: config - .property_or_default::( - ("store", store_id.as_str(), "purge.frequency"), - "0 4 *", - ) - .unwrap_or_else(|| SimpleCron::parse_value("0 4 *").unwrap()), - store_id, - store: PurgeStore::Blobs { - store: store.clone(), - blob_store: blob_store.clone(), - }, - }); - } - } - for (store_id, store) in &self.in_memory_stores { - if matches!(store, InMemoryStore::Store(_)) - && config.is_active_in_memory_store(store_id) - { - self.purge_schedules.push(PurgeSchedule { - cron: config - .property_or_default::( - ("store", store_id.as_str(), "purge.frequency"), - "0 5 *", - ) - .unwrap_or_else(|| SimpleCron::parse_value("0 5 *").unwrap()), - store_id: store_id.clone(), - store: PurgeStore::Lookup(store.clone()), - }); - } - } - } -} - -#[allow(dead_code)] -trait IsActiveStore { - fn is_active_store(&self, id: &str) -> bool; - fn is_active_in_memory_store(&self, id: &str) -> bool; - fn is_active_search_store(&self, id: &str) -> bool; -} - -impl IsActiveStore for Config { - fn is_active_store(&self, id: &str) -> bool { - for key in [ - "storage.data", - "storage.blob", - "storage.lookup", - "tracing.history.store", - "metrics.history.store", - ] { - if let Some(store_id) = self.value(key) - && store_id == id - { - return true; - } - } - - false - } - - fn is_active_search_store(&self, id: &str) -> bool { - self.value("storage.fts") - .is_some_and(|store_id| store_id == id) - } - - fn is_active_in_memory_store(&self, id: &str) -> bool { - self.value("storage.lookup") - .is_some_and(|store_id| store_id == id) - } -} diff --git a/crates/store/src/dispatch/blob.rs b/crates/store/src/dispatch/blob.rs index 858f5e27..a38b169e 100644 --- a/crates/store/src/dispatch/blob.rs +++ b/crates/store/src/dispatch/blob.rs @@ -4,140 +4,145 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{borrow::Cow, ops::Range, time::Instant}; - +use crate::{BlobStore, CompressionAlgo, Store}; +use std::{ops::Range, time::Instant}; use trc::{AddContext, StoreEvent}; -use utils::config::utils::ParseValue; -use crate::{BlobBackend, BlobStore, CompressionAlgo, Store}; +const MAGIC_MARKER: u8 = 0xa0; +const LZ4_MARKER: u8 = MAGIC_MARKER | 0x01; +//const ZSTD_MARKER: u8 = MAGIC_MARKER | 0x02; +const NONE_MARKER: u8 = 0x00; impl BlobStore { pub async fn get_blob(&self, key: &[u8], range: Range) -> trc::Result>> { - let read_range = match self.compression { - CompressionAlgo::None => range.clone(), - CompressionAlgo::Lz4 => 0..usize::MAX, - }; let start_time = Instant::now(); - let result = match &self.backend { - BlobBackend::Store(store) => match store { + let result = match &self { + BlobStore::Store(store) => match store { #[cfg(feature = "sqlite")] - Store::SQLite(store) => store.get_blob(key, read_range).await, + Store::SQLite(store) => store.get_blob(key, 0..usize::MAX).await, #[cfg(feature = "foundation")] - Store::FoundationDb(store) => store.get_blob(key, read_range).await, + Store::FoundationDb(store) => store.get_blob(key, 0..usize::MAX).await, #[cfg(feature = "postgres")] - Store::PostgreSQL(store) => store.get_blob(key, read_range).await, + Store::PostgreSQL(store) => store.get_blob(key, 0..usize::MAX).await, #[cfg(feature = "mysql")] - Store::MySQL(store) => store.get_blob(key, read_range).await, + Store::MySQL(store) => store.get_blob(key, 0..usize::MAX).await, #[cfg(feature = "rocks")] - Store::RocksDb(store) => store.get_blob(key, read_range).await, + Store::RocksDb(store) => store.get_blob(key, 0..usize::MAX).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] - Store::SQLReadReplica(store) => store.get_blob(key, read_range).await, + Store::SQLReadReplica(store) => store.get_blob(key, 0..usize::MAX).await, // SPDX-SnippetEnd Store::None => Err(trc::StoreEvent::NotConfigured.into()), }, - BlobBackend::Fs(store) => store.get_blob(key, read_range).await, + BlobStore::Fs(store) => store.get_blob(key, 0..usize::MAX).await, #[cfg(feature = "s3")] - BlobBackend::S3(store) => store.get_blob(key, read_range).await, + BlobStore::S3(store) => store.get_blob(key, 0..usize::MAX).await, #[cfg(feature = "azure")] - BlobBackend::Azure(store) => store.get_blob(key, read_range).await, + BlobStore::Azure(store) => store.get_blob(key, 0..usize::MAX).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] - BlobBackend::Sharded(store) => store.get_blob(key, read_range).await, + BlobStore::Sharded(store) => store.get_blob(key, 0..usize::MAX).await, // SPDX-SnippetEnd - }; + } + .caused_by(trc::location!())?; trc::event!( Store(StoreEvent::BlobRead), Key = key, Elapsed = start_time.elapsed(), - Size = result - .as_ref() - .map_or(0, |data| data.as_ref().map_or(0, |data| data.len())), + Size = result.as_ref().map_or(0, |data| data.len()), ); - let decompressed = match self.compression { - CompressionAlgo::Lz4 => match result.caused_by(trc::location!())? { - Some(data) - if data.last().copied().unwrap_or_default() - == CompressionAlgo::Lz4.marker() => - { - lz4_flex::decompress_size_prepended( - data.get(..data.len() - 1).unwrap_or_default(), - ) + let Some(data) = result else { + return Ok(None); + }; + + let data = match data.last().copied() { + Some(LZ4_MARKER) => { + lz4_flex::decompress_size_prepended(data.get(..data.len() - 1).unwrap_or_default()) .map_err(|err| { trc::StoreEvent::DecompressError .reason(err) .ctx(trc::Key::Key, key) .ctx(trc::Key::CausedBy, trc::location!()) })? - } - Some(data) => { - trc::event!(Store(StoreEvent::BlobMissingMarker), Key = key,); - data - } - None => return Ok(None), - }, - _ => return result, + } + Some(NONE_MARKER) => { + trc::event!(Store(StoreEvent::BlobMissingMarker), Key = key); + data + } + Some(_) => data, + None => { + return Ok(Some(data)); + } }; - if range.end > decompressed.len() { - Ok(Some(decompressed)) + if range.end > data.len() { + Ok(Some(data)) } else { Ok(Some( - decompressed - .get(range.start..range.end) + data.get(range.start..range.end) .unwrap_or_default() .to_vec(), )) } } - pub async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> { - let data: Cow<[u8]> = match self.compression { - CompressionAlgo::None => data.into(), + pub async fn put_blob( + &self, + key: &[u8], + data: &[u8], + compression: CompressionAlgo, + ) -> trc::Result<()> { + let data = match compression { + CompressionAlgo::None => { + let mut uncompressed = Vec::with_capacity(data.len() + 1); + uncompressed.extend_from_slice(data); + uncompressed.push(NONE_MARKER); + uncompressed + } CompressionAlgo::Lz4 => { let mut compressed = lz4_flex::compress_prepend_size(data); - compressed.push(CompressionAlgo::Lz4.marker()); - compressed.into() + compressed.push(LZ4_MARKER); + compressed } }; let start_time = Instant::now(); - let result = match &self.backend { - BlobBackend::Store(store) => match store { + let result = match &self { + BlobStore::Store(store) => match store { #[cfg(feature = "sqlite")] - Store::SQLite(store) => store.put_blob(key, data.as_ref()).await, + Store::SQLite(store) => store.put_blob(key, &data).await, #[cfg(feature = "foundation")] - Store::FoundationDb(store) => store.put_blob(key, data.as_ref()).await, + Store::FoundationDb(store) => store.put_blob(key, &data).await, #[cfg(feature = "postgres")] - Store::PostgreSQL(store) => store.put_blob(key, data.as_ref()).await, + Store::PostgreSQL(store) => store.put_blob(key, &data).await, #[cfg(feature = "mysql")] - Store::MySQL(store) => store.put_blob(key, data.as_ref()).await, + Store::MySQL(store) => store.put_blob(key, &data).await, #[cfg(feature = "rocks")] - Store::RocksDb(store) => store.put_blob(key, data.as_ref()).await, + Store::RocksDb(store) => store.put_blob(key, &data).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] - Store::SQLReadReplica(store) => store.put_blob(key, data.as_ref()).await, + Store::SQLReadReplica(store) => store.put_blob(key, &data).await, // SPDX-SnippetEnd Store::None => Err(trc::StoreEvent::NotConfigured.into()), }, - BlobBackend::Fs(store) => store.put_blob(key, data.as_ref()).await, + BlobStore::Fs(store) => store.put_blob(key, &data).await, #[cfg(feature = "s3")] - BlobBackend::S3(store) => store.put_blob(key, data.as_ref()).await, + BlobStore::S3(store) => store.put_blob(key, &data).await, #[cfg(feature = "azure")] - BlobBackend::Azure(store) => store.put_blob(key, data.as_ref()).await, + BlobStore::Azure(store) => store.put_blob(key, &data).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] - BlobBackend::Sharded(store) => store.put_blob(key, data.as_ref()).await, + BlobStore::Sharded(store) => store.put_blob(key, &data).await, // SPDX-SnippetEnd } .caused_by(trc::location!()); @@ -154,8 +159,8 @@ impl BlobStore { pub async fn delete_blob(&self, key: &[u8]) -> trc::Result { let start_time = Instant::now(); - let result = match &self.backend { - BlobBackend::Store(store) => match store { + let result = match &self { + BlobStore::Store(store) => match store { #[cfg(feature = "sqlite")] Store::SQLite(store) => store.delete_blob(key).await, #[cfg(feature = "foundation")] @@ -174,16 +179,16 @@ impl BlobStore { // SPDX-SnippetEnd Store::None => Err(trc::StoreEvent::NotConfigured.into()), }, - BlobBackend::Fs(store) => store.delete_blob(key).await, + BlobStore::Fs(store) => store.delete_blob(key).await, #[cfg(feature = "s3")] - BlobBackend::S3(store) => store.delete_blob(key).await, + BlobStore::S3(store) => store.delete_blob(key).await, #[cfg(feature = "azure")] - BlobBackend::Azure(store) => store.delete_blob(key).await, + BlobStore::Azure(store) => store.delete_blob(key).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] - BlobBackend::Sharded(store) => store.delete_blob(key).await, + BlobStore::Sharded(store) => store.delete_blob(key).await, // SPDX-SnippetEnd } .caused_by(trc::location!()); @@ -196,34 +201,4 @@ impl BlobStore { result } - - pub fn with_compression(self, compression: CompressionAlgo) -> Self { - Self { - backend: self.backend, - compression, - } - } -} - -const MAGIC_MARKER: u8 = 0xa0; - -impl CompressionAlgo { - pub fn marker(&self) -> u8 { - match self { - CompressionAlgo::Lz4 => MAGIC_MARKER | 0x01, - //CompressionAlgo::Zstd => MAGIC_MARKER | 0x02, - CompressionAlgo::None => 0, - } - } -} - -impl ParseValue for CompressionAlgo { - fn parse_value(value: &str) -> Result { - match value { - "lz4" => Ok(CompressionAlgo::Lz4), - //"zstd" => Ok(CompressionAlgo::Zstd), - "none" | "false" | "disable" | "disabled" => Ok(CompressionAlgo::None), - algo => Err(format!("Invalid compression algorithm: {algo}",)), - } - } } diff --git a/crates/store/src/dispatch/lookup.rs b/crates/store/src/dispatch/lookup.rs index 52f086f8..1fe54ce8 100644 --- a/crates/store/src/dispatch/lookup.rs +++ b/crates/store/src/dispatch/lookup.rs @@ -4,10 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use registry::schema::structs::Rate; use std::borrow::Cow; - use trc::AddContext; -use utils::config::Rate; #[allow(unused_imports)] use crate::{ @@ -20,7 +19,7 @@ use crate::{ }; use crate::{ SerializeInfallible, - backend::http::lookup::HttpStoreGet, + backend::{http::lookup::HttpStoreGet, memory::StaticMemoryStore}, write::{InMemoryClass, assert::AssertValue}, }; @@ -226,9 +225,18 @@ impl InMemoryStore { #[cfg(feature = "enterprise")] InMemoryStore::Sharded(store) => store.key_get(key).await, // SPDX-SnippetEnd - InMemoryStore::Static(store) => Ok(store - .get(key.into().as_str()) - .map(|value| T::from(value.clone()))), + InMemoryStore::Static(store) => Ok(match store.as_ref() { + StaticMemoryStore::Map(map) => map + .get(key.into().as_str()) + .map(|value| T::from(value.clone())), + StaticMemoryStore::Set(set) => { + if set.contains(key.into().as_str()) { + Some(T::from(Value::Bool(true))) + } else { + None + } + } + }), InMemoryStore::Http(store) => { Ok(store.get(key.into().as_str()).map(|value| T::from(value))) } @@ -276,7 +284,10 @@ impl InMemoryStore { #[cfg(feature = "enterprise")] InMemoryStore::Sharded(store) => store.key_exists(key).await, // SPDX-SnippetEnd - InMemoryStore::Static(store) => Ok(store.get(key.into().as_str()).is_some()), + InMemoryStore::Static(store) => Ok(match store.as_ref() { + StaticMemoryStore::Map(map) => map.get(key.into().as_str()).is_some(), + StaticMemoryStore::Set(set) => set.contains(key.into().as_str()), + }), InMemoryStore::Http(store) => Ok(store.contains(key.into().as_str())), } .caused_by(trc::location!()) @@ -307,7 +318,7 @@ impl InMemoryStore { self.counter_get(bucket).await.caused_by(trc::location!())? + 1 }; - if requests <= rate.requests as i64 { + if requests <= rate.count as i64 { Ok(None) } else { Ok(Some(expires_in)) diff --git a/crates/store/src/dispatch/registry.rs b/crates/store/src/dispatch/registry.rs index 247a5e04..9d749b56 100644 --- a/crates/store/src/dispatch/registry.rs +++ b/crates/store/src/dispatch/registry.rs @@ -15,7 +15,11 @@ impl RegistryStore { todo!() } - pub async fn put(&self, id: Id, object: &T) -> trc::Result> { + pub async fn insert(&self, object: &T) -> trc::Result { + todo!() + } + + pub async fn update(&self, id: Id, object: &T) -> trc::Result { todo!() } diff --git a/crates/store/src/dispatch/search.rs b/crates/store/src/dispatch/search.rs index c718d7fc..013160aa 100644 --- a/crates/store/src/dispatch/search.rs +++ b/crates/store/src/dispatch/search.rs @@ -4,8 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use trc::AddContext; - use crate::{ SearchStore, Store, search::{ @@ -16,6 +14,7 @@ use crate::{ write::SearchIndex, }; use std::cmp::Ordering; +use trc::AddContext; impl SearchStore { pub async fn query_account(&self, query: SearchQuery) -> trc::Result> { @@ -343,6 +342,32 @@ impl SearchStore { pub fn is_meilisearch(&self) -> bool { matches!(self, SearchStore::MeiliSearch(_)) } + + pub async fn create_indexes(&self) -> trc::Result<()> { + match self { + SearchStore::Store(store) => match store { + #[cfg(feature = "postgres")] + Store::PostgreSQL(store) => store.create_search_tables().await, + #[cfg(feature = "mysql")] + Store::MySQL(store) => store.create_search_tables().await, + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] + Store::SQLReadReplica(store) => match store.primary_store() { + #[cfg(feature = "postgres")] + Store::PostgreSQL(primary) => primary.create_search_tables().await, + #[cfg(feature = "mysql")] + Store::MySQL(primary) => primary.create_search_tables().await, + _ => Ok(()), + }, + // SPDX-SnippetEnd + _ => Ok(()), + }, + SearchStore::ElasticSearch(store) => store.create_indexes().await, + SearchStore::MeiliSearch(store) => store.create_indexes().await, + } + } } impl SearchFilter { diff --git a/crates/store/src/dispatch/store.rs b/crates/store/src/dispatch/store.rs index c543b3f3..34aa8401 100644 --- a/crates/store/src/dispatch/store.rs +++ b/crates/store/src/dispatch/store.rs @@ -457,4 +457,22 @@ impl Store { } .caused_by(trc::location!()) } + + pub async fn create_tables(&self) -> trc::Result<()> { + match self { + #[cfg(feature = "sqlite")] + Self::SQLite(store) => store.create_tables(), + #[cfg(feature = "postgres")] + Self::PostgreSQL(store) => store.create_storage_tables().await, + #[cfg(feature = "mysql")] + Self::MySQL(store) => store.create_storage_tables().await, + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + Store::SQLReadReplica(store) => Box::pin(store.primary_store().create_tables()).await, + // SPDX-SnippetEnd + _ => Ok(()), + } + } } diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 13116f65..0d44eb9f 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -5,13 +5,14 @@ */ pub mod backend; -pub mod config; +pub mod bootstrap; pub mod dispatch; pub mod query; pub mod registry; pub mod search; pub mod write; +use ::registry::schema::enums::CompressionAlgo; pub use ahash; pub use blake3; pub use parking_lot; @@ -125,12 +126,8 @@ pub struct IterateParams { } #[derive(Clone, Default)] -pub struct Stores { - pub stores: AHashMap, - pub blob_stores: AHashMap, - pub search_stores: AHashMap, - pub in_memory_stores: AHashMap, - pub purge_schedules: Vec, +pub struct LookupStores { + pub stores: AHashMap, } #[derive(Clone, Default)] @@ -156,19 +153,7 @@ pub enum Store { } #[derive(Clone)] -pub struct BlobStore { - pub backend: BlobBackend, - pub compression: CompressionAlgo, -} - -#[derive(Clone, Copy, Debug)] -pub enum CompressionAlgo { - None, - Lz4, -} - -#[derive(Clone)] -pub enum BlobBackend { +pub enum BlobStore { Store(Store), Fs(Arc), #[cfg(feature = "s3")] @@ -246,35 +231,6 @@ impl From for Store { } } -impl From for BlobStore { - fn from(store: FsStore) -> Self { - BlobStore { - backend: BlobBackend::Fs(Arc::new(store)), - compression: CompressionAlgo::None, - } - } -} - -#[cfg(feature = "s3")] -impl From for BlobStore { - fn from(store: backend::s3::S3Store) -> Self { - BlobStore { - backend: BlobBackend::S3(Arc::new(store)), - compression: CompressionAlgo::None, - } - } -} - -#[cfg(feature = "azure")] -impl From for BlobStore { - fn from(store: backend::azure::AzureStore) -> Self { - BlobStore { - backend: BlobBackend::Azure(Arc::new(store)), - compression: CompressionAlgo::None, - } - } -} - impl From for SearchStore { fn from(store: ElasticSearchStore) -> Self { Self::ElasticSearch(Arc::new(store)) @@ -300,30 +256,12 @@ impl From for SearchStore { } } -impl From for BlobStore { - fn from(store: Store) -> Self { - BlobStore { - backend: BlobBackend::Store(store), - compression: CompressionAlgo::None, - } - } -} - impl From for InMemoryStore { fn from(store: Store) -> Self { Self::Store(store) } } -impl Default for BlobStore { - fn default() -> Self { - Self { - backend: BlobBackend::Store(Store::None), - compression: CompressionAlgo::None, - } - } -} - impl Default for InMemoryStore { fn default() -> Self { Self::Store(Store::None) @@ -784,20 +722,3 @@ impl From> for () { unreachable!() } } - -impl Stores { - pub fn disable_enterprise_only(&mut self) { - // SPDX-SnippetBegin - // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - // SPDX-License-Identifier: LicenseRef-SEL - #[cfg(feature = "enterprise")] - { - #[cfg(any(feature = "postgres", feature = "mysql"))] - self.stores - .retain(|_, store| !matches!(store, Store::SQLReadReplica(_))); - self.blob_stores - .retain(|_, store| !matches!(store.backend, BlobBackend::Sharded(_))); - } - // SPDX-SnippetEnd - } -} diff --git a/crates/common/src/manager/bootstrap.rs b/crates/store/src/registry/bootstrap.rs similarity index 68% rename from crates/common/src/manager/bootstrap.rs rename to crates/store/src/registry/bootstrap.rs index 257e2090..e0d421ee 100644 --- a/crates/common/src/manager/bootstrap.rs +++ b/crates/store/src/registry/bootstrap.rs @@ -4,15 +4,18 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::{RegistryStore, registry::RegistryObject}; use registry::{ - schema::{prelude::Property, structs::Node}, + schema::{ + prelude::Property, + structs::{LocalSettings, Node}, + }, types::{ ObjectType, error::{Error, ValidationError, Warning}, id::Id, }, }; -use store::{RegistryStore, registry::RegistryObject}; pub struct Bootstrap { pub registry: RegistryStore, @@ -20,6 +23,7 @@ pub struct Bootstrap { pub warnings: Vec, pub has_fatal_errors: bool, pub node: Node, + pub local: LocalSettings, } impl Bootstrap { @@ -30,6 +34,7 @@ impl Bootstrap { warnings: Vec::new(), has_fatal_errors: false, node: Node::default(), + local: LocalSettings::default(), } } @@ -63,6 +68,46 @@ impl Bootstrap { } } + pub async fn get_infallible(&mut self, id: Id) -> Option { + if id.object() != T::object() { + match self.registry.get::(id).await { + Ok(Some(setting)) => { + let mut errors = Vec::new(); + if setting.validate(&mut errors) { + Some(setting) + } else { + self.errors.push(Error::Validation { + object_id: id, + errors, + }); + None + } + } + Ok(None) => { + self.errors.push(Error::NotFound { object_id: id }); + None + } + Err(err) => { + if !self.has_fatal_errors { + self.errors.push(Error::Internal { + object_id: Some(id), + error: err, + }); + self.has_fatal_errors = true; + } + None + } + } + } else { + self.errors.push(Error::TypeMismatch { + object_id: id, + object_type: id.object(), + expected_type: T::object(), + }); + None + } + } + pub async fn list_infallible(&mut self) -> Vec> { match self.registry.list::().await { Ok(objects) => objects diff --git a/crates/store/src/registry/mod.rs b/crates/store/src/registry/mod.rs index e2da0869..99f089d0 100644 --- a/crates/store/src/registry/mod.rs +++ b/crates/store/src/registry/mod.rs @@ -4,6 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +pub mod bootstrap; + use registry::types::{ObjectType, id::Id}; pub struct RegistryObject { diff --git a/crates/trc/src/event/enums.rs b/crates/trc/src/event/enums.rs index 6a9a907b..eb84bc7c 100644 --- a/crates/trc/src/event/enums.rs +++ b/crates/trc/src/event/enums.rs @@ -6,7 +6,7 @@ // This file is auto-generated. Do not edit directly. -pub const TOTAL_EVENT_COUNT: usize = 593; +pub const TOTAL_EVENT_COUNT: usize = 595; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum EventType { @@ -628,6 +628,8 @@ pub enum SecurityEvent { ScanBan = 558, LoiterBan = 550, IpBlocked = 318, + IpBlockExpired = 593, + IpAllowExpired = 594, Unauthorized = 552, } diff --git a/crates/trc/src/event/enums_impl.rs b/crates/trc/src/event/enums_impl.rs index 63b94563..48796eb5 100644 --- a/crates/trc/src/event/enums_impl.rs +++ b/crates/trc/src/event/enums_impl.rs @@ -410,6 +410,8 @@ impl EventType { b"security.scan-ban" => EventType::Security(SecurityEvent::ScanBan), b"security.loiter-ban" => EventType::Security(SecurityEvent::LoiterBan), b"security.ip-blocked" => EventType::Security(SecurityEvent::IpBlocked), + b"security.ip-block-expired" => EventType::Security(SecurityEvent::IpBlockExpired), + b"security.ip-allow-expired" => EventType::Security(SecurityEvent::IpAllowExpired), b"security.unauthorized" => EventType::Security(SecurityEvent::Unauthorized), b"server.startup" => EventType::Server(ServerEvent::Startup), b"server.shutdown" => EventType::Server(ServerEvent::Shutdown), @@ -1111,6 +1113,8 @@ impl EventType { EventType::Security(SecurityEvent::ScanBan) => "security.scan-ban", EventType::Security(SecurityEvent::LoiterBan) => "security.loiter-ban", EventType::Security(SecurityEvent::IpBlocked) => "security.ip-blocked", + EventType::Security(SecurityEvent::IpBlockExpired) => "security.ip-block-expired", + EventType::Security(SecurityEvent::IpAllowExpired) => "security.ip-allow-expired", EventType::Security(SecurityEvent::Unauthorized) => "security.unauthorized", EventType::Server(ServerEvent::Startup) => "server.startup", EventType::Server(ServerEvent::Shutdown) => "server.shutdown", @@ -1723,6 +1727,8 @@ impl EventType { EventType::Security(SecurityEvent::ScanBan) => 558, EventType::Security(SecurityEvent::LoiterBan) => 550, EventType::Security(SecurityEvent::IpBlocked) => 318, + EventType::Security(SecurityEvent::IpBlockExpired) => 593, + EventType::Security(SecurityEvent::IpAllowExpired) => 594, EventType::Security(SecurityEvent::Unauthorized) => 552, EventType::Server(ServerEvent::Startup) => 393, EventType::Server(ServerEvent::Shutdown) => 392, @@ -2357,6 +2363,8 @@ impl EventType { 558 => Some(EventType::Security(SecurityEvent::ScanBan)), 550 => Some(EventType::Security(SecurityEvent::LoiterBan)), 318 => Some(EventType::Security(SecurityEvent::IpBlocked)), + 593 => Some(EventType::Security(SecurityEvent::IpBlockExpired)), + 594 => Some(EventType::Security(SecurityEvent::IpAllowExpired)), 552 => Some(EventType::Security(SecurityEvent::Unauthorized)), 393 => Some(EventType::Server(ServerEvent::Startup)), 392 => Some(EventType::Server(ServerEvent::Shutdown)), @@ -2746,6 +2754,8 @@ impl EventType { EventType::Security(SecurityEvent::ScanBan) => Level::Info, EventType::Security(SecurityEvent::LoiterBan) => Level::Info, EventType::Security(SecurityEvent::IpBlocked) => Level::Info, + EventType::Security(SecurityEvent::IpBlockExpired) => Level::Info, + EventType::Security(SecurityEvent::IpAllowExpired) => Level::Info, EventType::Security(SecurityEvent::Unauthorized) => Level::Info, EventType::Server(ServerEvent::Startup) => Level::Info, EventType::Server(ServerEvent::Shutdown) => Level::Info, @@ -3402,6 +3412,8 @@ impl EventType { EventType::Security(SecurityEvent::ScanBan) => "Banned due to scan", EventType::Security(SecurityEvent::LoiterBan) => "Banned due to loitering", EventType::Security(SecurityEvent::IpBlocked) => "Blocked IP address", + EventType::Security(SecurityEvent::IpBlockExpired) => "IP block expired", + EventType::Security(SecurityEvent::IpAllowExpired) => "IP allow expired", EventType::Security(SecurityEvent::Unauthorized) => "Unauthorized access", EventType::Server(ServerEvent::Startup) => "Starting Stalwart Server v0.15.4", EventType::Server(ServerEvent::Shutdown) => "Shutting down Stalwart Server v0.15.4", @@ -4348,6 +4360,12 @@ impl EventType { EventType::Security(SecurityEvent::IpBlocked) => { "Rejected connection from blocked IP address" } + EventType::Security(SecurityEvent::IpBlockExpired) => { + "A previously blocked IP address block has expired" + } + EventType::Security(SecurityEvent::IpAllowExpired) => { + "A previously allowed IP address allow has expired" + } EventType::Security(SecurityEvent::Unauthorized) => { "Account does not have permission to access resource" } @@ -4865,6 +4883,8 @@ impl EventType { EventType::Security(SecurityEvent::ScanBan) => "Insufficient permissions", EventType::Security(SecurityEvent::LoiterBan) => "Insufficient permissions", EventType::Security(SecurityEvent::IpBlocked) => "Insufficient permissions", + EventType::Security(SecurityEvent::IpBlockExpired) => "Insufficient permissions", + EventType::Security(SecurityEvent::IpAllowExpired) => "Insufficient permissions", EventType::Security(SecurityEvent::Unauthorized) => "Insufficient permissions", EventType::Smtp(SmtpEvent::ConnectionStart) => "SMTP error", EventType::Smtp(SmtpEvent::ConnectionEnd) => "SMTP error", @@ -5388,6 +5408,8 @@ impl EventType { EventType::Security(SecurityEvent::ScanBan), EventType::Security(SecurityEvent::LoiterBan), EventType::Security(SecurityEvent::IpBlocked), + EventType::Security(SecurityEvent::IpBlockExpired), + EventType::Security(SecurityEvent::IpAllowExpired), EventType::Security(SecurityEvent::Unauthorized), EventType::Server(ServerEvent::Startup), EventType::Server(ServerEvent::Shutdown), diff --git a/crates/utils/src/glob.rs b/crates/utils/src/glob.rs index 8fb8f989..45b83e53 100644 --- a/crates/utils/src/glob.rs +++ b/crates/utils/src/glob.rs @@ -4,10 +4,18 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::borrow::Cow; - use ahash::{AHashMap, AHashSet}; use serde::Deserialize; +use std::borrow::Cow; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MatchType { + Equal(String), + StartsWith(String), + EndsWith(String), + Matches(GlobPattern), + All, +} #[derive(Debug, Clone, PartialEq, Eq)] pub struct GlobPattern { @@ -161,7 +169,7 @@ impl GlobSet { GlobSet::default() } - pub fn insert(&mut self, pattern: &str) { + pub fn insert_pattern(&mut self, pattern: &str) { match GlobPattern::try_compile(pattern, false) { Ok(glob) => { self.patterns.push(glob); @@ -172,6 +180,10 @@ impl GlobSet { } } + pub fn insert_entry(&mut self, entry: String) { + self.entries.insert(entry); + } + pub fn contains(&self, key: &str) -> bool { self.entries.contains(key) || self.patterns.iter().any(|pattern| pattern.matches(key)) } @@ -185,7 +197,7 @@ impl GlobMap { } } - pub fn insert(&mut self, pattern: &str, value: V) { + pub fn insert_pattern(&mut self, pattern: &str, value: V) { match GlobPattern::try_compile(pattern, false) { Ok(glob) => { self.patterns.push((glob, value)); @@ -196,6 +208,10 @@ impl GlobMap { } } + pub fn insert_entry(&mut self, entry: String, value: V) { + self.entries.insert(entry, value); + } + pub fn get(&self, key: &str) -> Option<&V> { self.entries.get(key).or_else(|| { self.patterns @@ -211,6 +227,32 @@ impl Default for GlobMap { } } +impl MatchType { + pub fn parse(value: &str) -> Self { + if value == "*" { + MatchType::All + } else if let Some(value) = value.strip_suffix('*') { + MatchType::StartsWith(value.to_string()) + } else if let Some(value) = value.strip_prefix('*') { + MatchType::EndsWith(value.to_string()) + } else if value.contains('*') { + MatchType::Matches(GlobPattern::compile(value, false)) + } else { + MatchType::Equal(value.to_string()) + } + } + + pub fn matches(&self, value: &str) -> bool { + match self { + MatchType::Equal(pattern) => value == pattern, + MatchType::StartsWith(pattern) => value.starts_with(pattern), + MatchType::EndsWith(pattern) => value.ends_with(pattern), + MatchType::Matches(pattern) => pattern.matches(value), + MatchType::All => true, + } + } +} + impl<'de> Deserialize<'de> for GlobPattern { fn deserialize(deserializer: D) -> Result where diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 141903f4..c0ef26d5 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -27,6 +27,7 @@ use rustls::{ use rustls_pki_types::TrustAnchor; use std::sync::Arc; +pub use reqwest::Client; pub use reqwest::header::HeaderMap; pub trait HttpLimitResponse: Sync + Send { diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 15e8eaf3..0bca1d0d 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -4,9 +4,9 @@ version = "0.15.5" edition = "2024" [features] -#default = ["sqlite", "postgres", "mysql", "rocks", "s3", "redis", "nats", "azure", "foundationdb"] +default = ["sqlite", "postgres", "mysql", "rocks", "s3", "redis", "nats", "azure", "foundationdb"] #default = ["sqlite", "postgres", "mysql", "rocks", "s3", "redis", "foundationdb"] -default = ["rocks"] +#default = ["rocks"] sqlite = ["store/sqlite"] foundationdb = ["store/foundation", "common/foundation"] postgres = ["store/postgres"]