From 3df5327ac86afdba8478c3db0b3d298ddb2f3adb Mon Sep 17 00:00:00 2001 From: mdecimus <11444311+mdecimus@users.noreply.github.com> Date: Thu, 11 Dec 2025 18:43:53 +0100 Subject: [PATCH] Migration implementation (part 2) --- Cargo.lock | 223 +++++++------- crates/common/src/manager/backup.rs | 3 +- crates/common/src/manager/restore.rs | 37 ++- crates/imap/src/op/search.rs | 3 +- crates/jmap/src/email/query.rs | 2 +- crates/migration/Cargo.toml | 1 + crates/migration/src/blob.rs | 2 +- crates/migration/src/email_v1.rs | 214 +++++++------ crates/migration/src/email_v2.rs | 64 ++-- crates/migration/src/encryption_v2.rs | 2 +- crates/migration/src/identity_v2.rs | 39 --- crates/migration/src/lib.rs | 103 ++++--- crates/migration/src/queue_v2.rs | 6 +- crates/migration/src/tasks_v2.rs | 2 + crates/migration/src/v014.rs | 282 +++++++++++++++++- crates/nlp/src/classifier/reservoir.rs | 10 + crates/spam-filter/src/modules/classifier.rs | 26 +- crates/store/src/backend/mysql/search.rs | 11 +- crates/store/src/backend/postgres/write.rs | 1 + crates/store/src/search/document.rs | 12 +- tests/Cargo.toml | 3 +- tests/resources/smtp/antispam/classifier.test | 6 +- .../smtp/antispam/classifier_features.test | 44 --- tests/src/directory/oidc.rs | 12 +- tests/src/imap/basic.rs | 40 ++- tests/src/imap/body_structure.rs | 9 +- tests/src/imap/mailbox.rs | 4 +- tests/src/imap/mod.rs | 3 + tests/src/jmap/contacts/contact.rs | 7 +- tests/src/jmap/core/push_subscription.rs | 4 +- tests/src/jmap/mail/crypto.rs | 6 +- tests/src/jmap/mail/delivery.rs | 2 + tests/src/jmap/mail/query.rs | 12 +- tests/src/jmap/mod.rs | 4 +- tests/src/store/cleanup.rs | 4 +- tests/src/store/import_export.rs | 1 + tests/src/store/mod.rs | 16 +- tests/src/store/ops.rs | 56 ++-- tests/src/store/query.rs | 4 +- 39 files changed, 791 insertions(+), 489 deletions(-) delete mode 100644 crates/migration/src/identity_v2.rs diff --git a/Cargo.lock b/Cargo.lock index 5c297f56..90028b35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -556,9 +556,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "base64ct" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" +checksum = "0e050f626429857a27ddccb31e0aca21356bfa709c04041aefddac081a8f068a" [[package]] name = "bincode" @@ -943,9 +943,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.48" +version = "1.2.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c481bdbf0ed3b892f6f806287d72acd515b352a4ec27a208489b8c1bc839633a" +checksum = "90583009037521a116abf44494efecd645ba48b6622457080f080b85544e2215" dependencies = [ "find-msvc-tools", "jobserver", @@ -3231,9 +3231,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52e9a2a24dc5c6821e71a7030e1e14b7b632acac55c40e9d2e082c621261bb56" +checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" dependencies = [ "base64 0.22.1", "bytes", @@ -3325,9 +3325,9 @@ checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" [[package]] name = "icu_properties" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e93fcd3157766c0c8da2f8cff6ce651a31f0810eaa1c51ec363ef790bbb5fb99" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" dependencies = [ "icu_collections", "icu_locale_core", @@ -3339,9 +3339,9 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02845b3647bb045f1100ecd6480ff52f34c35f82d9880e029d329c21d1054899" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" [[package]] name = "icu_provider" @@ -4082,9 +4082,9 @@ dependencies = [ [[package]] name = "libz-rs-sys" -version = "0.5.2" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "840db8cf39d9ec4dd794376f38acc40d0fc65eec2a8f484f7fd375b84602becd" +checksum = "15413ef615ad868d4d65dce091cb233b229419c7c0c4bcaa746c0901c49ff39c" dependencies = [ "zlib-rs", ] @@ -4130,9 +4130,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.28" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "lru" @@ -4380,6 +4380,7 @@ dependencies = [ "dav-proto", "directory", "email", + "futures", "groupware", "lz4_flex 0.12.0", "mail-auth", @@ -4433,9 +4434,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" dependencies = [ "libc", "wasi 0.11.1+wasi-snapshot-preview1", @@ -5608,9 +5609,9 @@ dependencies = [ [[package]] name = "psl" -version = "2.1.166" +version = "2.1.168" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2085080c7de45d70a59d96aa7d5b0870fc1ccbd27adb780ee1e1ec905da42035" +checksum = "6faf5b87f44516ca3989e1e315a0413f7a4d4886e63664cbd4a5f492c73cad93" dependencies = [ "psl-types", ] @@ -6164,9 +6165,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.24" +version = "0.12.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f" +checksum = "b6eff9328d40131d43bd911d42d79eb6a47312002a4daefc9e37f17e74a7701a" dependencies = [ "base64 0.22.1", "bytes", @@ -6322,14 +6323,15 @@ dependencies = [ [[package]] name = "ron" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db09040cc89e461f1a265139777a2bde7f8d8c67c4936f700c63ce3e2904d468" +checksum = "fd490c5b18261893f14449cbd28cb9c0b637aebf161cd77900bfdedaff21ec32" dependencies = [ - "base64 0.22.1", "bitflags", + "once_cell", "serde", "serde_derive", + "typeid", "unicode-ident", ] @@ -6635,7 +6637,6 @@ version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ - "ring", "rustls-pki-types", "untrusted", ] @@ -6714,19 +6715,6 @@ dependencies = [ "parking_lot", ] -[[package]] -name = "schemars" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" -dependencies = [ - "dyn-clone", - "either", - "schemars_derive", - "serde", - "serde_json", -] - [[package]] name = "schemars" version = "0.9.0" @@ -6746,16 +6734,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289" dependencies = [ "dyn-clone", + "either", "ref-cast", + "schemars_derive", "serde", "serde_json", ] [[package]] name = "schemars_derive" -version = "0.8.22" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d" +checksum = "301858a4023d78debd2353c7426dc486001bddc91ae31a76fb1f55132f7e2633" dependencies = [ "proc-macro2", "quote", @@ -7288,9 +7278,9 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" [[package]] name = "simdutf8" @@ -7477,9 +7467,9 @@ dependencies = [ [[package]] name = "stabby" -version = "36.2.2" +version = "72.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b7e94eaf470c2e76b5f15fb2fb49714471a36cc512df5ee231e62e82ec79f8" +checksum = "976399a0c48ea769ef7f5dc303bb88240ab8d84008647a6b2303eced3dab3945" dependencies = [ "rustversion", "stabby-abi", @@ -7487,9 +7477,9 @@ dependencies = [ [[package]] name = "stabby-abi" -version = "36.2.2" +version = "72.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc7a63b8276b54e51bfffe3d85da56e7906b2dcfcb29018a8ab666c06734c1a" +checksum = "f7b54832a9a1f92a0e55e74a5c0332744426edc515bb3fbad82f10b874a87f0d" dependencies = [ "rustc_version", "rustversion", @@ -7499,9 +7489,9 @@ dependencies = [ [[package]] name = "stabby-macros" -version = "36.2.2" +version = "72.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eecb7ec5611ec93ec79d120fbe55f31bea234dc1bed1001d4a071bb688651615" +checksum = "a768b1e51e4dbfa4fa52ae5c01241c0a41e2938fdffbb84add0c8238092f9091" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -8090,9 +8080,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.23.7" +version = "0.23.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d" +checksum = "5d7cbc3b4b49633d57a0509303158ca50de80ae32c265093b24c414705807832" dependencies = [ "indexmap 2.12.1", "toml_datetime", @@ -8190,9 +8180,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf146f99d442e8e68e585f5d798ccd3cad9a7835b917e09728880a862706456" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ "bitflags", "bytes", @@ -8368,6 +8358,12 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + [[package]] name = "typenum" version = "1.19.0" @@ -8464,9 +8460,9 @@ checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" [[package]] name = "unicode-script" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb421b350c9aff471779e262955939f565ec18b86c15364e6bdf0d662ca7c1f" +checksum = "383ad40bb927465ec0ce7720e033cb4ca06912855fc35db31b5755d0de75b1ee" [[package]] name = "unicode-security" @@ -8526,13 +8522,13 @@ checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" [[package]] name = "unzip-n" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e7e85a0596447f0f2ac090e16bc4c516c6fe91771fb0c0ccf7fa3dae896b9c" +checksum = "3b5bb2756c16fb66f80cfbf5fb0e0c09a7001e739f453c9ec241b9c8b1556fda" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.111", ] [[package]] @@ -9377,15 +9373,16 @@ dependencies = [ [[package]] name = "zenoh" -version = "1.6.2" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5df040795169d4b6e4fc8a5f9dce81b5210d1d606399634ec380deda30c347f" +checksum = "4e2df46d36ba8d173f18e9cce5011117fed81a90a22bc3cfe8bbcfaf66b54114" dependencies = [ "ahash", "arc-swap", "async-trait", "bytes", "const_format", + "flate2", "flume", "futures", "git-version", @@ -9394,7 +9391,6 @@ dependencies = [ "lazy_static", "nonempty-collections", "once_cell", - "paste", "petgraph 0.8.3", "phf 0.13.1", "rand 0.8.5", @@ -9429,18 +9425,18 @@ dependencies = [ [[package]] name = "zenoh-buffers" -version = "1.6.2" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "510921897b03793f399b7eaec176355a12293afa1e791c3f844ef12fa08aecc4" +checksum = "82d610eaa2099c16844b092c2b16605d115f354e332eb1d37e9af46e5430ecac" dependencies = [ "zenoh-collections", ] [[package]] name = "zenoh-codec" -version = "1.6.2" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585a77eff7a781726aeeafab52ad175be0dcfc92376a756c23a51f6701a73ba0" +checksum = "7f1cff28c30bca3d07e7b84f71f1c9b81d505449f6fec36a023cc8b601c01150" dependencies = [ "tracing", "uhlc", @@ -9450,18 +9446,18 @@ dependencies = [ [[package]] name = "zenoh-collections" -version = "1.6.2" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa17a7ecf321aba18eabf99a37390b7a6524e826b6f8cb38f9361479ece28887" +checksum = "a72cac7d4bbe99e41ef46c175e0dd2933e883b17cef3e60151e4461b5a088ad3" dependencies = [ "ahash", ] [[package]] name = "zenoh-config" -version = "1.6.2" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02801ef8fd8d0b10437055dac10fd961fdc3a6356439dd52ee627e7f31b9295" +checksum = "df65e4ab337079d897aaaa6d01474b4f8683daaa05a8671b18db34e71a872af7" dependencies = [ "json5", "nonempty-collections", @@ -9484,9 +9480,9 @@ dependencies = [ [[package]] name = "zenoh-core" -version = "1.6.2" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "325645d09d0f74f48051b51c2ab73e270c16d9d7d408c41eb47ed8f067451cd9" +checksum = "869deb22305e8f7a6aa324142ab6717f0cc979be8e4a2de646e8f31313224a1c" dependencies = [ "lazy_static", "tokio", @@ -9496,9 +9492,9 @@ dependencies = [ [[package]] name = "zenoh-crypto" -version = "1.6.2" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5244d615594f1c2149cba6177f15e76cc825e2839230e26693f2b723367b6c2f" +checksum = "7f5b09559108c9e11128730d5a6f551dbcc226adfd82423439ad7eeed024c736" dependencies = [ "aes", "hmac 0.12.1", @@ -9510,15 +9506,15 @@ dependencies = [ [[package]] name = "zenoh-keyexpr" -version = "1.6.2" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9495c0ca85331f87442b465760a7fb7334370354742a4201b719cc2533afee7" +checksum = "b8b754ea6460c4c3c5dd67e6a8c62bfc076c16f8b794898fb3ea6f0039a5939e" dependencies = [ "getrandom 0.2.16", "hashbrown 0.16.1", "keyed-set", "rand 0.8.5", - "schemars 0.8.22", + "schemars 1.1.0", "serde", "token-cell", "zenoh-result", @@ -9526,9 +9522,9 @@ dependencies = [ [[package]] name = "zenoh-link" -version = "1.6.2" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "001a7f07522b06ae6364ae787ab83eddf4e43f56f7f138fe3e54ea123ee3523b" +checksum = "0d0c04d554b67600967cb520061ef9f635caf7704978357334e66cd7179a204a" dependencies = [ "zenoh-config", "zenoh-link-commons", @@ -9542,9 +9538,9 @@ dependencies = [ [[package]] name = "zenoh-link-commons" -version = "1.6.2" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5be05b499b789afb2024de4253e12ccaa3e296f66a0db6147deb9acec491f622" +checksum = "52d1c652a706da1da46eae54a65c4b658153cdebfa213f061beea3bef71088f2" dependencies = [ "async-trait", "base64 0.22.1", @@ -9554,7 +9550,7 @@ dependencies = [ "rustls 0.23.35", "rustls-pemfile 2.2.0", "rustls-pki-types", - "rustls-webpki 0.102.8", + "rustls-webpki 0.103.8", "secrecy", "serde", "socket2 0.5.10", @@ -9562,7 +9558,7 @@ dependencies = [ "tokio", "tokio-util", "tracing", - "webpki-roots 0.26.11", + "webpki-roots 1.0.4", "x509-parser", "zenoh-buffers", "zenoh-codec", @@ -9576,22 +9572,22 @@ dependencies = [ [[package]] name = "zenoh-link-quic" -version = "1.6.2" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4168034bb7005d313ba834d105ede7c7f9fac48684f80a140fccb8302e9ca4aa" +checksum = "0069d889388d06e56eb86139021a1af12b9846222847aeabaa05eb92fc2ee108" dependencies = [ "async-trait", "base64 0.22.1", "quinn", "rustls 0.23.35", "rustls-pemfile 2.2.0", - "rustls-webpki 0.102.8", + "rustls-webpki 0.103.8", "secrecy", "time", "tokio", "tokio-util", "tracing", - "webpki-roots 0.26.11", + "webpki-roots 1.0.4", "zenoh-config", "zenoh-core", "zenoh-link-commons", @@ -9602,9 +9598,9 @@ dependencies = [ [[package]] name = "zenoh-link-tcp" -version = "1.6.2" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b968d3506682350760fb69c07b3c9aea07f351ac39993c7d948bdc5a19d5645" +checksum = "279ff42ab4a65508069596045ab8a3c6fb03f94343c290a9b75f9568c49094e4" dependencies = [ "async-trait", "socket2 0.5.10", @@ -9620,16 +9616,16 @@ dependencies = [ [[package]] name = "zenoh-link-tls" -version = "1.6.2" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a810a4ba44011cc4c75713eadcf1a03fb2e22f0907d62f9b12c04d42ba1d596f" +checksum = "d3172615126b64456beb0c537351eb9c715f20f641e37ae8f84cb209816d205a" dependencies = [ "async-trait", "base64 0.22.1", "rustls 0.23.35", "rustls-pemfile 2.2.0", "rustls-pki-types", - "rustls-webpki 0.102.8", + "rustls-webpki 0.103.8", "secrecy", "socket2 0.5.10", "time", @@ -9638,7 +9634,7 @@ dependencies = [ "tokio-rustls 0.26.4", "tokio-util", "tracing", - "webpki-roots 0.26.11", + "webpki-roots 1.0.4", "x509-parser", "zenoh-config", "zenoh-core", @@ -9650,9 +9646,9 @@ dependencies = [ [[package]] name = "zenoh-link-udp" -version = "1.6.2" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4178b940cc613b4d0db3f06ceb71377199cacb22087f589bc60d1b4df6a2365" +checksum = "11006545a887167d431e8c3f2867ad386e7d678ee8babd3df4a7537b168e47de" dependencies = [ "async-trait", "libc", @@ -9672,9 +9668,9 @@ dependencies = [ [[package]] name = "zenoh-macros" -version = "1.6.2" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bbff17ef687e10ed4f6a956fb78fc97205a4a6242c657638e8e5da726840aca" +checksum = "21df1681ca014a4bd69db91d356bcf699d346f854ff8a6b73123cee75f5073cd" dependencies = [ "proc-macro2", "quote", @@ -9684,9 +9680,9 @@ dependencies = [ [[package]] name = "zenoh-plugin-trait" -version = "1.6.2" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "968571b0b585c5be1f19c553b368994424a869af46a4785bf5a3248194a97172" +checksum = "4651bc51951eb004e71bba71583f9ea60240bdd11351b61dbd6956e906088567" dependencies = [ "git-version", "libloading", @@ -9702,9 +9698,9 @@ dependencies = [ [[package]] name = "zenoh-protocol" -version = "1.6.2" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879a66ef51831f54f5f824bf9005bf73129bc2485053311ffb7a245f907f721c" +checksum = "ab17d39b24f6fbd0a24e03a5ab96c236d3d5ebb5071978338c9bc3fe1ccff626" dependencies = [ "const_format", "rand 0.8.5", @@ -9717,18 +9713,18 @@ dependencies = [ [[package]] name = "zenoh-result" -version = "1.6.2" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0f564d3324e443d0d48a4656d89bab1d0b263660fb7e17ba149c464e62f94bd" +checksum = "1b7d23a2f69c4b96ae95ac2f80808cce5f2ed9697ba37e769a75428fb6806a57" dependencies = [ "anyhow", ] [[package]] name = "zenoh-runtime" -version = "1.6.2" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3a5cef13fd469aeb146518532751dbecd2b0f99ac56b6df667f954f11348dcf" +checksum = "a71f356bb63c1c9d37cb4bc89112e83a9fd6079458d1440232d67ca41a52c687" dependencies = [ "lazy_static", "ron", @@ -9741,9 +9737,9 @@ dependencies = [ [[package]] name = "zenoh-sync" -version = "1.6.2" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04a23123511fb7ce5990a778dcdcf41b5129ad8c5ef8d1e7ff719a538c3270e3" +checksum = "e3b1113111e247d6b1ebd5d406b43e23e78faca2c7525dca22a2a075c64787bd" dependencies = [ "arc-swap", "event-listener 5.4.1", @@ -9756,9 +9752,9 @@ dependencies = [ [[package]] name = "zenoh-task" -version = "1.6.2" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ad997618409140b23a2ef0957f7995afec2667ff3951f580de8a506c7814e41" +checksum = "7a1f690f81704b1e934b9fb7a63ce544af73521ca9d11b5a439cdfc74108dd28" dependencies = [ "futures", "tokio", @@ -9770,16 +9766,15 @@ dependencies = [ [[package]] name = "zenoh-transport" -version = "1.6.2" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2abd4521cd556093161156fc0799a506743304e1510fc7ed83adff99ca601f34" +checksum = "41cdc4dcb4a68161afe25e44871a66a5c3c347d86a77628ab101d859a9518db9" dependencies = [ "async-trait", "crossbeam-utils", "flume", "lazy_static", "lz4_flex 0.11.5", - "paste", "rand 0.8.5", "ringbuffer-spsc", "rsa", @@ -9805,9 +9800,9 @@ dependencies = [ [[package]] name = "zenoh-util" -version = "1.6.2" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42a736159ed0935c9ef258c151067ccce40f95f08d7f9360993e858e268b0a9d" +checksum = "ea9b52e8beceffc0deca01e10ca726c604286a1ac475f95acdd87ee55f18b120" dependencies = [ "async-trait", "const_format", @@ -9818,7 +9813,7 @@ dependencies = [ "libc", "libloading", "pnet_datalink", - "schemars 0.8.22", + "schemars 1.1.0", "serde", "serde_json", "shellexpand", @@ -9953,9 +9948,9 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.5.2" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f06ae92f42f5e5c42443fd094f245eb656abf56dd7cce9b8b263236565e00f2" +checksum = "51f936044d677be1a1168fae1d03b583a285a5dd9d8cbf7b24c23aa1fc775235" [[package]] name = "zopfli" diff --git a/crates/common/src/manager/backup.rs b/crates/common/src/manager/backup.rs index f9b4e602..5c4062d6 100644 --- a/crates/common/src/manager/backup.rs +++ b/crates/common/src/manager/backup.rs @@ -180,7 +180,8 @@ impl Core { subspace, key: vec![u8::MAX; 32], }, - ), + ) + .set_values(subspace != SUBSPACE_INDEXES), |key, value| { writer .send((key.to_vec(), value.to_vec())) diff --git a/crates/common/src/manager/restore.rs b/crates/common/src/manager/restore.rs index babd317d..efa2acb8 100644 --- a/crates/common/src/manager/restore.rs +++ b/crates/common/src/manager/restore.rs @@ -13,9 +13,10 @@ use std::{ path::{Path, PathBuf}, }; use store::{ - BlobStore, SUBSPACE_BLOBS, SUBSPACE_COUNTER, SUBSPACE_QUOTA, Store, - write::{AnyClass, BatchBuilder, ValueClass}, + BlobStore, SUBSPACE_BLOBS, SUBSPACE_COUNTER, SUBSPACE_INDEXES, SUBSPACE_QUOTA, Store, U32_LEN, + write::{AnyClass, BatchBuilder, ValueClass, key::DeserializeBigEndian}, }; +use types::{collection::Collection, field::Field}; use utils::{UnwrapFailure, failed}; impl Core { @@ -82,6 +83,38 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { } } } + SUBSPACE_INDEXES => { + while let Some((key, _)) = reader.next() { + let account_id = key + .as_slice() + .deserialize_be_u32(0) + .failed("Failed to deserialize account ID"); + let collection = *key.get(U32_LEN).failed("Missing collection byte"); + let field = *key.get(U32_LEN + 1).failed("Missing field byte"); + let value = key + .get(U32_LEN + 2..key.len() - U32_LEN) + .failed("Missing index key") + .to_vec(); + let document_id = key + .as_slice() + .deserialize_be_u32(key.len() - U32_LEN) + .failed("Failed to deserialize document ID"); + + batch + .with_account_id(account_id) + .with_collection(Collection::from(collection)) + .with_document(document_id) + .index(Field::new(field), value); + + if batch.is_large_batch() { + store + .write(batch.build_all()) + .await + .failed("Failed to write batch"); + batch = BatchBuilder::new(); + } + } + } _ => { while let Some((key, value)) = reader.next() { batch.set( diff --git a/crates/imap/src/op/search.rs b/crates/imap/src/op/search.rs index a480c128..81ab35ba 100644 --- a/crates/imap/src/op/search.rs +++ b/crates/imap/src/op/search.rs @@ -485,7 +485,8 @@ impl SessionData { EmailSearchField::Headers, op, SearchValue::KeyValues( - VecMap::with_capacity(1).with_append(header.into_string(), value), + VecMap::with_capacity(1) + .with_append(header.as_str().to_lowercase(), value), ), )); } diff --git a/crates/jmap/src/email/query.rs b/crates/jmap/src/email/query.rs index b1d6e208..03b19aeb 100644 --- a/crates/jmap/src/email/query.rs +++ b/crates/jmap/src/email/query.rs @@ -149,7 +149,7 @@ impl EmailQuery for Server { EmailSearchField::Headers, op, SearchValue::KeyValues(VecMap::with_capacity(1).with_append( - header_name.into_string(), + header_name.as_str().to_lowercase(), value.unwrap_or_default(), )), )); diff --git a/crates/migration/Cargo.toml b/crates/migration/Cargo.toml index 7958230c..76b11d3a 100644 --- a/crates/migration/Cargo.toml +++ b/crates/migration/Cargo.toml @@ -30,6 +30,7 @@ compact_str = "0.9.0" bincode = "1.3.3" lz4_flex = { version = "0.12", default-features = false } base64 = "0.22" +futures = "0.3" [features] test_mode = [] diff --git a/crates/migration/src/blob.rs b/crates/migration/src/blob.rs index 28dc3ca6..3ec8df4e 100644 --- a/crates/migration/src/blob.rs +++ b/crates/migration/src/blob.rs @@ -154,7 +154,7 @@ pub(crate) async fn migrate_blobs_v014(server: &Server) -> trc::Result<()> { } else { OldType::Temp } - } else if value.len() == U64_LEN + U32_LEN { + } else if value.len() == U64_LEN + U32_LEN + 1 { let size = value.deserialize_be_u32(0)?; let deleted_at = value.deserialize_be_u64(U32_LEN)?; OldType::Undelete { deleted_at, size } diff --git a/crates/migration/src/email_v1.rs b/crates/migration/src/email_v1.rs index 536adb39..0f4b8ccd 100644 --- a/crates/migration/src/email_v1.rs +++ b/crates/migration/src/email_v1.rs @@ -8,22 +8,30 @@ use super::{LegacyBincode, get_properties}; use crate::{email_v2::LegacyKeyword, get_bitmap, get_document_ids, v014::SUBSPACE_BITMAP_TAG}; use common::Server; use email::{ - mailbox::UidMailbox, - message::metadata::{ - MESSAGE_HAS_ATTACHMENT, MESSAGE_RECEIVED_MASK, MessageDataBuilder, MessageMetadata, - MessageMetadataContents, MessageMetadataPart, MetadataHeader, MetadataPartType, - PART_ENCODING_BASE64, PART_ENCODING_PROBLEM, PART_ENCODING_QP, PART_SIZE_MASK, + mailbox::*, + message::{ + index::extractors::VisitTextArchived, + ingest::ThreadInfo, + metadata::{ + MESSAGE_HAS_ATTACHMENT, MESSAGE_RECEIVED_MASK, MessageDataBuilder, MessageMetadata, + MessageMetadataContents, MessageMetadataPart, MetadataHeader, MetadataHeaderName, + MetadataHeaderValue, MetadataPartType, PART_ENCODING_BASE64, PART_ENCODING_PROBLEM, + PART_ENCODING_QP, PART_SIZE_MASK, + }, }, }; use mail_parser::{ Address, Attribute, ContentType, DateTime, Encoding, HeaderName, HeaderValue, Received, + parsers::fields::thread::thread_name, }; use std::{borrow::Cow, collections::VecDeque}; use store::{ - Deserialize, SUBSPACE_INDEXES, SUBSPACE_PROPERTY, Serialize, U32_LEN, U64_LEN, ValueKey, + Deserialize, SUBSPACE_INDEXES, SUBSPACE_PROPERTY, Serialize, SerializeInfallible, U32_LEN, + U64_LEN, ValueKey, ahash::AHashMap, write::{ - AlignedBytes, AnyKey, Archive, Archiver, BatchBuilder, ValueClass, key::KeySerializer, + AlignedBytes, AnyKey, Archive, Archiver, BatchBuilder, IndexPropertyClass, ValueClass, + key::KeySerializer, }, }; use trc::AddContext; @@ -33,7 +41,7 @@ use types::{ field::{EmailField, Field}, keyword::*, }; -use utils::codec::leb128::Leb128Iterator; +use utils::{cheeky_hash::CheekyHash, codec::leb128::Leb128Iterator}; const FIELD_KEYWORDS: u8 = 4; const FIELD_THREAD_ID: u8 = 33; @@ -44,11 +52,11 @@ const BM_MARKER: u8 = 1 << 7; pub(crate) async fn migrate_emails_v011(server: &Server, account_id: u32) -> trc::Result { // Obtain email ids - let mut message_ids = get_document_ids(server, account_id, Collection::Email) + let mut document_ids = get_document_ids(server, account_id, Collection::Email) .await .caused_by(trc::location!())? .unwrap_or_default(); - let num_emails = message_ids.len(); + let num_emails = document_ids.len(); if num_emails == 0 { return Ok(0); } @@ -114,100 +122,126 @@ pub(crate) async fn migrate_emails_v011(server: &Server, account_id: u32) -> trc message_data.entry(message_id).or_default().thread_id = thread_id; } - // Migrate message metadata - for message_id in &message_ids { - match server - .store() - .get_value::>(ValueKey { - account_id, - collection: Collection::Email.into(), - document_id: message_id, - class: ValueClass::Property(EmailField::Metadata.into()), - }) - .await - { - Ok(Some(legacy_metadata)) => { - message_data.entry(message_id).or_default().size = - legacy_metadata.inner.size as u32; + // Write message data + for (message_id, mut data) in message_data { + if !tombstoned_ids.contains(message_id) { + let (size, metadata) = match server + .store() + .get_value::>(ValueKey { + account_id, + collection: Collection::Email.into(), + document_id: message_id, + class: ValueClass::Property(EmailField::Metadata.into()), + }) + .await + { + Ok(Some(legacy_metadata)) => ( + legacy_metadata.inner.size as u32, + MessageMetadata::from_legacy(legacy_metadata.inner), + ), + Ok(None) => { + continue; + } + Err(err) => { + match server + .store() + .get_value::>(ValueKey { + account_id, + collection: Collection::Email.into(), + document_id: message_id, + class: ValueClass::Property(EmailField::Metadata.into()), + }) + .await + { + Ok(Some(archive)) => { + let metadata: MessageMetadata = archive + .deserialize_untrusted() + .caused_by(trc::location!())?; + (metadata.root_part().offset_end, metadata) + } + _ => { + return Err(err + .account_id(account_id) + .document_id(message_id) + .caused_by(trc::location!())); + } + } + } + }; - let metadata = MessageMetadata::from_legacy(legacy_metadata.inner); + did_migrate = true; + document_ids.insert(message_id); - let mut batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_collection(Collection::Email) - .with_document(message_id); - - let todo = "reindex"; - /*for header in metadata.root_part().headers.iter().rev() { - if matches!(header.name, HeaderName::MessageId) { + let mut message_ids = Vec::new(); + let mut subject = ""; + for header in &metadata.contents[0].parts[0].headers { + match &header.name { + MetadataHeaderName::MessageId => { header.value.visit_text(|id| { - if id.len() < MAX_ID_LENGTH { - batch.index(EmailField::References, encode_message_id(id)); + if !id.is_empty() { + message_ids.push(CheekyHash::new(id.as_bytes())); } }); } - }*/ - - batch.set( - EmailField::Metadata, - Archiver::new(metadata) - .serialize() - .caused_by(trc::location!())?, - ); - - did_migrate = true; - - server - .store() - .write(batch.build_all()) - .await - .caused_by(trc::location!())?; - } - Ok(None) => (), - Err(err) => { - if server - .store() - .get_value::>(ValueKey { - account_id, - collection: Collection::Email.into(), - document_id: message_id, - class: ValueClass::Property(EmailField::Metadata.into()), - }) - .await - .is_err() - { - return Err(err - .account_id(account_id) - .document_id(message_id) - .caused_by(trc::location!())); + MetadataHeaderName::InReplyTo + | MetadataHeaderName::References + | MetadataHeaderName::ResentMessageId => { + header.value.visit_text(|id| { + if !id.is_empty() { + message_ids.push(CheekyHash::new(id.as_bytes())); + } + }); + } + MetadataHeaderName::Subject if subject.is_empty() => { + subject = thread_name(match &header.value { + MetadataHeaderValue::Text(text) => text.as_ref(), + MetadataHeaderValue::TextList(list) if !list.is_empty() => { + list.first().unwrap().as_ref() + } + _ => "", + }); + } + _ => (), } } - } - } - // Write message data - for (message_id, data) in message_data { - if !tombstoned_ids.contains(message_id) { - message_ids.insert(message_id); let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Email) .with_document(message_id); - /*for mailbox in &data.mailboxes { - batch.untag(EmailField::MailboxIds, TagValue::Id(mailbox.mailbox_id)); - }*/ - - did_migrate = true; - - batch.set( - Field::ARCHIVE, - Archiver::new(data.seal()) - .serialize() - .caused_by(trc::location!())?, - ); + if data + .mailboxes + .iter() + .any(|mailbox| mailbox.mailbox_id == TRASH_ID || mailbox.mailbox_id == JUNK_ID) + { + batch.set( + ValueClass::Property(EmailField::DeletedAt.into()), + (metadata.rcvd_attach & MESSAGE_RECEIVED_MASK).serialize(), + ); + } + data.size = size; + batch + .set( + ValueClass::IndexProperty(IndexPropertyClass::Hash { + property: EmailField::Threading.into(), + hash: CheekyHash::new(if !subject.is_empty() { subject } else { "!" }), + }), + ThreadInfo::serialize(data.thread_id, &message_ids), + ) + .set( + Field::ARCHIVE, + Archiver::new(data.seal()) + .serialize() + .caused_by(trc::location!())?, + ) + .set( + EmailField::Metadata, + Archiver::new(metadata) + .serialize() + .caused_by(trc::location!())?, + ); server .store() .write(batch.build_all()) @@ -308,7 +342,7 @@ pub(crate) async fn migrate_emails_v011(server: &Server, account_id: u32) -> trc .assign_document_ids( account_id, Collection::Email, - message_ids.max().map(|id| id as u64).unwrap_or(num_emails) + 1, + document_ids.max().map(|id| id as u64).unwrap_or(num_emails) + 1, ) .await .caused_by(trc::location!())?; diff --git a/crates/migration/src/email_v2.rs b/crates/migration/src/email_v2.rs index 291825e3..9d8c34cd 100644 --- a/crates/migration/src/email_v2.rs +++ b/crates/migration/src/email_v2.rs @@ -7,7 +7,7 @@ use crate::{email_v1::FIELD_MAILBOX_IDS, get_bitmap, v014::SUBSPACE_BITMAP_TAG}; use common::Server; use email::{ - mailbox::UidMailbox, + mailbox::{JUNK_ID, TRASH_ID, UidMailbox}, message::{ index::extractors::VisitTextArchived, ingest::ThreadInfo, @@ -21,7 +21,8 @@ use email::{ }; use mail_parser::{Encoding, Header, parsers::fields::thread::thread_name}; use store::{ - SUBSPACE_INDEXES, Serialize, U32_LEN, U64_LEN, ValueKey, + Serialize, SerializeInfallible, U32_LEN, U64_LEN, ValueKey, + rand::{self, seq::SliceRandom}, write::{ AlignedBytes, AnyKey, Archive, Archiver, BatchBuilder, IndexPropertyClass, ValueClass, key::KeySerializer, @@ -57,8 +58,6 @@ pub(crate) async fn migrate_emails_v014(server: &Server, account_id: u32) -> trc .caused_by(trc::location!())? .unwrap_or_default(); - let todo = "delete indexes"; - let mut migrate = Vec::new(); server @@ -87,6 +86,8 @@ pub(crate) async fn migrate_emails_v014(server: &Server, account_id: u32) -> trc .await .caused_by(trc::location!())?; + migrate.shuffle(&mut rand::rng()); + let num_emails = migrate.len(); for (document_id, legacy_data) in migrate { let mut batch = BatchBuilder::new(); @@ -162,13 +163,24 @@ pub(crate) async fn migrate_emails_v014(server: &Server, account_id: u32) -> trc } } + if data + .mailboxes + .iter() + .any(|mailbox| mailbox.mailbox_id == TRASH_ID || mailbox.mailbox_id == JUNK_ID) + { + batch.set( + ValueClass::Property(EmailField::DeletedAt.into()), + (metadata.rcvd_attach & MESSAGE_RECEIVED_MASK).serialize(), + ); + } + batch .set( ValueClass::IndexProperty(IndexPropertyClass::Hash { property: EmailField::Threading.into(), hash: CheekyHash::new(if !subject.is_empty() { subject } else { "!" }), }), - ThreadInfo::serialize(legacy_data.thread_id, &message_ids), + ThreadInfo::serialize(data.thread_id, &message_ids), ) .set( EmailField::Archive, @@ -193,48 +205,6 @@ pub(crate) async fn migrate_emails_v014(server: &Server, account_id: u32) -> trc .caused_by(trc::location!())?; } - /* - - EmailField::From => 87, - EmailField::To => 35, - EmailField::Cc => 74, - EmailField::Bcc => 69, - EmailField::Subject => 29, - EmailField::Size => 27, - EmailField::References => 20, - EmailField::MailboxIds => 7, - EmailField::ReceivedAt => 19, - EmailField::SentAt => 26, - EmailField::HasAttachment => 89, - - */ - - for index in [87u8, 35, 74, 69, 29, 27, 20, 7, 19, 26, 89] { - server - .store() - .delete_range( - AnyKey { - subspace: SUBSPACE_INDEXES, - key: KeySerializer::new(U64_LEN * 3) - .write(account_id) - .write(u8::from(Collection::Email)) - .write(index) - .finalize(), - }, - AnyKey { - subspace: SUBSPACE_INDEXES, - key: KeySerializer::new(U64_LEN * 4) - .write(account_id) - .write(u8::from(Collection::Email)) - .write(index) - .write(&[u8::MAX; 8][..]) - .finalize(), - }, - ) - .await - .caused_by(trc::location!())?; - } - Ok(num_emails as u64) } diff --git a/crates/migration/src/encryption_v2.rs b/crates/migration/src/encryption_v2.rs index 76924505..89f77469 100644 --- a/crates/migration/src/encryption_v2.rs +++ b/crates/migration/src/encryption_v2.rs @@ -10,7 +10,7 @@ use email::message::crypto::{ EncryptionParams, }; use store::{ - Deserialize, Serialize, ValueKey, + Serialize, ValueKey, write::{AlignedBytes, Archive, Archiver, BatchBuilder, ValueClass}, }; use trc::AddContext; diff --git a/crates/migration/src/identity_v2.rs b/crates/migration/src/identity_v2.rs deleted file mode 100644 index a91c6d8b..00000000 --- a/crates/migration/src/identity_v2.rs +++ /dev/null @@ -1,39 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use crate::get_document_ids; -use common::Server; -use store::write::BatchBuilder; -use trc::AddContext; -use types::{collection::Collection, field::IdentityField}; - -pub(crate) async fn migrate_identities_v014(server: &Server, account_id: u32) -> trc::Result { - let identity_ids = get_document_ids(server, account_id, Collection::Identity) - .await - .caused_by(trc::location!())? - .unwrap_or_default(); - let num_identities = identity_ids.len(); - if num_identities == 0 { - return Ok(0); - } - - let mut batch = BatchBuilder::new(); - batch.with_account_id(account_id); - - for document_id in identity_ids { - batch - .with_document(document_id) - .tag(IdentityField::DocumentId); - } - - server - .store() - .write(batch.build_all()) - .await - .caused_by(trc::location!())?; - - Ok(num_identities as u64) -} diff --git a/crates/migration/src/lib.rs b/crates/migration/src/lib.rs index 0e920b43..e0e09a43 100644 --- a/crates/migration/src/lib.rs +++ b/crates/migration/src/lib.rs @@ -5,11 +5,13 @@ */ use crate::{ + blob::migrate_blobs_v014, queue_v1::{migrate_queue_v011, migrate_queue_v012}, + queue_v2::migrate_queue_v014, v011::migrate_v0_11, v012::migrate_v0_12, v013::migrate_v0_13, - v014::SUBSPACE_BITMAP_ID, + v014::{SUBSPACE_BITMAP_ID, migrate_principal_v0_14, migrate_v0_14}, }; use common::{DATABASE_SCHEMA_VERSION, Server, manager::boot::DEFAULT_SETTINGS}; use std::time::Duration; @@ -38,7 +40,6 @@ pub mod encryption_v2; pub mod event_v1; pub mod event_v2; pub mod identity_v1; -pub mod identity_v2; pub mod mailbox; pub mod object; pub mod principal_v1; @@ -64,45 +65,75 @@ const LOCK_WAIT_TIME_CORE: u64 = 5 * 60; const LOCK_RETRY_TIME: Duration = Duration::from_secs(30); pub async fn try_migrate(server: &Server) -> trc::Result<()> { - if let Some(version) = std::env::var("FORCE_MIGRATE_QUEUE") - .ok() - .and_then(|s| s.parse::().ok()) - { - if version == 12 || version <= 2 { - migrate_queue_v012(server) - .await - .caused_by(trc::location!())?; - } else { - migrate_queue_v011(server) - .await - .caused_by(trc::location!())?; - } - return Ok(()); - } - if let Some(version) = std::env::var("FORCE_MIGRATE") - .ok() - .and_then(|s| s.parse::().ok()) - { - match version { - 1 => { - migrate_v0_12(server, true) + for var in [ + "FORCE_MIGRATE_QUEUE", + "FORCE_MIGRATE_BLOBS", + "FORCE_MIGRATE_ACCOUNT", + "FORCE_MIGRATE", + ] { + let Some(version) = std::env::var(var).ok().and_then(|s| s.parse::().ok()) else { + continue; + }; + match var { + "FORCE_MIGRATE_QUEUE" => match version { + 1 => { + migrate_queue_v011(server) + .await + .caused_by(trc::location!())?; + } + 2 => { + migrate_queue_v012(server) + .await + .caused_by(trc::location!())?; + } + 4 => { + migrate_queue_v014(server) + .await + .caused_by(trc::location!())?; + } + _ => { + panic!("Unknown migration queue version: {version}"); + } + }, + "FORCE_MIGRATE_BLOBS" => { + migrate_blobs_v014(server) .await .caused_by(trc::location!())?; - migrate_v0_13(server).await.caused_by(trc::location!())?; } - 2 => { - migrate_v0_12(server, false) + "FORCE_MIGRATE" => match version { + 1 => { + migrate_v0_12(server, true) + .await + .caused_by(trc::location!())?; + migrate_v0_13(server).await.caused_by(trc::location!())?; + migrate_v0_14(server).await.caused_by(trc::location!())?; + } + 2 => { + migrate_v0_12(server, false) + .await + .caused_by(trc::location!())?; + migrate_v0_13(server).await.caused_by(trc::location!())?; + migrate_v0_14(server).await.caused_by(trc::location!())?; + } + 3 => { + migrate_v0_13(server).await.caused_by(trc::location!())?; + migrate_v0_14(server).await.caused_by(trc::location!())?; + } + 4 => { + migrate_v0_14(server).await.caused_by(trc::location!())?; + } + _ => { + panic!("Unknown migration version: {version}"); + } + }, + "FORCE_MIGRATE_ACCOUNT" => { + migrate_principal_v0_14(server, version) .await .caused_by(trc::location!())?; - migrate_v0_13(server).await.caused_by(trc::location!())?; - } - 3 => { - migrate_v0_13(server).await.caused_by(trc::location!())?; - } - _ => { - panic!("Unknown migration version: {version}"); } + _ => unreachable!(), } + return Ok(()); } @@ -136,6 +167,10 @@ pub async fn try_migrate(server: &Server) -> trc::Result<()> { migrate_v0_13(server).await.caused_by(trc::location!())?; false } + Some(4) => { + migrate_v0_14(server).await.caused_by(trc::location!())?; + false + } Some(version) => { panic!( "Unknown database schema version, expected {} or below, found {}", diff --git a/crates/migration/src/queue_v2.rs b/crates/migration/src/queue_v2.rs index f48de657..a68968b8 100644 --- a/crates/migration/src/queue_v2.rs +++ b/crates/migration/src/queue_v2.rs @@ -18,7 +18,7 @@ use store::{ Deserialize, IterateParams, Serialize, ValueKey, write::{ AlignedBytes, Archive, Archiver, BatchBuilder, QueueClass, ValueClass, - key::DeserializeBigEndian, now, + key::DeserializeBigEndian, }, }; use trc::AddContext; @@ -153,10 +153,8 @@ pub enum LegacyQuotaKey { pub(crate) async fn migrate_queue_v014(server: &Server) -> trc::Result<()> { let mut count = 0; - let now = now(); let mut messages = Vec::new(); - server .store() .iterate( @@ -184,7 +182,7 @@ pub(crate) async fn migrate_queue_v014(server: &Server) -> trc::Result<()> { }, ) .await - .caused_by(trc::location!()); + .caused_by(trc::location!())?; let mut batch = BatchBuilder::new(); for (queue_id, message) in messages { diff --git a/crates/migration/src/tasks_v2.rs b/crates/migration/src/tasks_v2.rs index f3f440ca..c5a20b54 100644 --- a/crates/migration/src/tasks_v2.rs +++ b/crates/migration/src/tasks_v2.rs @@ -31,6 +31,8 @@ pub(crate) async fn migrate_tasks_v014(server: &Server) -> trc::Result<()> { }), }; + let todo = "task epochs"; + let mut delete_tasks = Vec::new(); server .core diff --git a/crates/migration/src/v014.rs b/crates/migration/src/v014.rs index dc8971b9..2bc60b54 100644 --- a/crates/migration/src/v014.rs +++ b/crates/migration/src/v014.rs @@ -4,13 +4,30 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::Server; -use store::{ - SUBSPACE_INDEXES, U64_LEN, - write::{AnyKey, key::KeySerializer}, +use crate::{ + blob::migrate_blobs_v014, email_v2::migrate_emails_v014, + encryption_v2::migrate_encryption_params_v014, queue_v2::migrate_queue_v014, + tasks_v2::migrate_tasks_v014, }; +use common::Server; +use directory::backend::internal::manage::ManageDirectory; +use email::submission::EmailSubmission; +use groupware::{calendar::CalendarEventNotification, contact::ContactCard}; +use std::sync::Arc; +use store::{ + SUBSPACE_INDEXES, SerializeInfallible, U32_LEN, U64_LEN, + rand::{self, seq::SliceRandom}, + write::{ + AnyKey, BatchBuilder, IndexPropertyClass, Operation, ValueClass, ValueOp, + key::KeySerializer, + }, +}; +use tokio::sync::Semaphore; use trc::AddContext; -use types::collection::Collection; +use types::{ + collection::Collection, + field::{CalendarNotificationField, ContactField, EmailSubmissionField, IdentityField}, +}; pub const SUBSPACE_BITMAP_ID: u8 = b'b'; pub const SUBSPACE_BITMAP_TAG: u8 = b'c'; @@ -19,16 +36,98 @@ pub const SUBSPACE_FTS_INDEX: u8 = b'g'; pub const SUBSPACE_TELEMETRY_INDEX: u8 = b'w'; pub(crate) async fn migrate_v0_14(server: &Server) -> trc::Result<()> { - /* + // Migrate global data + let mut tasks = Vec::new(); + let _server = server.clone(); + tasks.push(tokio::spawn( + async move { migrate_queue_v014(&_server).await }, + )); + let _server = server.clone(); + tasks.push(tokio::spawn( + async move { migrate_blobs_v014(&_server).await }, + )); + let _server = server.clone(); + tasks.push(tokio::spawn( + async move { migrate_tasks_v014(&_server).await }, + )); + futures::future::join_all(tasks) + .await + .into_iter() + .collect::, _>>() + .map_err(|err| { + trc::EventType::Server(trc::ServerEvent::ThreadError) + .reason(err) + .caused_by(trc::location!()) + .details("Join Error") + })??; - - ContactField - - CalendarField - - EmailSubmissionField - - CalendarNotificationField + // Migrate account data + let mut principal_ids = server + .store() + .principal_ids(None, None) + .await + .unwrap_or_default() + .into_iter() + .collect::>(); + principal_ids.shuffle(&mut rand::rng()); + let semaphore = Arc::new(Semaphore::new( + std::env::var("NUM_THREADS") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(8), + )); + let mut tasks = Vec::with_capacity(principal_ids.len()); + for principal_id in principal_ids { + let permit = semaphore.clone().acquire_owned().await.unwrap(); + let _server = server.clone(); + tasks.push(tokio::spawn(async move { + let result = migrate_principal_v0_14(&_server, principal_id).await; + drop(permit); + result + })); + } + futures::future::join_all(tasks) + .await + .into_iter() + .collect::, _>>() + .map_err(|err| { + trc::EventType::Server(trc::ServerEvent::ThreadError) + .reason(err) + .caused_by(trc::location!()) + .details("Join Error") + })??; - */ + // Delete old subspaces + for subspace in [ + SUBSPACE_BITMAP_ID, + SUBSPACE_BITMAP_TAG, + SUBSPACE_BITMAP_TEXT, + SUBSPACE_FTS_INDEX, + SUBSPACE_TELEMETRY_INDEX, + ] { + server + .store() + .delete_range( + AnyKey { + subspace, + key: vec![0u8], + }, + AnyKey { + subspace, + key: vec![u8::MAX; 32], + }, + ) + .await + .caused_by(trc::location!())?; + } - todo!() + Ok(()) +} + +pub(crate) async fn migrate_principal_v0_14(server: &Server, account_id: u32) -> trc::Result<()> { + migrate_emails_v014(server, account_id).await?; + migrate_encryption_params_v014(server, account_id).await?; + migrate_indexes(server, account_id).await } pub(crate) async fn migrate_indexes(server: &Server, account_id: u32) -> trc::Result<()> { @@ -58,9 +157,29 @@ pub(crate) async fn migrate_indexes(server: &Server, account_id: u32) -> trc::Re CalendarField::EventId => 5, */ + /* + + EmailField::From => 87, + EmailField::To => 35, + EmailField::Cc => 74, + EmailField::Bcc => 69, + EmailField::Subject => 29, + EmailField::Size => 27, + EmailField::References => 20, + EmailField::MailboxIds => 7, + EmailField::ReceivedAt => 19, + EmailField::SentAt => 26, + EmailField::HasAttachment => 89, + + */ + for (collection, fields) in [ - (Collection::EmailSubmission, &[41u8, 83, 33, 95, 24][..]), - (Collection::ContactCard, &[2, 3, 4][..]), + ( + Collection::Email, + &[87u8, 35, 74, 69, 29, 27, 20, 7, 19, 26, 89][..], + ), + (Collection::EmailSubmission, &[41, 83, 33, 95, 24][..]), + (Collection::ContactCard, &[1, 2, 3, 4][..]), (Collection::CalendarEvent, &[1, 2, 3, 4][..]), (Collection::CalendarEventNotification, &[2, 5][..]), ] { @@ -91,5 +210,140 @@ pub(crate) async fn migrate_indexes(server: &Server, account_id: u32) -> trc::Re } } + let mut indexes = Vec::new(); + for collection in [ + Collection::ContactCard, + Collection::CalendarEventNotification, + Collection::EmailSubmission, + Collection::Identity, + ] { + server + .archives( + account_id, + Collection::ContactCard, + &(), + |document_id, archive| { + match collection { + Collection::ContactCard => { + let data = archive + .unarchive_untrusted::() + .caused_by(trc::location!())?; + + if let Some(email) = data.emails().next() { + indexes.push(( + collection, + document_id, + Operation::Index { + field: ContactField::Email.into(), + key: email.into_bytes(), + set: true, + }, + )); + } + indexes.push(( + collection, + document_id, + Operation::Value { + class: ValueClass::IndexProperty(IndexPropertyClass::Integer { + property: ContactField::CreatedToUpdated.into(), + value: data.created.to_native() as u64, + }), + op: ValueOp::Set( + (data.modified.to_native() as u64).serialize(), + ), + }, + )); + } + Collection::CalendarEventNotification => { + let data = archive + .unarchive_untrusted::() + .caused_by(trc::location!())?; + indexes.push(( + collection, + document_id, + Operation::Value { + class: ValueClass::IndexProperty(IndexPropertyClass::Integer { + property: CalendarNotificationField::CreatedToId.into(), + value: data.created.to_native() as u64, + }), + op: ValueOp::Set( + data.event_id + .as_ref() + .map(|v| v.to_native()) + .unwrap_or(u32::MAX) + .serialize(), + ), + }, + )); + } + Collection::EmailSubmission => { + let data = archive + .unarchive_untrusted::() + .caused_by(trc::location!())?; + indexes.push(( + collection, + document_id, + Operation::Value { + class: ValueClass::IndexProperty(IndexPropertyClass::Integer { + property: EmailSubmissionField::Metadata.into(), + value: data.send_at.to_native(), + }), + op: ValueOp::Set( + KeySerializer::new(U32_LEN * 3 + 1) + .write(data.email_id.to_native()) + .write(data.thread_id.to_native()) + .write(data.identity_id.to_native()) + .write(data.undo_status.as_index()) + .finalize(), + ), + }, + )); + } + Collection::Identity => { + indexes.push(( + collection, + document_id, + Operation::Index { + field: IdentityField::DocumentId.into(), + key: vec![], + set: true, + }, + )); + } + _ => unreachable!(), + } + + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + } + + let mut batch = BatchBuilder::new(); + for (collection, document_id, op) in indexes { + batch + .with_account_id(account_id) + .with_collection(collection) + .with_document(document_id) + .any_op(op); + if batch.is_large_batch() || batch.len() == 255 { + server + .store() + .write(batch.build_all()) + .await + .caused_by(trc::location!())?; + batch = BatchBuilder::new(); + } + } + + if !batch.is_empty() { + server + .store() + .write(batch.build_all()) + .await + .caused_by(trc::location!())?; + } + Ok(()) } diff --git a/crates/nlp/src/classifier/reservoir.rs b/crates/nlp/src/classifier/reservoir.rs index 90a0cc61..aff79d3d 100644 --- a/crates/nlp/src/classifier/reservoir.rs +++ b/crates/nlp/src/classifier/reservoir.rs @@ -38,6 +38,16 @@ impl SampleReservoir { } } + pub fn update_counts(&mut self, is_spam: bool) { + let class = if is_spam { + &mut self.spam + } else { + &mut self.ham + }; + + class.total_seen += 1; + } + pub fn replay_samples( &mut self, count_needed: usize, diff --git a/crates/spam-filter/src/modules/classifier.rs b/crates/spam-filter/src/modules/classifier.rs index 43794ace..4b8c9c2d 100644 --- a/crates/spam-filter/src/modules/classifier.rs +++ b/crates/spam-filter/src/modules/classifier.rs @@ -222,6 +222,8 @@ impl SpamClassifier for Server { is_spam, config.reservoir_capacity, ); + } else { + trainer.reservoir.update_counts(is_spam); } samples.push(TrainingTask { @@ -254,6 +256,20 @@ impl SpamClassifier for Server { Elapsed = started.elapsed() ); + return Ok(()); + } else if (trainer.reservoir.ham.total_seen < config.min_ham_samples) + || (trainer.reservoir.spam.total_seen < config.min_spam_samples) + { + trc::event!( + Spam(SpamEvent::ModelNotReady), + Reason = "Not enough samples for training", + Details = vec![ + trc::Value::from(ham_count + trainer.reservoir.ham.total_seen), + trc::Value::from(spam_count + trainer.reservoir.spam.total_seen) + ], + Elapsed = started.elapsed() + ); + return Ok(()); } @@ -263,7 +279,7 @@ impl SpamClassifier for Server { samples.extend( trainer .reservoir - .replay_samples(spam_count - ham_count, false) + .replay_samples((spam_count - ham_count) as usize, false) .map(|sample| TrainingTask { sample: sample.clone(), is_spam: false, @@ -276,7 +292,7 @@ impl SpamClassifier for Server { samples.extend( trainer .reservoir - .replay_samples(ham_count - spam_count, true) + .replay_samples((ham_count - spam_count) as usize, true) .map(|sample| TrainingTask { sample: sample.clone(), is_spam: true, @@ -290,11 +306,7 @@ impl SpamClassifier for Server { samples.shuffle(&mut StdRng::seed_from_u64(42)); // Spawn training task - let epochs = match trainer.reservoir.ham.total_seen - + trainer.reservoir.spam.total_seen - + spam_count as u64 - + ham_count as u64 - { + let epochs = match trainer.reservoir.ham.total_seen + trainer.reservoir.spam.total_seen { 0..=2500 => 3, // Bootstrap 2_501..=10_000 => 2, // Refinement _ => 1, // Full online training diff --git a/crates/store/src/backend/mysql/search.rs b/crates/store/src/backend/mysql/search.rs index 265018c1..236e984c 100644 --- a/crates/store/src/backend/mysql/search.rs +++ b/crates/store/src/backend/mysql/search.rs @@ -140,10 +140,9 @@ fn build_filter(query: &mut String, filters: &[SearchFilter]) -> Vec { if field.is_text() && matches!(op, SearchOperator::Equal | SearchOperator::Contains) { let (value, mode) = match (value, op) { - (SearchValue::Text { value, .. }, SearchOperator::Equal) => ( - Value::Bytes(format!("{value:?}").into_bytes()), - "NATURAL LANGUAGE", - ), + (SearchValue::Text { value, .. }, SearchOperator::Equal) => { + (Value::Bytes(format!("{value:?}").into_bytes()), "BOOLEAN") + } (SearchValue::Text { value, .. }, ..) => { let mut text_query = String::with_capacity(value.len() + 1); @@ -167,12 +166,12 @@ fn build_filter(query: &mut String, filters: &[SearchFilter]) -> Vec { } else if let SearchValue::KeyValues(kv) = value { let (key, value) = kv.iter().next().unwrap(); - values.push(Value::Bytes(format!("$.{key}").into_bytes())); + values.push(Value::Bytes(format!("$.{key:?}").into_bytes())); if !value.is_empty() { if op == &SearchOperator::Equal { let _ = write!(query, "JSON_EXTRACT({}, ?) = ?", field.column()); - values.push(Value::Bytes(format!("{value:?}").into_bytes())); + values.push(Value::Bytes(value.as_bytes().to_vec())); } else { let _ = write!(query, "JSON_EXTRACT({}, ?) LIKE ?", field.column(),); values.push(Value::Bytes(format!("%{value}%").into_bytes())); diff --git a/crates/store/src/backend/postgres/write.rs b/crates/store/src/backend/postgres/write.rs index 88b7f23d..4067d826 100644 --- a/crates/store/src/backend/postgres/write.rs +++ b/crates/store/src/backend/postgres/write.rs @@ -48,6 +48,7 @@ impl PostgresStore { Some(&SqlState::UNIQUE_VIOLATION) => { return Err(trc::StoreEvent::AssertValueFailed .into_err() + .reason("Unique violation") .caused_by(trc::location!())); } _ => return Err(into_error(err)), diff --git a/crates/store/src/search/document.rs b/crates/store/src/search/document.rs index a5ac3461..2d18796b 100644 --- a/crates/store/src/search/document.rs +++ b/crates/store/src/search/document.rs @@ -82,20 +82,26 @@ impl IndexDocument { pub fn insert_key_value( &mut self, field: impl Into, - key: impl Into, + key: impl AsRef, value: impl Into, ) { let search_field = field.into(); + let key = key.as_ref().to_lowercase(); match self.fields.entry(search_field) { Entry::Occupied(mut entry) => { if let SearchValue::KeyValues(existing_key_values) = entry.get_mut() { - existing_key_values.append(key.into(), value.into()); + if let Some(existing_value) = existing_key_values.get_mut(&key) { + existing_value.push(' '); + existing_value.push_str(&value.into()); + } else { + existing_key_values.append(key, value.into()); + } } } Entry::Vacant(entry) => { let mut new_key_values = VecMap::new(); - new_key_values.append(key.into(), value.into()); + new_key_values.append(key, value.into()); entry.insert(SearchValue::KeyValues(new_key_values)); } } diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 4d0b57ab..1cbb22d9 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -5,7 +5,8 @@ edition = "2024" [features] #default = ["sqlite", "postgres", "mysql", "rocks", "s3", "redis", "nats", "azure", "foundationdb"] -default = ["rocks", "foundationdb"] +default = ["sqlite", "postgres", "mysql", "rocks", "s3", "redis", "foundationdb"] +#default = ["rocks", "foundationdb"] sqlite = ["store/sqlite"] foundationdb = ["store/foundation", "common/foundation"] postgres = ["store/postgres"] diff --git a/tests/resources/smtp/antispam/classifier.test b/tests/resources/smtp/antispam/classifier.test index 4b369b9a..d3a6d66a 100644 --- a/tests/resources/smtp/antispam/classifier.test +++ b/tests/resources/smtp/antispam/classifier.test @@ -1,5 +1,5 @@ envelope_to hello@world.com -expect PROB_SPAM_MEDIUM +expect PROB_SPAM_HIGH Subject: save up to NUMBER on life insurance @@ -7,14 +7,14 @@ why spend more than you have to life quote savings ensuring your family s financ envelope_to hello@world.com -expect PROB_HAM_MEDIUM +expect PROB_HAM_HIGH Subject: can someone explain what type of operating system solaris is as ive never seen or used it i dont know wheather to get a server from sun or from dell i would prefer a linux based server and sun seems to be the one for that but im not sure if solaris is a distro of linux or a completely different operating system can someone explain kiall mac innes irish linux users group ilug URL URL for un subscription information list maintainer listmaster URL envelope_to hello@world.com -expect PROB_SPAM_LOW +expect PROB_SPAM_UNCERTAIN Subject: Lorem ipsum dolor sit amet, consectetur adipiscing elit diff --git a/tests/resources/smtp/antispam/classifier_features.test b/tests/resources/smtp/antispam/classifier_features.test index 357f13c4..1f86b820 100644 --- a/tests/resources/smtp/antispam/classifier_features.test +++ b/tests/resources/smtp/antispam/classifier_features.test @@ -9,10 +9,6 @@ I'm going to need those TPS reports ASAP. So, if you could do that, that'd be gr "type": "word", "value": "_allcaps" }, - { - "type": "word", - "value": "_word" - }, { "type": "word", "value": "asap" @@ -88,10 +84,6 @@ This is the end of MIME multipart. [ - { - "type": "word", - "value": "_word" - }, { "type": "word", "value": "anoth" @@ -180,10 +172,6 @@ Portal: IP-Sperre [ - { - "type": "word", - "value": "_word" - }, { "type": "word", "value": "adress" @@ -337,10 +325,6 @@ OteuCc4AZGN7L3Jzv953ptzc3Nzb3JvBpybfF9KgeQd5/2cN2nfX855TlqJbsKGAAIIIIAAAggg= "type": "word", "value": "_null" }, - { - "type": "word", - "value": "_word" - }, { "type": "word", "value": "browser" @@ -587,10 +571,6 @@ e; CLICK AQUÍ CLICK AQUÍ CLICK AQUÍ CLICK AQUÍ"= "type": "word", "value": "_null" }, - { - "type": "word", - "value": "_word" - }, { "type": "word", "value": "acces" @@ -1025,10 +1005,6 @@ vοΌ₯ⓑ𝔂 π”½π•ŒΕ‡β„•ο½™ ţ乇𝕏𝓣 [ - { - "type": "word", - "value": "_word" - }, { "type": "word", "value": "email" @@ -1210,10 +1186,6 @@ velsignet m=C3=A5ned med stor succes.
Hr. Charles Jackson Jr. [ - { - "type": "word", - "value": "_word" - }, { "type": "word", "value": "anbefal" @@ -1567,10 +1539,6 @@ Uniting UK is a unionist campaign opposite news reader that's memory dug nobody [ - { - "type": "word", - "value": "_word" - }, { "type": "word", "value": "better" @@ -1722,10 +1690,6 @@ enlarged prostate as well.
"type": "word", "value": "_allcaps" }, - { - "type": "word", - "value": "_word" - }, { "type": "word", "value": "accident" @@ -2138,10 +2102,6 @@ important;"> [ - { - "type": "word", - "value": "_word" - }, { "type": "word", "value": "amplifi" @@ -2636,10 +2596,6 @@ poHmsYINCg== [ - { - "type": "word", - "value": "_word" - }, { "type": "word", "value": "ad" diff --git a/tests/src/directory/oidc.rs b/tests/src/directory/oidc.rs index 1363ac04..93929ca6 100644 --- a/tests/src/directory/oidc.rs +++ b/tests/src/directory/oidc.rs @@ -8,21 +8,19 @@ * */ -use std::sync::Arc; - +use crate::{ + directory::DirectoryTest, + http_server::{HttpMessage, spawn_mock_http_server}, +}; use base64::{Engine, engine::general_purpose}; use directory::QueryParams; use http_proto::{JsonProblemResponse, JsonResponse, ToHttpResponse}; use hyper::{Method, StatusCode}; use mail_send::Credentials; use serde_json::json; +use std::sync::Arc; use trc::{AuthEvent, EventType}; -use crate::{ - directory::DirectoryTest, - http_server::{HttpMessage, spawn_mock_http_server}, -}; - static TEST_TOKEN: &str = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ"; #[tokio::test] diff --git a/tests/src/imap/basic.rs b/tests/src/imap/basic.rs index 2099a22e..921b3af4 100644 --- a/tests/src/imap/basic.rs +++ b/tests/src/imap/basic.rs @@ -14,6 +14,25 @@ use super::{AssertResult, ImapConnection, Type}; pub async fn test(imap: &mut ImapConnection, _imap_check: &mut ImapConnection) { println!("Running basic tests..."); + // Test OAuth Bearer decoding + assert!( + Credentials::OAuthBearer { + token: "vF9dft4qmTc2Nvb3RlckBhbHRhdmlzdGEuY29tCg==".to_string() + } == sasl_decode_challenge_oauth( + &base64_decode( + concat!( + "bixhPXVzZXJAZXhhbXBsZS5jb20sAWhv", + "c3Q9c2VydmVyLmV4YW1wbGUuY29tAXBvcnQ9MTQzAWF1dGg9QmVhcmVyI", + "HZGOWRmdDRxbVRjMk52YjNSbGNrQmhiSFJoZG1semRHRXVZMjl0Q2c9PQ", + "EB" + ) + .as_bytes(), + ) + .unwrap(), + ) + .unwrap() + ); + // Test CAPABILITY imap.send("CAPABILITY").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; @@ -38,24 +57,3 @@ pub async fn test(imap: &mut ImapConnection, _imap_check: &mut ImapConnection) { imap.send_untagged("AGJvYXR5AG1jYm9hdGZhY2U=").await; imap.assert_read(Type::Tagged, ResponseType::No).await; } - -#[test] -fn decode_challenge() { - assert!( - Credentials::OAuthBearer { - token: "vF9dft4qmTc2Nvb3RlckBhbHRhdmlzdGEuY29tCg==".to_string() - } == sasl_decode_challenge_oauth( - &base64_decode( - concat!( - "bixhPXVzZXJAZXhhbXBsZS5jb20sAWhv", - "c3Q9c2VydmVyLmV4YW1wbGUuY29tAXBvcnQ9MTQzAWF1dGg9QmVhcmVyI", - "HZGOWRmdDRxbVRjMk52YjNSbGNrQmhiSFJoZG1semRHRXVZMjl0Q2c9PQ", - "EB" - ) - .as_bytes(), - ) - .unwrap(), - ) - .unwrap() - ); -} diff --git a/tests/src/imap/body_structure.rs b/tests/src/imap/body_structure.rs index c8db6d87..de10cda0 100644 --- a/tests/src/imap/body_structure.rs +++ b/tests/src/imap/body_structure.rs @@ -4,8 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::fs; - +use super::resources_dir; use email::message::metadata::{MessageMetadata, build_metadata_contents}; use imap::op::fetch::AsImapDataItem; use imap_proto::{ @@ -13,16 +12,14 @@ use imap_proto::{ protocol::fetch::{BodyContents, DataItem, Section}, }; use mail_parser::MessageParser; +use std::fs; use store::{ Deserialize, Serialize, write::{Archive, Archiver}, }; use utils::chained_bytes::ChainedBytes; -use super::resources_dir; - -#[test] -fn imap_test_body_structure() { +pub fn test() { println!("Running BODYSTRUCTURE..."); for file_name in fs::read_dir(resources_dir()).unwrap() { diff --git a/tests/src/imap/mailbox.rs b/tests/src/imap/mailbox.rs index 88cdae35..06cf55de 100644 --- a/tests/src/imap/mailbox.rs +++ b/tests/src/imap/mailbox.rs @@ -12,6 +12,9 @@ use super::{AssertResult, ImapConnection, Type}; pub async fn test(mut imap: &mut ImapConnection, mut imap_check: &mut ImapConnection) { println!("Running mailbox tests..."); + // Pattern matching tests + mailbox_matches_pattern(); + // Create third connection for testing let mut other_conn = ImapConnection::connect(b"_z ").await; other_conn @@ -323,7 +326,6 @@ pub async fn test(mut imap: &mut ImapConnection, mut imap_check: &mut ImapConnec imap.assert_read(Type::Tagged, ResponseType::Ok).await; } -#[test] fn mailbox_matches_pattern() { let mailboxes = [ "imaptest", diff --git a/tests/src/imap/mod.rs b/tests/src/imap/mod.rs index 30fba5f9..4e269d65 100644 --- a/tests/src/imap/mod.rs +++ b/tests/src/imap/mod.rs @@ -66,6 +66,9 @@ pub async fn imap_tests() { let delete = true; let handle = init_imap_tests(delete).await; + // Body structure tests + body_structure::test(); + // Connect to IMAP server let mut imap_check = ImapConnection::connect(b"_y ").await; let mut imap = ImapConnection::connect(b"_x ").await; diff --git a/tests/src/jmap/contacts/contact.rs b/tests/src/jmap/contacts/contact.rs index 1b02b86b..e9877dbf 100644 --- a/tests/src/jmap/contacts/contact.rs +++ b/tests/src/jmap/contacts/contact.rs @@ -337,6 +337,11 @@ pub async fn test(params: &mut JMAPTest) { // Query tests wait_for_index(¶ms.server).await; + let email = if !params.server.search_store().is_mysql() { + "sarah.johnson@example.com" + } else { + "sarah.johnson@example" + }; assert_eq!( account .jmap_query( @@ -345,7 +350,7 @@ pub async fn test(params: &mut JMAPTest) { ("text", "Sarah"), ("inAddressBook", book1_id.as_str()), ("uid", "urn:uuid:f81d4fae-7dec-11d0-a765-00a0c91e6bf6"), - ("email", "sarah.johnson@example.com"), + ("email", email), ], ["created"], Vec::<(&str, &str)>::new(), diff --git a/tests/src/jmap/core/push_subscription.rs b/tests/src/jmap/core/push_subscription.rs index 0ce3a565..682b8f3e 100644 --- a/tests/src/jmap/core/push_subscription.rs +++ b/tests/src/jmap/core/push_subscription.rs @@ -50,6 +50,9 @@ default = true pub async fn test(params: &mut JMAPTest) { println!("Running Push Subscription tests..."); + // ECE roundtrip test + ece_roundtrip(); + // Create test account let account = params.account("jdoe@example.com"); let client = account.client(); @@ -357,7 +360,6 @@ async fn assert_state(event_rx: &mut mpsc::Receiver, id: &Id, state ); } -#[test] fn ece_roundtrip() { for len in [1, 2, 5, 16, 256, 1024, 2048, 4096, 1024 * 1024] { let (keypair, auth_secret) = ece::generate_keypair_and_auth_secret().unwrap(); diff --git a/tests/src/jmap/mail/crypto.rs b/tests/src/jmap/mail/crypto.rs index 1c941193..7c151772 100644 --- a/tests/src/jmap/mail/crypto.rs +++ b/tests/src/jmap/mail/crypto.rs @@ -18,6 +18,10 @@ use store::{ pub async fn test(params: &mut JMAPTest) { println!("Running Encryption-at-rest tests..."); + // Check encryption + check_is_encrypted(); + import_certs_and_encrypt().await; + // Create test account let account = params.account("jdoe@example.com"); let client = account.client(); @@ -157,7 +161,6 @@ pub async fn test(params: &mut JMAPTest) { } } -#[tokio::test] pub async fn import_certs_and_encrypt() { for (name, method, expected_certs) in [ ("cert_pgp.pem", EncryptionMethod::PGP, 1), @@ -223,7 +226,6 @@ pub async fn import_certs_and_encrypt() { ); } -#[test] pub fn check_is_encrypted() { let messages = std::fs::read_to_string( PathBuf::from(env!("CARGO_MANIFEST_DIR")) diff --git a/tests/src/jmap/mail/delivery.rs b/tests/src/jmap/mail/delivery.rs index 996ffc9e..d4711bd4 100644 --- a/tests/src/jmap/mail/delivery.rs +++ b/tests/src/jmap/mail/delivery.rs @@ -295,6 +295,8 @@ END:VCARD ) .await; + tokio::time::sleep(Duration::from_millis(200)).await; + for (account, num_messages) in [(john, 6), (jane, 1), (bill, 1)] { assert_eq!( server diff --git a/tests/src/jmap/mail/query.rs b/tests/src/jmap/mail/query.rs index 02750192..51dd5e0a 100644 --- a/tests/src/jmap/mail/query.rs +++ b/tests/src/jmap/mail/query.rs @@ -161,10 +161,14 @@ pub async fn query(client: &Client, can_stem: bool) { email::query::Comparator::subject(), email::query::Comparator::sent_at(), ], - vec![ - "T10330", "N01744", "N01743", "N04885", "N02688", "N02122", "A00059", "A00058", - "N02123", "T00651", "T09439", "N05001", "T05848", "T05508", - ], + if can_stem { + vec![ + "T10330", "N01744", "N01743", "N04885", "N02688", "N02122", "A00059", "A00058", + "N02123", "T00651", "T09439", "N05001", "T05848", "T05508", + ] + } else { + vec!["T10330", "N02122", "N02123", "T09439"] + }, ), ( Filter::and(vec![ diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 60cc7f44..2742b6fc 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -78,9 +78,9 @@ async fn jmap_tests() { server::webhooks::test(&mut params).await; - mail::get::test(&mut params).await; + /*mail::get::test(&mut params).await; mail::set::test(&mut params).await; - mail::parse::test(&mut params).await; + mail::parse::test(&mut params).await;*/ mail::query::test(&mut params, delete).await; mail::search_snippet::test(&mut params).await; mail::changes::test(&mut params).await; diff --git a/tests/src/store/cleanup.rs b/tests/src/store/cleanup.rs index 37c12842..497d0c05 100644 --- a/tests/src/store/cleanup.rs +++ b/tests/src/store/cleanup.rs @@ -155,7 +155,9 @@ pub async fn store_blob_expire_all(store: &Store) { until, })); } - _ => {} + _ => { + eprintln!("Unknown blob link type for key {key:?}: {value:?}",); + } } batch.clear(ValueClass::Blob(BlobOp::Link { diff --git a/tests/src/store/import_export.rs b/tests/src/store/import_export.rs index d28e30fd..ed0a696d 100644 --- a/tests/src/store/import_export.rs +++ b/tests/src/store/import_export.rs @@ -230,6 +230,7 @@ pub async fn test(db: Store) { // Destroy store store_destroy(&db).await; + store_assert_is_empty(&db, db.clone().into(), true).await; temp_dir.delete(); } diff --git a/tests/src/store/mod.rs b/tests/src/store/mod.rs index 49878393..9da56c92 100644 --- a/tests/src/store/mod.rs +++ b/tests/src/store/mod.rs @@ -55,7 +55,7 @@ pub async fn store_tests() { #[tokio::test(flavor = "multi_thread")] pub async fn search_tests() { - let insert = true; + let insert = std::env::var("NO_INSERT").is_err(); let temp_dir = TempDir::new("search_store_tests", insert); let mut config = Config::new(build_store_config(&temp_dir.path.to_string_lossy())) .unwrap() @@ -171,13 +171,13 @@ disable = {ELASTIC_ENABLED} username = "elastic" secret = "changeme" -[store."s3"] -type = "s3" -access-key = "minioadmin" -secret-key = "minioadmin" -region = "eu-central-1" -endpoint = "http://localhost:9000" -bucket = "tmp" +#[store."s3"] +#type = "s3" +#access-key = "minioadmin" +#secret-key = "minioadmin" +#region = "eu-central-1" +#endpoint = "http://localhost:9000" +#bucket = "tmp" [store."fs"] type = "fs" diff --git a/tests/src/store/ops.rs b/tests/src/store/ops.rs index e749a46d..7a8279cd 100644 --- a/tests/src/store/ops.rs +++ b/tests/src/store/ops.rs @@ -195,27 +195,41 @@ pub async fn test(db: Store) { handles.push({ let db = db.clone(); tokio::spawn(async move { - let mut builder = BatchBuilder::new(); - builder - .with_account_id(0) - .with_collection(Collection::Email) - .with_document(0) - .merge_fnc( - ValueClass::Property(3), - Params::with_capacity(0), - |_, _, bytes| { - if let Some(bytes) = bytes { - Ok(MergeResult::Update( - (u64::from_be_bytes(bytes.try_into().unwrap()) + 1) - .to_be_bytes() - .to_vec(), - )) - } else { - Ok(MergeResult::Update(0u64.to_be_bytes().to_vec())) - } - }, - ); - db.write(builder.build_all()).await.unwrap() + for _ in 0..5 { + let mut builder = BatchBuilder::new(); + builder + .with_account_id(0) + .with_collection(Collection::Email) + .with_document(0) + .merge_fnc( + ValueClass::Property(3), + Params::with_capacity(0), + |_, _, bytes| { + if let Some(bytes) = bytes { + Ok(MergeResult::Update( + (u64::from_be_bytes(bytes.try_into().unwrap()) + 1) + .to_be_bytes() + .to_vec(), + )) + } else { + Ok(MergeResult::Update(0u64.to_be_bytes().to_vec())) + } + }, + ); + + match db.write(builder.build_all()).await { + Ok(_) => { + break; + } + Err(e) if e.is_assertion_failure() => { + // Retry on assertion failures + continue; + } + Err(e) => { + panic!("Merge failed: {:?}", e); + } + } + } }) }); } diff --git a/tests/src/store/query.rs b/tests/src/store/query.rs index b7d3393a..5a6865f1 100644 --- a/tests/src/store/query.rs +++ b/tests/src/store/query.rs @@ -179,7 +179,9 @@ pub async fn test(store: SearchStore, do_insert: bool) { | EmailSearchField::Attachment => { document.index_text( FIELD_MAPPINGS[pos].clone(), - &field.to_lowercase(), + &field + .replace(|ch: char| !ch.is_alphanumeric(), " ") + .to_lowercase(), Language::English, ); }