diff --git a/Cargo.lock b/Cargo.lock index b07d389a..d0b71419 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -36,7 +36,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ "crypto-common", - "generic-array 0.14.9", + "generic-array 0.14.7", ] [[package]] @@ -150,22 +150,22 @@ dependencies = [ [[package]] name = "anstyle-query" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "anstyle-wincon" -version = "3.0.10" +version = "3.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -252,7 +252,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", "synstructure", ] @@ -264,7 +264,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -334,7 +334,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -356,7 +356,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -367,7 +367,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -382,7 +382,7 @@ version = "0.28.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07a9b245ba0739fc90935094c29adbaee3f977218b5fb95e822e261cda7f56a3" dependencies = [ - "http 1.3.1", + "http 1.4.0", "log", "rustls 0.23.35", "serde", @@ -521,7 +521,7 @@ dependencies = [ "miniz_oxide", "object", "rustc-demangle", - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -606,7 +606,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -624,7 +624,7 @@ dependencies = [ "regex", "rustc-hash 2.1.1", "shlex", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -741,7 +741,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", ] [[package]] @@ -750,7 +750,7 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", ] [[package]] @@ -759,7 +759,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", ] [[package]] @@ -827,7 +827,7 @@ checksum = "89385e82b5d1821d2219e0b095efa2cc1f246cbf99080f3be46a1a85c0d392d9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -844,9 +844,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" dependencies = [ "serde", ] @@ -943,9 +943,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.44" +version = "1.2.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37521ac7aabe3d13122dc382493e20c9416f299d2ccd5b3a5340a2570cdeb0f3" +checksum = "c481bdbf0ed3b892f6f806287d72acd515b352a4ec27a208489b8c1bc839633a" dependencies = [ "find-msvc-tools", "jobserver", @@ -1021,7 +1021,7 @@ dependencies = [ "pure-rust-locales", "serde", "wasm-bindgen", - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -1040,7 +1040,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12f8e7987cbd042a63249497f41aed09f8e65add917ea6566effbc56578d6801" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", ] [[package]] @@ -1067,9 +1067,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.51" +version = "4.5.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c26d721170e0295f191a69bd9a1f93efcdb0aff38684b61ab5750468972e5f5" +checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8" dependencies = [ "clap_builder", "clap_derive", @@ -1077,9 +1077,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.51" +version = "4.5.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75835f0c7bf681bfd05abe44e965760fea999a5286c6eb2d59883634fd02011a" +checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00" dependencies = [ "anstream", "anstyle", @@ -1096,7 +1096,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -1164,11 +1164,11 @@ dependencies = [ "futures", "hashify", "hostname", - "hyper 1.7.0", + "hyper 1.8.1", "idna", "imagesize", "imap_proto", - "indexmap 2.12.0", + "indexmap 2.12.1", "infer 0.19.0", "jmap_proto", "libc", @@ -1219,7 +1219,7 @@ dependencies = [ "whatlang", "x509-parser", "xxhash-rust", - "zip 6.0.0", + "zip", ] [[package]] @@ -1367,9 +1367,9 @@ dependencies = [ [[package]] name = "crc" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" dependencies = [ "crc-catalog", ] @@ -1469,7 +1469,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", "rand_core 0.6.4", "subtle", "zeroize", @@ -1477,11 +1477,11 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", "rand_core 0.6.4", "typenum", ] @@ -1492,7 +1492,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", "subtle", ] @@ -1550,7 +1550,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -1584,7 +1584,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -1598,7 +1598,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -1609,7 +1609,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -1620,7 +1620,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core 0.21.3", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -1648,7 +1648,7 @@ dependencies = [ "groupware", "hashify", "http_proto", - "hyper 1.7.0", + "hyper 1.8.1", "percent-encoding", "rkyv", "store", @@ -1665,9 +1665,9 @@ dependencies = [ "chrono", "compact_str", "hashify", - "hyper 1.7.0", + "hyper 1.8.1", "mail-parser", - "quick-xml 0.38.3", + "quick-xml 0.38.4", "rkyv", "serde", "serde_json", @@ -1681,7 +1681,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd2735a791158376708f9347fe8faba9667589d82427ef3aed6794a8981de3d9" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", ] [[package]] @@ -1790,7 +1790,7 @@ checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -1808,7 +1808,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", ] [[package]] @@ -1914,7 +1914,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -2065,7 +2065,7 @@ dependencies = [ "crypto-bigint", "digest 0.10.7", "ff", - "generic-array 0.14.9", + "generic-array 0.14.7", "group", "hkdf", "pem-rfc7468", @@ -2156,7 +2156,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -2208,7 +2208,7 @@ version = "0.14.1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -2297,9 +2297,9 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "find-msvc-tools" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" +checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" [[package]] name = "fixedbitset" @@ -2377,7 +2377,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3757d8523a60f4d3516a018e767cd3bea0829a43045d290ace16dcc92ac4f4cd" dependencies = [ "bytes", - "http 1.3.1", + "http 1.4.0", "httparse", "memchr", "mime", @@ -2434,7 +2434,7 @@ checksum = "c4ca5370149145ec3741cd7e82832f17f893b9421ee4e484d9511c6702bd9911" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", "try_map", ] @@ -2535,7 +2535,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -2570,9 +2570,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.9" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -2596,7 +2596,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" dependencies = [ "rustix", - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -2670,7 +2670,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -2723,7 +2723,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.12.0", + "indexmap 2.12.1", "slab", "tokio", "tokio-util", @@ -2741,8 +2741,8 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.3.1", - "indexmap 2.12.0", + "http 1.4.0", + "indexmap 2.12.1", "slab", "tokio", "tokio-util", @@ -2778,9 +2778,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "allocator-api2", "equivalent", @@ -2795,7 +2795,7 @@ checksum = "149e3ea90eb5a26ad354cfe3cb7f7401b9329032d0235f2687d03a35f30e5d4c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -2975,13 +2975,13 @@ dependencies = [ [[package]] name = "hostname" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56f203cd1c76362b69e3863fd987520ac36cf70a8c92627449b2f64a8cf7d65" +checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd" dependencies = [ "cfg-if", "libc", - "windows-link 0.1.3", + "windows-link", ] [[package]] @@ -3011,7 +3011,7 @@ dependencies = [ "groupware", "http-body-util", "http_proto", - "hyper 1.7.0", + "hyper 1.8.1", "hyper-util", "jmap", "jmap_proto", @@ -3021,7 +3021,7 @@ dependencies = [ "mail-send", "mime", "pkcs8", - "quick-xml 0.38.3", + "quick-xml 0.38.4", "rev_lines", "rkyv", "rsa", @@ -3043,12 +3043,11 @@ dependencies = [ [[package]] name = "http" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" dependencies = [ "bytes", - "fnv", "itoa", ] @@ -3070,7 +3069,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.3.1", + "http 1.4.0", ] [[package]] @@ -3081,7 +3080,7 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "pin-project-lite", ] @@ -3114,7 +3113,7 @@ dependencies = [ "compact_str", "form_urlencoded", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.1", "hyper-util", "percent-encoding", "serde", @@ -3171,16 +3170,16 @@ dependencies = [ [[package]] name = "hyper" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", "h2 0.4.12", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "httparse", "httpdate", @@ -3212,8 +3211,8 @@ version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "http 1.3.1", - "hyper 1.7.0", + "http 1.4.0", + "hyper 1.8.1", "hyper-util", "rustls 0.23.35", "rustls-pki-types", @@ -3229,7 +3228,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.7.0", + "hyper 1.8.1", "hyper-util", "pin-project-lite", "tokio", @@ -3238,18 +3237,18 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" +checksum = "52e9a2a24dc5c6821e71a7030e1e14b7b632acac55c40e9d2e082c621261bb56" dependencies = [ "base64 0.22.1", "bytes", "futures-channel", "futures-core", "futures-util", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", - "hyper 1.7.0", + "hyper 1.8.1", "ipnet", "libc", "percent-encoding", @@ -3417,7 +3416,7 @@ dependencies = [ "directory", "email", "imap_proto", - "indexmap 2.12.0", + "indexmap 2.12.1", "mail-parser", "mail-send", "md5 0.8.0", @@ -3472,7 +3471,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", "zstd", ] @@ -3499,12 +3498,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" +checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" dependencies = [ "equivalent", - "hashbrown 0.16.0", + "hashbrown 0.16.1", "serde", "serde_core", ] @@ -3544,7 +3543,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ "block-padding", - "generic-array 0.14.9", + "generic-array 0.14.7", ] [[package]] @@ -3715,7 +3714,7 @@ dependencies = [ "hkdf", "http-body-util", "http_proto", - "hyper 1.7.0", + "hyper 1.8.1", "hyper-util", "jmap-tools", "jmap_proto", @@ -3858,9 +3857,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.82" +version = "0.3.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b011eec8cc36da2aab2d5cff675ec18454fad408585853910a202391cf9f8e65" +checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" dependencies = [ "once_cell", "wasm-bindgen", @@ -3901,7 +3900,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.12.1", ] [[package]] @@ -4006,15 +4005,15 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.177" +version = "0.2.178" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" +checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" [[package]] name = "libflate" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "249fa21ba2b59e8cbd69e722f5b31e1b466db96c937ae3de23e8b99ead0d1383" +checksum = "e3248b8d211bd23a104a42d81b4fa8bb8ac4a3b75e7a43d85d2c9ccb6179cd74" dependencies = [ "adler32", "core2", @@ -4030,7 +4029,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a599cb10a9cd92b1300debcef28da8f70b935ec937f44fcd1b70a7c986a11c5c" dependencies = [ "core2", - "hashbrown 0.16.0", + "hashbrown 0.16.1", "rle-decode-fast", ] @@ -4041,7 +4040,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" dependencies = [ "cfg-if", - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -4098,9 +4097,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.22" +version = "1.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" +checksum = "15d118bbf3771060e7311cc7bb0545b01d08a8b4a7de949198dec1fa0ca1c0f7" dependencies = [ "cc", "libc", @@ -4205,9 +4204,9 @@ dependencies = [ [[package]] name = "mail-auth" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94cf926be0f7007993709797fcb467b16b0d22acea729af7543c8b4eed5a8bb7" +checksum = "4028c805e3302c7b38bd0582090525a927eeb05c8d1da6ae67cd3d3d92d9b906" dependencies = [ "ahash", "flate2", @@ -4215,7 +4214,7 @@ dependencies = [ "hickory-resolver", "mail-builder", "mail-parser", - "quick-xml 0.38.3", + "quick-xml 0.38.4", "quick_cache", "rand 0.8.5", "ring", @@ -4224,7 +4223,7 @@ dependencies = [ "rustls-pemfile 2.2.0", "serde", "serde_json", - "zip 5.1.1", + "zip", ] [[package]] @@ -4316,7 +4315,7 @@ checksum = "5cf92c10c7e361d6b99666ec1c6f9805b0bea2c3bd8c78dc6fe98ac5bd78db11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -4483,7 +4482,7 @@ checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -4499,7 +4498,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", "termcolor", "thiserror 2.0.17", ] @@ -4704,9 +4703,9 @@ dependencies = [ [[package]] name = "num-bigint-dig" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82c79c15c05d4bf82b6f5ef163104cc81a760d8e874d38ac50ab67c8877b647b" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" dependencies = [ "lazy_static", "libm", @@ -4783,7 +4782,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -4846,9 +4845,9 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "openssl" -version = "0.10.74" +version = "0.10.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24ad14dd45412269e1a30f52ad8f0664f0f4f4a89ee8fe28c3b3527021ebb654" +checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" dependencies = [ "bitflags", "cfg-if", @@ -4867,7 +4866,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -4878,9 +4877,9 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-sys" -version = "0.9.110" +version = "0.9.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a9f0075ba3c21b09f8e8b2026584b1d18d49388648f2fbbf3c97ea8deced8e2" +checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" dependencies = [ "cc", "libc", @@ -4910,7 +4909,7 @@ checksum = "46d7ab32b827b5b495bd90fa95a6cb65ccc293555dcc3199ae2937d2d237c8ed" dependencies = [ "async-trait", "bytes", - "http 1.3.1", + "http 1.4.0", "opentelemetry", "reqwest", "tracing", @@ -4923,7 +4922,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d899720fe06916ccba71c01d04ecd77312734e2de3467fd30d9d580c8ce85656" dependencies = [ "futures-core", - "http 1.3.1", + "http 1.4.0", "opentelemetry", "opentelemetry-http", "opentelemetry-proto", @@ -5052,7 +5051,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -5111,9 +5110,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.3" +version = "2.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "989e7521a040efde50c3ab6bbadafbe15ab6dc042686926be59ac35d74607df4" +checksum = "cbcfd20a6d4eeba40179f05735784ad32bdaef05ce8e8af05f180d45bb3e7e22" dependencies = [ "memchr", "ucd-trie", @@ -5121,9 +5120,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.8.3" +version = "2.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187da9a3030dbafabbbfb20cb323b976dc7b7ce91fcd84f2f74d6e31d378e2de" +checksum = "51f72981ade67b1ca6adc26ec221be9f463f2b5839c7508998daa17c23d94d7f" dependencies = [ "pest", "pest_generator", @@ -5131,22 +5130,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.8.3" +version = "2.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49b401d98f5757ebe97a26085998d6c0eecec4995cad6ab7fc30ffdf4b052843" +checksum = "dee9efd8cdb50d719a80088b76f81aec7c41ed6d522ee750178f83883d271625" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] name = "pest_meta" -version = "2.8.3" +version = "2.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f27a2cfee9f9039c4d86faa5af122a0ac3851441a34865b8a043b46be0065a" +checksum = "bf1d70880e76bdc13ba52eafa6239ce793d85c8e43896507e43dd8984ff05b82" dependencies = [ "pest", "sha2 0.10.9", @@ -5159,7 +5158,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset 0.4.2", - "indexmap 2.12.0", + "indexmap 2.12.1", ] [[package]] @@ -5170,7 +5169,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset 0.5.7", "hashbrown 0.15.5", - "indexmap 2.12.0", + "indexmap 2.12.1", "serde", ] @@ -5224,7 +5223,7 @@ dependencies = [ "phf_shared 0.13.1", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -5271,7 +5270,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -5456,7 +5455,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -5544,7 +5543,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -5562,7 +5561,7 @@ version = "0.14.1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -5599,7 +5598,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -5614,9 +5613,9 @@ dependencies = [ [[package]] name = "psl" -version = "2.1.157" +version = "2.1.166" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dccfdc08041021deeacc4b8ccc2441e0cce1cdc61e0f91e695514a363a00bd0" +checksum = "2085080c7de45d70a59d96aa7d5b0870fc1ccbd27adb780ee1e1ec905da42035" dependencies = [ "psl-types", ] @@ -5644,7 +5643,7 @@ checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -5690,9 +5689,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.38.3" +version = "0.38.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42a232e7487fc2ef313d96dde7948e7a3c05101870d8985e4fd8d26aedd27b89" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" dependencies = [ "memchr", ] @@ -5705,7 +5704,7 @@ checksum = "7ada44a88ef953a3294f6eb55d2007ba44646015e18613d2f213016379203ef3" dependencies = [ "ahash", "equivalent", - "hashbrown 0.16.0", + "hashbrown 0.16.1", "parking_lot", ] @@ -5768,9 +5767,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.41" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" dependencies = [ "proc-macro2", ] @@ -6137,7 +6136,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -6190,10 +6189,10 @@ dependencies = [ "futures-core", "futures-util", "h2 0.4.12", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.1", "hyper-rustls 0.27.7", "hyper-util", "js-sys", @@ -6224,9 +6223,9 @@ dependencies = [ [[package]] name = "resolv-conf" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b3789b30bd25ba102de4beabd95d21ac45b69b1be7d14522bab988c526d6799" +checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] name = "rev_lines" @@ -6289,7 +6288,7 @@ dependencies = [ "bytecheck", "bytes", "hashbrown 0.15.5", - "indexmap 2.12.0", + "indexmap 2.12.1", "munge", "ptr_meta", "rancor", @@ -6307,7 +6306,7 @@ checksum = "bd83f5f173ff41e00337d97f6572e416d022ef8a19f371817259ae960324c482" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -6362,9 +6361,9 @@ dependencies = [ [[package]] name = "rsa" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78928ac1ed176a5ca1d17e578a1825f3d81ca54cf41053a592584b020cfd691b" +checksum = "40a0376c50d0358279d9d643e4bf7b7be212f1f4ff1da9070a7b54d22ef75c88" dependencies = [ "const-oid", "digest 0.10.7", @@ -6600,9 +6599,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94182ad936a0c91c324cd46c6511b9510ed16af436d7b5bab34beab0afd55f7a" +checksum = "708c0f9d5f54ba0272468c1d306a52c495b31fa155e91bc25371e6df7996908c" dependencies = [ "web-time", "zeroize", @@ -6757,9 +6756,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.0.5" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1317c3bf3e7df961da95b0a56a172a02abead31276215a0497241a7624b487ce" +checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289" dependencies = [ "dyn-clone", "ref-cast", @@ -6776,7 +6775,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -6821,7 +6820,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct", "der", - "generic-array 0.14.9", + "generic-array 0.14.7", "pkcs8", "subtle", "zeroize", @@ -6881,9 +6880,9 @@ checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" [[package]] name = "sequoia-openpgp" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "015e5fc3d023418b9db98ca9a7f3e90b305872eeafe5ca45c5c32b5eb335c1e8" +checksum = "f0e334ce3ec5b9b47d86a80563b3ecec435f59acf37e86058b3b686a42c5a2ba" dependencies = [ "aes", "aes-gcm", @@ -6895,6 +6894,7 @@ dependencies = [ "buffered-reader", "camellia", "cast5", + "cbc", "cfb-mode", "chrono", "cipher 0.4.4", @@ -6974,7 +6974,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -6985,7 +6985,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -6994,7 +6994,7 @@ version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.12.1", "itoa", "memchr", "ryu", @@ -7030,7 +7030,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -7047,17 +7047,17 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.15.1" +version = "3.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa66c845eee442168b2c8134fec70ac50dc20e760769c8ba0ad1319ca1959b04" +checksum = "4fa237f2807440d238e0364a218270b98f767a00d3dada77b1c53ae88940e2e7" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.12.0", + "indexmap 2.12.1", "schemars 0.9.0", - "schemars 1.0.5", + "schemars 1.1.0", "serde_core", "serde_json", "serde_with_macros", @@ -7066,14 +7066,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.15.1" +version = "3.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91a903660542fced4e99881aa481bdbaec1634568ee02e0b8bd57c64cb38955" +checksum = "52a8e3ca0ca629121f70ab50f95249e5a6f925cc0f6ffe8256c45b728875706c" dependencies = [ "darling 0.21.3", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -7082,7 +7082,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.12.1", "itoa", "ryu", "serde", @@ -7111,7 +7111,7 @@ checksum = "5d69265a08751de7844521fd15003ae0a888e035773ba05695c5c759a6f89eef" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -7272,9 +7272,9 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.6" +version = "1.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" +checksum = "7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad" dependencies = [ "libc", ] @@ -7344,7 +7344,7 @@ dependencies = [ "email", "form_urlencoded", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.1", "hyper-util", "lru-cache", "mail-auth", @@ -7440,7 +7440,7 @@ dependencies = [ "common", "compact_str", "decancer", - "hyper 1.7.0", + "hyper 1.8.1", "idna", "infer 0.19.0", "mail-auth", @@ -7456,6 +7456,7 @@ dependencies = [ "store", "tokio", "trc", + "types", "unicode-security", "utils", ] @@ -7686,9 +7687,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.108" +version = "2.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da58917d35242480a05c2897064da0a80589a2a0476c9a3f2fdc83b53502e917" +checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" dependencies = [ "proc-macro2", "quote", @@ -7712,7 +7713,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -7773,7 +7774,7 @@ dependencies = [ "http 0.14.1", "http-body-util", "http_proto", - "hyper 1.7.0", + "hyper 1.8.1", "hyper-util", "imap", "imap_proto", @@ -7790,7 +7791,7 @@ dependencies = [ "nlp", "num_cpus", "pop3", - "quick-xml 0.38.3", + "quick-xml 0.38.4", "rayon", "reqwest", "ring", @@ -7840,7 +7841,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -7851,7 +7852,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -7931,9 +7932,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tls-listener" -version = "0.11.0" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab41256c16d6fc2b3021545f20bf77a73200b18bd54040ac656dddfca6205bfa" +checksum = "1461056cc1ef47003f7ee16e4cef3741068d4c7f6b627bfce49b7c00c120a530" dependencies = [ "futures-util", "pin-project-lite", @@ -7944,11 +7945,13 @@ dependencies = [ [[package]] name = "token-cell" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9c7b0772e96c7fa6646b16c116753b3d1db503400209237230aa992c9e3a269" +checksum = "fb48920ae769b58126c8c93269805011c793201f95fde28b479b81a9a531bbde" dependencies = [ "paste", + "portable-atomic", + "rustversion", ] [[package]] @@ -7976,7 +7979,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -8076,7 +8079,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", - "http 1.3.1", + "http 1.4.0", "httparse", "rand 0.8.5", "ring", @@ -8102,7 +8105,7 @@ version = "0.23.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.12.1", "toml_datetime", "toml_parser", "winnow", @@ -8126,10 +8129,10 @@ dependencies = [ "async-trait", "base64 0.22.1", "bytes", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.1", "hyper-timeout", "hyper-util", "percent-encoding", @@ -8198,14 +8201,14 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" +checksum = "9cf146f99d442e8e68e585f5d798ccd3cad9a7835b917e09728880a862706456" dependencies = [ "bitflags", "bytes", "futures-util", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "iri-string", "pin-project-lite", @@ -8228,9 +8231,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.41" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647" dependencies = [ "log", "pin-project-lite", @@ -8240,20 +8243,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] name = "tracing-core" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c" dependencies = [ "once_cell", "valuable", @@ -8282,9 +8285,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.20" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" +checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" dependencies = [ "matchers", "nu-ansi-term", @@ -8350,7 +8353,7 @@ checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" dependencies = [ "bytes", "data-encoding", - "http 1.3.1", + "http 1.4.0", "httparse", "log", "rand 0.9.2", @@ -8617,13 +8620,13 @@ dependencies = [ [[package]] name = "uuid" -version = "1.18.1" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" dependencies = [ "getrandom 0.3.4", "js-sys", - "serde", + "serde_core", "sha1_smol", "wasm-bindgen", ] @@ -8648,7 +8651,7 @@ checksum = "8c44ce98e7227a04eeb4cf9c784109a5c9710e54849ceb4f09f8597247897f1e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", "unzip-n", ] @@ -8736,9 +8739,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.105" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da95793dfc411fbbd93f5be7715b0578ec61fe87cb1a42b12eb625caa5c5ea60" +checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" dependencies = [ "cfg-if", "once_cell", @@ -8749,9 +8752,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.55" +version = "0.4.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "551f88106c6d5e7ccc7cd9a16f312dd3b5d36ea8b4954304657d5dfba115d4a0" +checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" dependencies = [ "cfg-if", "js-sys", @@ -8762,9 +8765,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.105" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04264334509e04a7bf8690f2384ef5265f05143a4bff3889ab7a3269adab59c2" +checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -8772,22 +8775,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.105" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "420bc339d9f322e562942d52e115d57e950d12d88983a14c79b86859ee6c7ebc" +checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.105" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76f218a38c84bcb33c25ec7059b07847d465ce0e0a76b995e134a45adcb6af76" +checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" dependencies = [ "unicode-ident", ] @@ -8807,9 +8810,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.82" +version = "0.3.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a1f95c0d03a47f4ae1f7a64643a6bb97465d9b740f0fa8f90ea33915c99a9a1" +checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" dependencies = [ "js-sys", "wasm-bindgen", @@ -8917,7 +8920,7 @@ checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ "windows-implement", "windows-interface", - "windows-link 0.2.1", + "windows-link", "windows-result", "windows-strings", ] @@ -8930,7 +8933,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -8941,15 +8944,9 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] -[[package]] -name = "windows-link" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" - [[package]] name = "windows-link" version = "0.2.1" @@ -8962,7 +8959,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -8971,7 +8968,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -9025,7 +9022,7 @@ version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -9080,7 +9077,7 @@ version = "0.53.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" dependencies = [ - "windows-link 0.2.1", + "windows-link", "windows_aarch64_gnullvm 0.53.1", "windows_aarch64_msvc 0.53.1", "windows_i686_gnu 0.53.1", @@ -9273,9 +9270,9 @@ checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] name = "winnow" -version = "0.7.13" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" +checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" dependencies = [ "memchr", ] @@ -9379,7 +9376,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", "synstructure", ] @@ -9523,7 +9520,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9495c0ca85331f87442b465760a7fb7334370354742a4201b719cc2533afee7" dependencies = [ "getrandom 0.2.16", - "hashbrown 0.16.0", + "hashbrown 0.16.1", "keyed-set", "rand 0.8.5", "schemars 0.8.22", @@ -9686,7 +9683,7 @@ checksum = "2bbff17ef687e10ed4f6a956fb78fc97205a4a6242c657638e8e5da726840aca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", "zenoh-keyexpr", ] @@ -9840,22 +9837,22 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.27" +version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.27" +version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -9875,7 +9872,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", "synstructure", ] @@ -9896,7 +9893,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.111", ] [[package]] @@ -9929,34 +9926,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", -] - -[[package]] -name = "zip" -version = "5.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f852905151ac8d4d06fdca66520a661c09730a74c6d4e2b0f27b436b382e532" -dependencies = [ - "aes", - "arbitrary", - "bzip2", - "constant_time_eq", - "crc32fast", - "deflate64", - "flate2", - "getrandom 0.3.4", - "hmac 0.12.1", - "indexmap 2.12.0", - "lzma-rust2", - "memchr", - "pbkdf2", - "ppmd-rust", - "sha1", - "time", - "zeroize", - "zopfli", - "zstd", + "syn 2.0.111", ] [[package]] @@ -9974,7 +9944,7 @@ dependencies = [ "flate2", "getrandom 0.3.4", "hmac 0.12.1", - "indexmap 2.12.0", + "indexmap 2.12.1", "lzma-rust2", "memchr", "pbkdf2", diff --git a/crates/common/src/config/spamfilter.rs b/crates/common/src/config/spamfilter.rs index e0789038..43a7478e 100644 --- a/crates/common/src/config/spamfilter.rs +++ b/crates/common/src/config/spamfilter.rs @@ -90,8 +90,7 @@ pub struct ClassifierConfig { pub alpha: f32, pub auto_learn_reply_ham: bool, pub auto_learn_card_is_ham: bool, - pub score_spam: f32, - pub score_ham: f32, + pub hold_samples_for: u64, } #[derive(Debug, Clone)] @@ -464,20 +463,18 @@ impl ClassifierConfig { .property_or_default("spam-filter.classifier.epochs", "1000") .unwrap_or(1000), alpha: config - .property_or_default("spam-filter.classifier.alpha", "0.0001") - .unwrap_or(0.0001), - score_spam: config - .property_or_default("spam-filter.classifier.score.spam", "0.7") - .unwrap_or(0.7), - score_ham: config - .property_or_default("spam-filter.classifier.score.ham", "0.5") - .unwrap_or(0.5), + .property_or_default("spam-filter.classifier.alpha", "0.00001") + .unwrap_or(0.00001), auto_learn_card_is_ham: config .property_or_default("spam-filter.classifier.auto-learn.card-is-ham", "true") .unwrap_or(true), auto_learn_reply_ham: config .property_or_default("spam-filter.classifier.auto-learn.trusted-reply", "true") .unwrap_or(true), + hold_samples_for: config + .property_or_default::("spam-filter.classifier.hold-samples-for", "365d") + .unwrap_or(Duration::from_secs(365 * 24 * 60 * 60)) + .as_secs(), } .into() } diff --git a/crates/common/src/core.rs b/crates/common/src/core.rs index 4f0b4111..d8746f9f 100644 --- a/crates/common/src/core.rs +++ b/crates/common/src/core.rs @@ -961,7 +961,7 @@ impl Server { hash: hash.clone(), to: BlobLink::Temporary { until }, }, - vec![], + vec![BlobLink::QUOTA_LINK], ) .set( BlobOp::Quota { diff --git a/crates/email/src/message/ingest.rs b/crates/email/src/message/ingest.rs index ee14bbec..e9b61d38 100644 --- a/crates/email/src/message/ingest.rs +++ b/crates/email/src/message/ingest.rs @@ -11,7 +11,7 @@ use crate::{ message::{ crypto::EncryptionParams, index::{IndexMessage, extractors::VisitText}, - metadata::MessageData, + metadata::{MessageData, MessageMetadata}, }, }; use common::{Server, auth::AccessToken}; @@ -21,13 +21,13 @@ use groupware::{ scheduling::{ItipError, ItipMessages}, }; use mail_parser::{ - Header, HeaderName, HeaderValue, Message, MessageParser, MimeHeaders, PartType, + DateTime, Header, HeaderName, HeaderValue, Message, MessageParser, MimeHeaders, PartType, parsers::fields::thread::thread_name, }; use std::{borrow::Cow, cmp::Ordering, fmt::Write, time::Instant}; use std::{future::Future, hash::Hasher}; use store::{ - IndexKeyPrefix, IterateParams, SerializeInfallible, U32_LEN, ValueKey, + IndexKeyPrefix, IterateParams, U32_LEN, ValueKey, ahash::{AHashMap, AHashSet}, write::{ AssignedId, AssignedIds, BatchBuilder, BlobLink, BlobOp, IndexPropertyClass, SearchIndex, @@ -100,6 +100,20 @@ pub trait EmailIngest: Sync + Send { mailbox_ids: impl IntoIterator + Sync + Send, generate_email_id: bool, ) -> impl Future + 'static>> + Send; + fn add_account_spam_sample( + &self, + batch: &mut BatchBuilder, + account_id: u32, + document_id: u32, + is_spam: bool, + ) -> impl Future> + Send; + fn add_spam_sample( + &self, + batch: &mut BatchBuilder, + hash: BlobHash, + hold_override: Option, + is_spam: bool, + ); } pub struct ThreadResult { @@ -255,7 +269,7 @@ impl EmailIngest for Server { { is_spam = false; train_spam = Some(false); - overridden = Some("card-found"); + overridden = Some("card-exists"); } // Check if the message is a trusted reply to a previous message @@ -590,28 +604,11 @@ impl EmailIngest for Server { // Request spam training if let Some(learn_spam) = train_spam { - let blob_hash = params.blob_hash.unwrap_or(&blob_hash).clone(); - let blob_expiry = if is_encrypted { - let hold_for = now() + 86400; // 24 hours - batch.set( - BlobOp::Link { - hash: blob_hash.clone(), - to: BlobLink::Temporary { until: hold_for }, - }, - vec![], - ); - hold_for.serialize() - } else { - vec![] - }; - - batch.set( - ValueClass::TaskQueue(TaskQueueClass::SpamTrain { - due: TaskEpoch::now(), - blob_hash, - learn_spam, - }), - blob_expiry, + self.add_spam_sample( + &mut batch, + params.blob_hash.unwrap_or(&blob_hash).clone(), + is_encrypted.then_some(2 * 86400), + learn_spam, ); } @@ -810,6 +807,58 @@ impl EmailIngest for Server { .ctx(trc::Key::Reason, "No all document ids were generated")) } } + + async fn add_account_spam_sample( + &self, + batch: &mut BatchBuilder, + account_id: u32, + document_id: u32, + is_spam: bool, + ) -> trc::Result<()> { + if let Some(archive) = self + .archive_by_property( + account_id, + Collection::Email, + document_id, + EmailField::Metadata.into(), + ) + .await + .caused_by(trc::location!())? + { + let metadata = archive + .to_unarchived::() + .caused_by(trc::location!())?; + self.add_spam_sample(batch, (&metadata.inner.blob_hash).into(), None, is_spam); + } + + Ok(()) + } + + fn add_spam_sample( + &self, + batch: &mut BatchBuilder, + hash: BlobHash, + hold_override: Option, + is_spam: bool, + ) { + if let Some(config) = &self.core.spam.classifier { + let mut dt = DateTime::from_timestamp(now() as i64); + dt.hour = 0; + dt.minute = 0; + dt.second = 0; + let until = dt.to_timestamp() as u64 + hold_override.unwrap_or(config.hold_samples_for); + + batch + .set( + BlobOp::Link { + hash: hash.clone(), + to: BlobLink::Temporary { until }, + }, + vec![BlobLink::SPAM_SAMPLE_LINK], + ) + .set(BlobOp::SpamSample { hash, until }, vec![u8::from(is_spam)]); + } + } } fn has_message_id(a: &[CheekyHash], b: &[u8]) -> bool { diff --git a/crates/imap/src/op/copy_move.rs b/crates/imap/src/op/copy_move.rs index 9fe14269..d8f12e76 100644 --- a/crates/imap/src/op/copy_move.rs +++ b/crates/imap/src/op/copy_move.rs @@ -27,7 +27,7 @@ use imap_proto::{ use std::{sync::Arc, time::Instant}; use store::{ roaring::RoaringBitmap, - write::{AlignedBytes, Archive, BatchBuilder, TaskEpoch, TaskQueueClass, ValueClass}, + write::{AlignedBytes, Archive, BatchBuilder}, }; use trc::AddContext; use types::{ @@ -204,7 +204,6 @@ impl SessionData { // Mailboxes are in the same account let account_id = src_mailbox.id.account_id; let dest_mailbox_id = UidMailbox::new_unassigned(dest_mailbox_id); - let mut has_spam_train_tasks = false; let mut batch = BatchBuilder::new(); for (id, imap_id) in ids { @@ -319,30 +318,23 @@ impl SessionData { ); } - // Add bayes train task - if dest_mailbox_id.mailbox_id == JUNK_ID { - batch.set( - ValueClass::TaskQueue(TaskQueueClass::SpamTrain { - due: TaskEpoch::now(), - blob_hash: Default::default(), - learn_spam: true, - }), - vec![], - ); - has_spam_train_tasks = true; + // Add message to training queue + let learn_spam = if dest_mailbox_id.mailbox_id == JUNK_ID { + Some(true) } else if src_mailbox.id.mailbox_id == JUNK_ID && dest_mailbox_id.mailbox_id != TRASH_ID { - batch.set( - ValueClass::TaskQueue(TaskQueueClass::SpamTrain { - due: TaskEpoch::now(), - blob_hash: Default::default(), - learn_spam: false, - }), - vec![], - ); - has_spam_train_tasks = true; + Some(false) + } else { + None + }; + if let Some(learn_spam) = learn_spam { + self.server + .add_account_spam_sample(&mut batch, account_id, id, learn_spam) + .await + .imap_ctx(&arguments.tag, trc::location!())?; } + batch.commit_point(); // Update changelog @@ -356,11 +348,6 @@ impl SessionData { .commit_batch(batch) .await .imap_ctx(&arguments.tag, trc::location!())?; - - // Trigger Bayes training - if has_spam_train_tasks { - self.server.notify_task_queue(); - } } else { // Obtain quota for target account let src_account_id = src_mailbox.id.account_id; diff --git a/crates/jmap/src/blob/copy.rs b/crates/jmap/src/blob/copy.rs index b1abae9c..05eccf05 100644 --- a/crates/jmap/src/blob/copy.rs +++ b/crates/jmap/src/blob/copy.rs @@ -78,7 +78,7 @@ impl BlobCopy for Server { hash: blob_id.hash.clone(), to: BlobLink::Temporary { until }, }, - 0u32.serialize(), + vec![], ); self.store() .write(batch.build_all()) diff --git a/crates/nlp/src/classifier/feature.rs b/crates/nlp/src/classifier/feature.rs index 9683ac44..39369df6 100644 --- a/crates/nlp/src/classifier/feature.rs +++ b/crates/nlp/src/classifier/feature.rs @@ -15,31 +15,72 @@ pub struct Sample { pub struct Features(pub(super) HashMap>); -pub struct SampleBuilder { +pub struct FeatureBuilder { pub(super) features_mask: u32, } -impl SampleBuilder { - pub fn build(&self, features: I, class: f32) -> Sample - where - I: IntoIterator, - I::Item: AsRef<[u8]>, - { - let mut features_map = HashMap::with_capacity_and_hasher(128, BuildNoHashHasher::default()); +pub trait Feature { + fn prefix(&self) -> u16; + fn value(&self) -> &[u8]; + fn is_global_feature(&self) -> bool; + fn is_local_feature(&self) -> bool; +} - for feature in features { - let feature = feature.as_ref(); - let hash = xxh3_64_with_seed(feature, 0) as u32; - let hash_sign = xxh3_64_with_seed(feature, 1); +impl FeatureBuilder { + pub fn scale(&self, features: &mut HashMap) { + // Log frequency scaling + for x in features.values_mut() { + *x = x.ln_1p(); + } + } - *features_map.entry(hash & self.features_mask).or_default() += - if hash_sign & 1 == 0 { 1.0 } else { -1.0 }; + pub fn build( + &self, + features: &HashMap, + account_id: Option, + ) -> Features { + // Do the "hash trick" + let mut features_map = + HashMap::with_capacity_and_hasher(features.len(), BuildNoHashHasher::default()); + let mut buf = Vec::with_capacity(2 + 4 + 63); + for (feature, count) in features { + buf.extend_from_slice(&feature.prefix().to_be_bytes()); + buf.extend_from_slice(feature.value()); + + if feature.is_global_feature() { + let big_hash = xxh3_64_with_seed(&buf, 0); + let hash = big_hash as u32 & self.features_mask; + let sign = if big_hash & (1 << 63) == 0 { 1.0 } else { -1.0 }; + + *features_map.entry(hash).or_default() += sign * count; + } + + if feature.is_local_feature() + && let Some(account_id) = account_id + { + buf.extend_from_slice(&account_id.to_be_bytes()); + let big_hash = xxh3_64_with_seed(&buf, 0); + let hash = big_hash as u32 & self.features_mask; + let sign = if big_hash & (1 << 63) == 0 { 1.0 } else { -1.0 }; + + *features_map.entry(hash).or_default() += sign * count; + } + buf.clear(); } - Sample { - features: Features(features_map), - class, + // L2 normalization + let sum_of_squares = features_map + .values() + .map(|&x| x as f64 * x as f64) + .sum::(); + if sum_of_squares > 0.0 { + let norm = sum_of_squares.sqrt() as f32; + for x in features_map.values_mut() { + *x /= norm; + } } + + Features(features_map) } } diff --git a/crates/nlp/src/classifier/sgd.rs b/crates/nlp/src/classifier/sgd.rs index 27b52d03..e218d374 100644 --- a/crates/nlp/src/classifier/sgd.rs +++ b/crates/nlp/src/classifier/sgd.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::classifier::feature::{Features, Sample, SampleBuilder}; +use crate::classifier::feature::{FeatureBuilder, Features, Sample}; use rand::{SeedableRng, rngs::StdRng, seq::SliceRandom}; #[derive(Default)] @@ -114,8 +114,8 @@ impl SGDClassifier { .collect() } - pub fn sample_builder(&self) -> SampleBuilder { - SampleBuilder { + pub fn feature_builder(&self) -> FeatureBuilder { + FeatureBuilder { features_mask: (self.weights.len() - 1) as u32, } } @@ -163,9 +163,13 @@ fn log1pexp(x: f32) -> f32 { #[cfg(test)] pub mod tests { - use crate::classifier::{feature::Sample, sgd::SGDClassifier}; + use crate::classifier::{ + feature::{Feature, Sample}, + sgd::SGDClassifier, + }; use rand::{SeedableRng, rngs::StdRng, seq::SliceRandom}; use std::{ + collections::HashMap, fs::File, io::{BufRead, BufReader}, time::Instant, @@ -262,6 +266,24 @@ pub mod tests { (train, test) } + impl Feature for String { + fn prefix(&self) -> u16 { + 0 + } + + fn value(&self) -> &[u8] { + self.as_bytes() + } + + fn is_global_feature(&self) -> bool { + true + } + + fn is_local_feature(&self) -> bool { + false + } + } + #[test] fn sgd_classifier() { let reader = BufReader::new( @@ -271,7 +293,7 @@ pub mod tests { let mut samples = Vec::with_capacity(1024); let mut model = SGDClassifier::new(1 << 20, 1000, 0.0001, 42); - let builder = model.sample_builder(); + let builder = model.feature_builder(); let time = Instant::now(); @@ -279,14 +301,18 @@ pub mod tests { let line = line.unwrap(); let (text, class) = line.trim().rsplit_once(',').unwrap(); let text = text.trim_start_matches('"').trim_end_matches('"'); - samples.push( - builder.build( - text.split_whitespace(), - class - .parse() - .unwrap_or_else(|_| panic!("Invalid class value: {line}")), - ), - ); + + let mut sample: HashMap = HashMap::new(); + for word in text.split_whitespace() { + *sample.entry(word.to_string()).or_default() += 1.0; + } + builder.scale(&mut sample); + samples.push(Sample { + features: builder.build(&sample, None), + class: class + .parse() + .unwrap_or_else(|_| panic!("Invalid class value: {line}")), + }); } println!("Loaded {} samples in {:?}", samples.len(), time.elapsed()); diff --git a/crates/nlp/src/tokenizers/stream.rs b/crates/nlp/src/tokenizers/stream.rs index 56056315..8212c249 100644 --- a/crates/nlp/src/tokenizers/stream.rs +++ b/crates/nlp/src/tokenizers/stream.rs @@ -15,21 +15,9 @@ use crate::{ }; use std::borrow::Cow; -pub struct StreamTokenizer>, I: StreamInputTokenTrait> { - stream: T, +pub struct WordStemTokenizer { stemmer: Stemmer, stop_words: Option, - tokens: Vec, -} - -pub enum StreamInputToken { - Word(String), - Other(T), -} - -pub trait StreamInputTokenTrait { - fn from_owned(word: String) -> Self; - fn from_borrowed(word: &str) -> Self; } enum Stemmer { @@ -39,8 +27,8 @@ enum Stemmer { None, } -impl>, I: StreamInputTokenTrait> StreamTokenizer { - pub fn new(text: &str, stream: T) -> Self { +impl WordStemTokenizer { + pub fn new(text: &str) -> Self { // Detect language let (mut language, score) = LanguageDetector::detect_single(text).unwrap_or((Language::English, 1.0)); @@ -49,7 +37,6 @@ impl>, I: StreamInputTokenTrait> StreamTo } Self { - stream, stemmer: match language { Language::Mandarin => Stemmer::Mandarin, Language::Japanese => Stemmer::Japanese, @@ -58,66 +45,31 @@ impl>, I: StreamInputTokenTrait> StreamTo .unwrap_or(Stemmer::None), }, stop_words: STOP_WORDS[language as usize], - tokens: vec![], } } -} -impl>, I: StreamInputTokenTrait> Iterator - for StreamTokenizer -{ - type Item = I; - - fn next(&mut self) -> Option { - if let Some(prev_token) = self.tokens.pop() { - return Some(prev_token); + pub fn tokenize<'x>(&self, word: &'x str, mut cb: impl FnMut(Cow<'x, str>)) { + if self.stop_words.is_some_and(|sw| sw(word)) { + return; } - - for token in self.stream.by_ref() { - return match token { - StreamInputToken::Word(word) => { - if self.stop_words.is_some_and(|sw| sw(word.as_str())) { - continue; - } - match &self.stemmer { - Stemmer::IndoEuropean(stemmer) => match stemmer.stem(&word) { - Cow::Borrowed(word) => I::from_borrowed(word), - Cow::Owned(stemmed_word) => I::from_owned(stemmed_word), - }, - Stemmer::Mandarin => { - let mut result = JIEBA.cut(&word, false).into_iter(); - if let Some(stemmed_word) = result.next() { - let stemmed_word = I::from_borrowed(stemmed_word); - self.tokens = result - .rev() - .map(|word| I::from_borrowed(word)) - .collect::>(); - stemmed_word - } else { - // This shouldn't happen, but just in case - continue; - } - } - Stemmer::Japanese => { - let mut result = japanese::tokenize(&word).into_iter(); - if let Some(stemmed_word) = result.next() { - self.tokens = - result.rev().map(|b| I::from_owned(b)).collect::>(); - I::from_owned(stemmed_word) - } else { - // This shouldn't happen, but just in case - continue; - } - } - Stemmer::None => I::from_owned(word), - } - } - StreamInputToken::Other(raw) => raw, + match &self.stemmer { + Stemmer::IndoEuropean(stemmer) => { + cb(stemmer.stem(word)); + } + Stemmer::Mandarin => { + for word in JIEBA.cut(word, false) { + cb(Cow::from(word)); + } + } + Stemmer::Japanese => { + for word in japanese::tokenize(word) { + cb(Cow::from(word)); + } + } + Stemmer::None => { + cb(Cow::from(word)); } - .into(); } - - None } } @@ -7872,89 +7824,12 @@ pub fn symbols(input: &str) -> bool { ) } -impl StreamInputTokenTrait for Vec { - fn from_owned(word: String) -> Self { - word.into_bytes() - } - - fn from_borrowed(word: &str) -> Self { - word.as_bytes().to_vec() - } -} - #[cfg(test)] pub mod tests { - use super::{StreamInputToken, symbols}; use crate::tokenizers::{ - stream::StreamTokenizer, + stream::WordStemTokenizer, types::{TokenType, TypesTokenizer}, }; - use std::{borrow::Cow, net::IpAddr}; - - pub trait ToStreamToken { - fn to_stream_token(&self) -> Option>>; - } - - impl, E: AsRef, U: AsRef, I: AsRef> ToStreamToken - for TokenType - { - fn to_stream_token(&self) -> Option>> { - match self { - TokenType::Alphabetic(word) => { - Some(StreamInputToken::Word(word.as_ref().to_lowercase())) - } - TokenType::Url(word) => { - let word = word.as_ref(); - word.split_once("://") - .map(|(_, host)| StreamInputToken::Other(url_host_as_bytes(host))) - } - TokenType::IpAddr(word) => word.as_ref().parse::().ok().map(|ip| { - StreamInputToken::Other(match ip { - IpAddr::V4(ip) => ip.octets().to_vec(), - IpAddr::V6(ip) => ip.octets().to_vec(), - }) - }), - TokenType::UrlNoScheme(word) => { - StreamInputToken::Other(url_host_as_bytes(word.as_ref())).into() - } - TokenType::Alphanumeric(word) | TokenType::UrlNoHost(word) => { - StreamInputToken::Other(word.as_ref().to_lowercase().into_bytes()).into() - } - TokenType::Email(word) => { - StreamInputToken::Other(word.as_ref().to_lowercase().into_bytes()).into() - } - TokenType::Other(ch) => { - let ch = ch.to_string(); - if symbols(&ch) { - Some(StreamInputToken::Other(ch.into_bytes())) - } else { - None - } - } - TokenType::Integer(word) => number_to_tag(false, word.as_ref()).into(), - TokenType::Float(word) => number_to_tag(true, word.as_ref()).into(), - TokenType::Punctuation(_) | TokenType::Space => None, - } - } - } - - fn url_host_as_bytes(host: &str) -> Vec { - host.split_once('/') - .map_or(host, |(h, _)| h.rsplit_once(':').map_or(h, |(h, _)| h)) - .to_lowercase() - .into_bytes() - } - - fn number_to_tag(is_float: bool, num: &str) -> StreamInputToken> { - let t = match (is_float, num.starts_with('-')) { - (true, true) => b'F', - (true, false) => b'f', - (false, true) => b'I', - (false, false) => b'i', - }; - - StreamInputToken::Other([t, num.len() as u8].to_vec()) - } #[test] fn stream_tokenizer() { @@ -8040,15 +7915,17 @@ pub mod tests { ]; for (input, expect) in inputs.iter() { - let input = StreamTokenizer::new( - input, - TypesTokenizer::new(input).filter_map(|t| t.word.to_stream_token()), - ) - .map(|word| String::from_utf8(word).unwrap()) - .collect::>(); - let expect = expect.iter().copied().map(Cow::from).collect::>(); + let tokenizer = WordStemTokenizer::new(input); + let mut result = Vec::new(); + for token in TypesTokenizer::new(input) { + if let TokenType::Alphabetic(word) = token.word { + tokenizer.tokenize(word, |t| { + result.push(t.into_owned()); + }); + } + } - assert_eq!(input, expect,); + assert_eq!(&result, expect,); } } } diff --git a/crates/services/src/task_manager/bayes.rs b/crates/services/src/task_manager/bayes.rs deleted file mode 100644 index 6365bed2..00000000 --- a/crates/services/src/task_manager/bayes.rs +++ /dev/null @@ -1,105 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use common::Server; -use email::message::metadata::MessageMetadata; -use mail_parser::MessageParser; -use spam_filter::{SpamFilterInput, analysis::init::SpamFilterInit}; -use std::time::Instant; -use trc::{SpamEvent, TaskQueueEvent}; -use types::{collection::Collection, field::EmailField}; - -pub trait SpamTrainTask: Sync + Send { - fn bayes_train( - &self, - account_id: u32, - document_id: u32, - learn_spam: bool, - ) -> impl Future + Send; -} - -impl SpamTrainTask for Server { - async fn bayes_train(&self, account_id: u32, document_id: u32, learn_spam: bool) -> bool { - let op_start = Instant::now(); - // Obtain metadata - let metadata_ = match self - .archive_by_property( - account_id, - Collection::Email, - document_id, - EmailField::Metadata.into(), - ) - .await - { - Ok(Some(metadata)) => metadata, - Ok(None) => { - trc::event!( - TaskQueue(TaskQueueEvent::MetadataNotFound), - AccountId = account_id, - Collection = Collection::Email, - DocumentId = document_id, - ); - return false; - } - Err(err) => { - trc::error!(err.caused_by(trc::location!())); - return false; - } - }; - - let metadata = match metadata_.unarchive::() { - Ok(metadata) => metadata, - Err(err) => { - trc::error!(err.caused_by(trc::location!())); - return false; - } - }; - - // Obtain raw message - match self - .blob_store() - .get_blob(metadata.blob_hash.0.as_slice(), 0..usize::MAX) - .await - { - Ok(Some(raw_message)) => { - // Train bayes classifier for account - let todo = "fix"; - /*self.bayes_train_if_balanced( - &self.spam_filter_init(SpamFilterInput::from_account_message( - &MessageParser::new().parse(&raw_message).unwrap_or_default(), - account_id, - 0, - )), - learn_spam, - ) - .await;*/ - - trc::event!( - Spam(SpamEvent::TrainAccount), - AccountId = account_id, - Collection = Collection::Email, - DocumentId = document_id, - Details = if learn_spam { "spam" } else { "ham" }, - Elapsed = op_start.elapsed(), - ); - true - } - Ok(None) => { - trc::event!( - TaskQueue(TaskQueueEvent::BlobNotFound), - AccountId = account_id, - DocumentId = document_id, - BlobId = metadata.blob_hash.0.as_slice(), - ); - false - } - Err(err) => { - trc::error!(err.caused_by(trc::location!())); - false - } - } - } -} diff --git a/crates/services/src/task_manager/index.rs b/crates/services/src/task_manager/index.rs index 9bd2aa79..6a2f1f10 100644 --- a/crates/services/src/task_manager/index.rs +++ b/crates/services/src/task_manager/index.rs @@ -593,7 +593,7 @@ async fn delete_email_metadata( hash: blob_hash.clone(), to: BlobLink::Temporary { until }, }, - vec![], + vec![BlobLink::UNDELETE_LINK], ) .set( BlobOp::Undelete { diff --git a/crates/services/src/task_manager/lock.rs b/crates/services/src/task_manager/lock.rs index b7a941e6..d1210388 100644 --- a/crates/services/src/task_manager/lock.rs +++ b/crates/services/src/task_manager/lock.rs @@ -71,7 +71,6 @@ impl TaskLockManager for Server { pub(crate) trait TaskLock { fn account_id(&self) -> u32; fn document_id(&self) -> u32; - fn remove_lock(&self) -> bool; fn lock_key(&self) -> Vec; fn lock_expiry(&self) -> u64; fn value_classes(&self) -> impl Iterator; @@ -86,10 +85,6 @@ impl TaskLock for Task { self.document_id } - fn remove_lock(&self) -> bool { - true - } - fn lock_key(&self) -> Vec { KeySerializer::new((U32_LEN * 2) + U64_LEN + 2) .write(0u8) @@ -113,42 +108,6 @@ impl TaskLock for Task { } } -impl TaskLock for Task { - fn account_id(&self) -> u32 { - self.account_id - } - - fn document_id(&self) -> u32 { - self.document_id - } - - fn remove_lock(&self) -> bool { - false - } - - fn lock_key(&self) -> Vec { - KeySerializer::new((U32_LEN * 2) + 1) - .write(1u8) - .write_leb128(self.account_id) - .write_leb128(self.document_id) - .finalize() - } - - fn lock_expiry(&self) -> u64 { - BAYES_LOCK_EXPIRY - } - - fn value_classes(&self) -> impl Iterator { - let todo = "fix"; - - std::iter::once(ValueClass::TaskQueue(TaskQueueClass::SpamTrain { - due: self.due, - blob_hash: Default::default(), - learn_spam: self.action, - })) - } -} - impl TaskLock for Task { fn account_id(&self) -> u32 { self.account_id @@ -158,10 +117,6 @@ impl TaskLock for Task { self.document_id } - fn remove_lock(&self) -> bool { - true - } - fn lock_key(&self) -> Vec { KeySerializer::new((U32_LEN * 2) + U64_LEN + 1) .write(2u8) @@ -194,10 +149,6 @@ impl TaskLock for Task { self.document_id } - fn remove_lock(&self) -> bool { - true - } - fn lock_key(&self) -> Vec { KeySerializer::new((U32_LEN * 2) + U64_LEN + 1) .write(3u8) @@ -235,10 +186,6 @@ impl TaskLock for Task>> { self.document_id } - fn remove_lock(&self) -> bool { - true - } - fn lock_key(&self) -> Vec { KeySerializer::new((U32_LEN * 2) + U64_LEN + 1) .write(4u8) @@ -263,7 +210,6 @@ impl Task { pub(crate) fn lock_expiry(&self) -> u64 { match &self.action { TaskAction::UpdateIndex(_) => INDEX_EXPIRY, - TaskAction::SpamTrain(_) => BAYES_LOCK_EXPIRY, TaskAction::SendAlarm(_) => ALARM_EXPIRY, _ => ALARM_EXPIRY, } @@ -285,7 +231,6 @@ impl Task { .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?, is_insert: *v == 7, }), - Some(v @ (1 | 2)) => TaskAction::SpamTrain(*v == 1), Some(3) => TaskAction::SendAlarm(CalendarAlarm { event_id: key.deserialize_be_u16(U64_LEN + U32_LEN + U32_LEN + 1)?, alarm_id: key.deserialize_be_u16(U64_LEN + U32_LEN + U32_LEN + U16_LEN + 1)?, diff --git a/crates/services/src/task_manager/mod.rs b/crates/services/src/task_manager/mod.rs index 48a6e757..ca2eea26 100644 --- a/crates/services/src/task_manager/mod.rs +++ b/crates/services/src/task_manager/mod.rs @@ -4,7 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::task_manager::bayes::SpamTrainTask; use crate::task_manager::imip::SendImipTask; use crate::task_manager::index::SearchIndexTask; use crate::task_manager::lock::{TaskLock, TaskLockManager}; @@ -39,7 +38,6 @@ use trc::TaskQueueEvent; use utils::snowflake::SnowflakeIdGenerator; pub mod alarm; -pub mod bayes; pub mod imip; pub mod index; pub mod lock; @@ -56,7 +54,6 @@ pub struct Task { #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub enum TaskAction { UpdateIndex(IndexAction), - SpamTrain(bool), SendAlarm(CalendarAlarm), SendImip, MergeThreads(MergeThreadIds>), @@ -72,13 +69,11 @@ pub struct IndexAction { pub(crate) struct ImipAction; const INDEX_EXPIRY: u64 = 60 * 5; // 5 minutes -const BAYES_LOCK_EXPIRY: u64 = 60 * 30; // 30 minutes const ALARM_EXPIRY: u64 = 60 * 2; // 2 minutes const QUEUE_REFRESH_INTERVAL: u64 = 60 * 5; // 5 minutes pub(crate) struct TaskManagerIpc { tx_fts: mpsc::Sender>, - tx_bayes: mpsc::Sender>, tx_alarm: mpsc::Sender>, tx_imip: mpsc::Sender>, tx_threads: mpsc::Sender>>>, @@ -94,10 +89,9 @@ struct Locked { pub fn spawn_task_manager(inner: Arc) { // Create mpsc channels for the different task types let (tx_index_1, mut rx_index_1) = mpsc::channel::>(IPC_CHANNEL_BUFFER); - let (tx_index_2, mut rx_index_2) = mpsc::channel::>(IPC_CHANNEL_BUFFER); - let (tx_index_3, mut rx_index_3) = mpsc::channel::>(IPC_CHANNEL_BUFFER); - let (tx_index_4, mut rx_index_4) = mpsc::channel::>(IPC_CHANNEL_BUFFER); - let (tx_index_5, mut rx_index_5) = + let (tx_index_2, mut rx_index_2) = mpsc::channel::>(IPC_CHANNEL_BUFFER); + let (tx_index_3, mut rx_index_3) = mpsc::channel::>(IPC_CHANNEL_BUFFER); + let (tx_index_4, mut rx_index_4) = mpsc::channel::>>>(IPC_CHANNEL_BUFFER); // Create dummy server instance for alarms @@ -177,49 +171,12 @@ pub fn spawn_task_manager(inner: Arc) { }); } - // Bayes training worker - { - let inner = inner.clone(); - tokio::spawn(async move { - while let Some(task) = rx_index_2.recv().await { - let server = inner.build_server(); - - // Lock task - if server - .try_lock_task( - task.account_id, - task.document_id, - task.lock_key(), - task.lock_expiry(), - ) - .await - { - let success = server - .bayes_train(task.account_id, task.document_id, task.action) - .await; - - // Remove entry from queue - if success { - delete_tasks(&server, &[task]).await; - } else { - trc::event!( - TaskQueue(TaskQueueEvent::TaskFailed), - AccountId = task.account_id, - DocumentId = task.document_id, - Details = "Bayes training task failed", - ); - } - } - } - }); - } - // Send alarm worker { let inner = inner.clone(); let server_instance = server_instance.clone(); tokio::spawn(async move { - while let Some(task) = rx_index_3.recv().await { + while let Some(task) = rx_index_2.recv().await { let server = inner.build_server(); // Lock task @@ -263,7 +220,7 @@ pub fn spawn_task_manager(inner: Arc) { let inner = inner.clone(); let server_instance = server_instance.clone(); tokio::spawn(async move { - while let Some(task) = rx_index_4.recv().await { + while let Some(task) = rx_index_3.recv().await { let server = inner.build_server(); // Lock task @@ -306,7 +263,7 @@ pub fn spawn_task_manager(inner: Arc) { { let inner = inner.clone(); tokio::spawn(async move { - while let Some(task) = rx_index_5.recv().await { + while let Some(task) = rx_index_4.recv().await { let server = inner.build_server(); // Lock task @@ -340,10 +297,9 @@ pub fn spawn_task_manager(inner: Arc) { tokio::spawn(async move { let mut ipc = TaskManagerIpc { tx_fts: tx_index_1, - tx_bayes: tx_index_2, - tx_alarm: tx_index_3, - tx_imip: tx_index_4, - tx_threads: tx_index_5, + tx_alarm: tx_index_2, + tx_imip: tx_index_3, + tx_threads: tx_index_4, locked: Default::default(), revision: 0, }; @@ -475,27 +431,6 @@ impl TaskQueueManager for Server { ); } } - TaskAction::SpamTrain(learn_spam) - if roles.bayes_training.is_enabled_for_hash(&event) => - { - if ipc - .tx_bayes - .send(Task { - account_id: event.account_id, - document_id: event.document_id, - due: event.due, - action: learn_spam, - }) - .await - .is_err() - { - trc::event!( - Server(trc::ServerEvent::ThreadError), - Details = "Error sending task.", - CausedBy = trc::location!() - ); - } - } TaskAction::SendAlarm(alarm) if roles.calendar_alerts.is_enabled_for_hash(&event) => { @@ -598,9 +533,7 @@ async fn delete_tasks(server: &Server, tasks: &[T]) { } for task in tasks { - if task.remove_lock() { - server.remove_index_lock(task.lock_key()).await; - } + server.remove_index_lock(task.lock_key()).await; } } @@ -608,7 +541,6 @@ impl TaskAction { pub fn name(&self) -> &'static str { match self { TaskAction::UpdateIndex(_) => "UpdateIndex", - TaskAction::SpamTrain(_) => "SpamTrain", TaskAction::SendAlarm(_) => "SendAlarm", TaskAction::SendImip => "SendImip", TaskAction::MergeThreads(_) => "MergeThreads", diff --git a/crates/spam-filter/Cargo.toml b/crates/spam-filter/Cargo.toml index 0befd4e6..5a9559c8 100644 --- a/crates/spam-filter/Cargo.toml +++ b/crates/spam-filter/Cargo.toml @@ -5,6 +5,7 @@ edition = "2024" [dependencies] utils = { path = "../utils" } +types = { path = "../types" } nlp = { path = "../nlp" } store = { path = "../store" } trc = { path = "../trc" } diff --git a/crates/spam-filter/src/analysis/classifier.rs b/crates/spam-filter/src/analysis/classifier.rs index b279630e..d6b90b17 100644 --- a/crates/spam-filter/src/analysis/classifier.rs +++ b/crates/spam-filter/src/analysis/classifier.rs @@ -22,22 +22,11 @@ pub trait SpamFilterAnalyzeClassify: Sync + Send { impl SpamFilterAnalyzeClassify for Server { async fn spam_filter_analyze_classify(&self, ctx: &mut SpamFilterContext<'_>) { - if self.core.spam.classifier.is_some() && !ctx.result.has_tag("SPAM_TRAP") - //&& !ctx.result.has_tag("TRUSTED_REPLY") + if self.core.spam.classifier.is_some() + && !ctx.result.has_tag("SPAM_TRAP") + && let Err(err) = self.spam_classify(ctx).await { - match self.spam_classify(ctx).await { - Ok(Some(score)) => { - /*if score > config.score_spam { - ctx.result.add_tag("BAYES_SPAM"); - } else if score < config.score_ham { - ctx.result.add_tag("BAYES_HAM"); - }*/ - } - Ok(None) => (), - Err(err) => { - trc::error!(err.span_id(ctx.input.span_id).caused_by(trc::location!())); - } - } + trc::error!(err.span_id(ctx.input.span_id).caused_by(trc::location!())); } } diff --git a/crates/spam-filter/src/analysis/from.rs b/crates/spam-filter/src/analysis/from.rs index 12c3c9ad..d724e4b0 100644 --- a/crates/spam-filter/src/analysis/from.rs +++ b/crates/spam-filter/src/analysis/from.rs @@ -4,14 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::future::Future; - +use crate::{Email, SpamFilterContext}; use common::Server; use mail_parser::HeaderName; use nlp::tokenizers::types::{TokenType, TypesTokenizer}; use smtp_proto::{MAIL_BODY_8BITMIME, MAIL_BODY_BINARYMIME, MAIL_SMTPUTF8}; - -use crate::{Email, SpamFilterContext}; +use std::future::Future; pub trait SpamFilterAnalyzeFrom: Sync + Send { fn spam_filter_analyze_from( diff --git a/crates/spam-filter/src/analysis/headers.rs b/crates/spam-filter/src/analysis/headers.rs index a7084864..a0a4ef04 100644 --- a/crates/spam-filter/src/analysis/headers.rs +++ b/crates/spam-filter/src/analysis/headers.rs @@ -59,8 +59,21 @@ impl SpamFilterAnalyzeHeaders for Server { ctx.result.add_tag("MULTIPLE_UNIQUE_HEADERS"); } - if !matches!(raw_message.get(header.offset_start as usize), Some(b' ')) { - ctx.result.add_tag("HEADER_EMPTY_DELIMITER"); + let mut value = raw_message + .get(header.offset_start as usize..) + .unwrap_or_default() + .iter(); + loop { + match value.next() { + Some(b' ' | b'\t') => { + break; + } + Some(b'\r' | b'\n') => {} + _ => { + ctx.result.add_tag("HEADER_EMPTY_DELIMITER"); + break; + } + } } } HeaderName::ListArchive diff --git a/crates/spam-filter/src/analysis/init.rs b/crates/spam-filter/src/analysis/init.rs index ed669627..0f6d792e 100644 --- a/crates/spam-filter/src/analysis/init.rs +++ b/crates/spam-filter/src/analysis/init.rs @@ -206,9 +206,7 @@ impl SpamFilterInit for Server { TokenType::UrlNoScheme(s) => TokenType::UrlNoScheme( UrlParts::new(format!("https://{}", s.trim())), ), - TokenType::IpAddr(i) => { - TokenType::IpAddr(IpParts::new(i.to_string())) - } + TokenType::IpAddr(i) => TokenType::IpAddr(IpParts::new(i)), TokenType::Email(e) => TokenType::Email(Email::new(e)), TokenType::Float(s) => TokenType::Float(s.to_string().into()), }) diff --git a/crates/spam-filter/src/analysis/ip.rs b/crates/spam-filter/src/analysis/ip.rs index 2f0408ce..17b60416 100644 --- a/crates/spam-filter/src/analysis/ip.rs +++ b/crates/spam-filter/src/analysis/ip.rs @@ -4,8 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{borrow::Cow, future::Future}; - +use super::ElementLocation; +use crate::{IpParts, SpamFilterContext, TextPart, modules::dnsbl::check_dnsbl}; use common::{ Server, config::spamfilter::{Element, IpResolver, Location}, @@ -13,12 +13,9 @@ use common::{ use mail_auth::IprevResult; use mail_parser::{HeaderName, HeaderValue, Host}; use nlp::tokenizers::types::TokenType; +use std::future::Future; use store::ahash::AHashSet; -use crate::{IpParts, SpamFilterContext, TextPart, modules::dnsbl::check_dnsbl}; - -use super::ElementLocation; - pub trait SpamFilterAnalyzeIp: Sync + Send { fn spam_filter_analyze_ip( &self, @@ -122,12 +119,10 @@ impl SpamFilterAnalyzeIp for Server { } } -impl<'x> IpParts<'x> { - pub fn new(text: impl Into>) -> IpParts<'x> { - let text = text.into(); +impl IpParts { + pub fn new(text: &str) -> IpParts { IpParts { ip: text.parse().ok(), - text, } } } diff --git a/crates/spam-filter/src/analysis/score.rs b/crates/spam-filter/src/analysis/score.rs index c0d75c68..f3883b96 100644 --- a/crates/spam-filter/src/analysis/score.rs +++ b/crates/spam-filter/src/analysis/score.rs @@ -79,7 +79,7 @@ impl SpamFilterAnalyzeScore for Server { let mut user_results = vec![false; ctx.result.classifier_confidence.len()]; if !ctx.result.classifier_confidence.is_empty() { for (idx, &confidence) in ctx.result.classifier_confidence.iter().enumerate() { - if confidence != 0.0 { + if let Some(confidence) = confidence { avg_confidence += confidence; let user_score = self .core diff --git a/crates/spam-filter/src/lib.rs b/crates/spam-filter/src/lib.rs index 8e75d609..5531b7ec 100644 --- a/crates/spam-filter/src/lib.rs +++ b/crates/spam-filter/src/lib.rs @@ -9,7 +9,6 @@ pub mod modules; use analysis::ElementLocation; use analysis::url::UrlParts; - use mail_auth::{ArcOutput, DkimOutput, DmarcResult, IprevOutput, SpfOutput, dmarc::Policy}; use mail_parser::Message; use modules::html::HtmlToken; @@ -68,7 +67,7 @@ pub struct SpamFilterOutput<'x> { pub subject_lc: String, pub subject_thread: String, pub subject_thread_lc: String, - pub subject_tokens: Vec, Email, UrlParts<'x>, IpParts<'x>>>, + pub subject_tokens: Vec, Email, UrlParts<'x>, IpParts>>, pub ips: AHashSet>, pub urls: HashSet>>, @@ -79,20 +78,19 @@ pub struct SpamFilterOutput<'x> { } #[derive(Debug)] -pub struct IpParts<'x> { +pub struct IpParts { ip: Option, - text: Cow<'x, str>, } pub enum TextPart<'x> { Plain { text_body: &'x str, - tokens: Vec, Email, UrlParts<'x>, IpParts<'x>>>, + tokens: Vec, Email, UrlParts<'x>, IpParts>>, }, Html { html_tokens: Vec, text_body: String, - tokens: Vec, Email, UrlParts<'x>, IpParts<'x>>>, + tokens: Vec, Email, UrlParts<'x>, IpParts>>, }, None, } @@ -100,7 +98,7 @@ pub enum TextPart<'x> { #[derive(Debug, Default)] pub struct SpamFilterResult { pub tags: AHashSet, - pub classifier_confidence: Vec, + pub classifier_confidence: Vec>, pub score: f32, pub rbl_ip_checks: usize, pub rbl_domain_checks: usize, @@ -191,6 +189,16 @@ impl Hash for Email { } impl Email { + pub fn classifier_parts(&self) -> Option<(&str, &str)> { + // Returns (local@, @domain) + if self.is_valid() { + let at_pos = self.address.find('@')?; + Some((&self.address[..=at_pos], &self.address[at_pos..])) + } else { + None + } + } + pub fn is_valid(&self) -> bool { self.domain_part.sld.is_some() && !self.local_part.is_empty() } diff --git a/crates/spam-filter/src/modules/bayes.rs b/crates/spam-filter/src/modules/bayes.rs deleted file mode 100644 index 3819d159..00000000 --- a/crates/spam-filter/src/modules/bayes.rs +++ /dev/null @@ -1,528 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use std::{borrow::Cow, collections::HashSet, future::Future, time::Duration}; - -use common::{Server, ip_to_bytes}; -use mail_auth::DmarcResult; -use nlp::{ - bayes::{ - BayesModel, TokenHash, Weights, - tokenize::{BayesInputToken, BayesTokenizer, symbols}, - }, - tokenizers::{ - osb::{Gram, OsbToken, OsbTokenizer}, - types::TokenType, - }, -}; -use store::dispatch::lookup::KeyValue; -use trc::AddContext; -use utils::cache::TtlEntry; - -use crate::{Email, IpParts, SpamFilterContext, TextPart, analysis::url::UrlParts}; - -pub trait BayesClassifier { - fn bayes_train( - &self, - ctx: &SpamFilterContext<'_>, - is_spam: bool, - is_train: bool, - ) -> impl Future> + Send; - - fn bayes_classify( - &self, - ctx: &SpamFilterContext<'_>, - ) -> impl Future>> + Send; - - fn bayes_is_balanced( - &self, - ctx: &SpamFilterContext<'_>, - learn_spam: bool, - ) -> impl Future> + Send; - - fn bayes_train_if_balanced( - &self, - ctx: &SpamFilterContext<'_>, - learn_spam: bool, - ) -> impl Future + Send; - - fn bayes_weights_for_token( - &self, - account_id: Option, - token: TokenHash, - ) -> impl Future> + Send; -} - -impl BayesClassifier for Server { - async fn bayes_train( - &self, - ctx: &SpamFilterContext<'_>, - is_spam: bool, - is_train: bool, - ) -> trc::Result<()> { - // Train the model - let mut model = BayesModel::default(); - - // Train metadata tokens - for token in ctx.spam_tokens() { - model.train_token(TokenHash::from(Gram::Uni { t1: &token }), is_spam); - } - - // Train the subject - model.train( - OsbTokenizer::new( - BayesTokenizer::new( - &ctx.output.subject_thread, - ctx.output.subject_tokens.iter().filter_map(to_bayes_token), - ), - 5, - ), - is_spam, - ); - - // Train the body - match ctx - .input - .message - .html_body - .first() - .or_else(|| ctx.input.message.text_body.first()) - .and_then(|idx| ctx.output.text_parts.get(*idx as usize)) - { - Some(TextPart::Html { - text_body, tokens, .. - }) => { - model.train( - OsbTokenizer::new( - BayesTokenizer::new(text_body, tokens.iter().filter_map(to_bayes_token)), - 5, - ), - is_spam, - ); - } - Some(TextPart::Plain { text_body, tokens }) => { - model.train( - OsbTokenizer::new( - BayesTokenizer::new(text_body, tokens.iter().filter_map(to_bayes_token)), - 5, - ), - is_spam, - ); - } - _ => {} - } - - if model.weights.is_empty() { - trc::bail!( - trc::SpamEvent::TrainError - .into_err() - .reason("No weights found") - ); - } - - trc::event!( - Spam(trc::SpamEvent::Train), - SpanId = ctx.input.span_id, - Details = is_spam, - Total = model.weights.len(), - ); - - // Update weight and invalidate cache - if is_train { - let (is_global, prefix) = if ctx.input.account_id.is_none() { - (true, KV_BAYES_MODEL_GLOBAL) - } else { - (false, KV_BAYES_MODEL_USER) - }; - for (hash, weights) in model.weights { - self.in_memory_store() - .counter_incr( - KeyValue::new( - hash.serialize(prefix, ctx.input.account_id), - i64::from(weights), - ), - false, - ) - .await - .caused_by(trc::location!())?; - if is_global { - self.inner.cache.bayes.remove(&hash); - } - } - if is_global { - self.inner.cache.bayes.remove(&TokenHash::default()); - } - - // Update training counts - let weights = if is_spam { - Weights { spam: 1, ham: 0 } - } else { - Weights { spam: 0, ham: 1 } - }; - self.in_memory_store() - .counter_incr( - KeyValue::new( - TokenHash::default().serialize(prefix, ctx.input.account_id), - i64::from(weights), - ), - false, - ) - .await - .caused_by(trc::location!()) - .map(|_| ()) - } else { - //TODO: Implement untrain - Ok(()) - } - } - - async fn bayes_classify(&self, ctx: &SpamFilterContext<'_>) -> trc::Result> { - let classifier = if let Some(config) = &self.core.spam.bayes { - &config.classifier - } else { - return Ok(None); - }; - - // Obtain training counts - let (spam_learns, ham_learns) = self - .bayes_weights_for_token(ctx.input.account_id, TokenHash::default()) - .await - .map(|w| (w.spam, w.ham))?; - - // Make sure we have enough training data - if spam_learns < classifier.min_learns || ham_learns < classifier.min_learns { - trc::event!( - Spam(trc::SpamEvent::ClassifyError), - SpanId = ctx.input.span_id, - AccountId = ctx.input.account_id, - Reason = "Not enough training data", - Details = vec![ - trc::Value::from(spam_learns), - trc::Value::from(ham_learns), - trc::Value::from(classifier.min_learns) - ], - ); - return Ok(None); - } - - // Classify the text - let mut osb_tokens = Vec::new(); - - // Classify metadata tokens - for token in ctx.spam_tokens() { - let weights = self - .bayes_weights_for_token( - ctx.input.account_id, - TokenHash::from(Gram::Uni { t1: &token }), - ) - .await?; - osb_tokens.push(OsbToken { - inner: weights, - idx: 1, - }); - } - - // Classify the subject - for token in OsbTokenizer::<_, TokenHash>::new( - BayesTokenizer::new( - &ctx.output.subject_thread, - ctx.output.subject_tokens.iter().filter_map(to_bayes_token), - ), - 5, - ) { - let weights = self - .bayes_weights_for_token(ctx.input.account_id, token.inner) - .await?; - osb_tokens.push(OsbToken { - inner: weights, - idx: token.idx, - }); - } - - // Classify the body - match ctx - .input - .message - .html_body - .first() - .or_else(|| ctx.input.message.text_body.first()) - .and_then(|idx| ctx.output.text_parts.get(*idx as usize)) - { - Some(TextPart::Html { - text_body, tokens, .. - }) => { - for token in OsbTokenizer::<_, TokenHash>::new( - BayesTokenizer::new(text_body, tokens.iter().filter_map(to_bayes_token)), - 5, - ) { - let weights = self - .bayes_weights_for_token(ctx.input.account_id, token.inner) - .await?; - osb_tokens.push(OsbToken { - inner: weights, - idx: token.idx, - }); - } - } - Some(TextPart::Plain { text_body, tokens }) => { - for token in OsbTokenizer::<_, TokenHash>::new( - BayesTokenizer::new(text_body, tokens.iter().filter_map(to_bayes_token)), - 5, - ) { - let weights = self - .bayes_weights_for_token(ctx.input.account_id, token.inner) - .await?; - osb_tokens.push(OsbToken { - inner: weights, - idx: token.idx, - }); - } - } - _ => {} - } - - let result = classifier.classify(osb_tokens.into_iter(), ham_learns, spam_learns); - - trc::event!( - Spam(trc::SpamEvent::Classify), - SpanId = ctx.input.span_id, - AccountId = ctx.input.account_id, - Details = vec![ - trc::Value::from(spam_learns), - trc::Value::from(ham_learns), - trc::Value::from(classifier.min_learns) - ], - Result = result.map(trc::Value::from).unwrap_or_default() - ); - - Ok(result) - } - - async fn bayes_is_balanced( - &self, - ctx: &SpamFilterContext<'_>, - learn_spam: bool, - ) -> trc::Result { - let min_balance = self - .core - .spam - .bayes - .as_ref() - .map_or(0.0, |c| c.classifier.min_balance); - - if min_balance == 0.0 { - return Ok(true); - } - - // Obtain training counts - let (spam_learns, ham_learns) = self - .bayes_weights_for_token(ctx.input.account_id, TokenHash::default()) - .await - .map(|w| (w.spam as f64, w.ham as f64))?; - - let result = if spam_learns > 0.0 || ham_learns > 0.0 { - if learn_spam { - (spam_learns / (ham_learns + 1.0)) <= 1.0 / min_balance - } else { - (ham_learns / (spam_learns + 1.0)) <= 1.0 / min_balance - } - } else { - true - }; - - trc::event!( - Spam(trc::SpamEvent::TrainBalance), - SpanId = ctx.input.span_id, - Details = vec![ - trc::Value::from(learn_spam), - trc::Value::from(min_balance), - trc::Value::from(spam_learns), - trc::Value::from(ham_learns), - ], - Result = result - ); - - Ok(result) - } - - async fn bayes_train_if_balanced(&self, ctx: &SpamFilterContext<'_>, learn_spam: bool) { - let err = match self.bayes_is_balanced(ctx, learn_spam).await { - Ok(true) => match self.bayes_train(ctx, learn_spam, true).await { - Ok(_) => { - return; - } - Err(err) => err, - }, - Ok(false) => { - return; - } - Err(err) => err, - }; - - if let Some(account_id) = ctx.input.account_id { - trc::error!( - err.span_id(ctx.input.span_id) - .account_id(account_id) - .caused_by(trc::location!()) - ); - } else { - trc::error!(err.span_id(ctx.input.span_id).caused_by(trc::location!())); - } - } - - async fn bayes_weights_for_token( - &self, - account_id: Option, - token: TokenHash, - ) -> trc::Result { - match account_id { - None => { - match self - .inner - .cache - .bayes - .get_value_or_guard_async(&token) - .await - { - Ok(weights) => Ok(weights), - Err(guard) => { - let weights = self - .in_memory_store() - .counter_get(token.serialize_global(KV_BAYES_MODEL_GLOBAL)) - .await?; - let expiry = if weights != 0 { - Duration::from_secs(3 * 60 * 60) - } else { - Duration::from_secs(60 * 60) - }; - let weights = Weights::from(weights); - let _ = guard.insert(TtlEntry::new(weights, expiry)); - Ok(weights) - } - } - } - Some(account_id) => self - .in_memory_store() - .counter_get(token.serialize_account(KV_BAYES_MODEL_USER, account_id)) - .await - .map(Weights::from), - } - } -} - -const P_FROM_NAME: u8 = 0; -const P_FROM_EMAIL: u8 = 1; -const P_FROM_DOMAIN: u8 = 2; -const P_ASN: u8 = 3; -const P_REMOTE_IP: u8 = 4; -const P_INTEGER_POS: u8 = 5; -const P_INTEGER_NEG: u8 = 6; -const P_FLOAT_POS: u8 = 7; -const P_FLOAT_NEG: u8 = 8; -const P_BODY_URL: u8 = 9; -const P_BODY_IP: u8 = 10; -const P_BODY_EMAIL: u8 = 11; - -impl SpamFilterContext<'_> { - pub fn spam_tokens(&self) -> HashSet> { - let mut tokens = HashSet::new(); - if matches!(self.input.dmarc_result, Some(DmarcResult::Pass)) - || self.input.account_id.is_some() - { - for addr in [&self.output.env_from_addr, &self.output.from.email] { - if !addr.address.is_empty() { - tokens.insert(add_prefix(P_FROM_EMAIL, addr.address.as_bytes())); - tokens.insert(add_prefix( - P_FROM_DOMAIN, - addr.domain_part.sld_or_default().as_bytes(), - )); - } - } - if let Some(name) = &self.output.from.name { - for name_part in name.to_lowercase().split_whitespace() { - tokens.insert(add_prefix(P_FROM_NAME, name_part.as_bytes())); - } - } - } - if let Some(asn) = self.input.asn { - tokens.insert(add_prefix(P_ASN, &asn.to_be_bytes())); - } - if !self.input.remote_ip.is_loopback() { - tokens.insert(add_prefix(P_REMOTE_IP, &ip_to_bytes(&self.input.remote_ip))); - } - tokens - } -} - -fn add_prefix(prefix: u8, key: &[u8]) -> Vec { - let mut buf = Vec::with_capacity(key.len() + 1); - buf.extend_from_slice(key); - buf.push(prefix); - buf -} - -fn to_bayes_token( - token: &TokenType, Email, UrlParts<'_>, IpParts<'_>>, -) -> Option { - match token { - TokenType::Alphabetic(word) => Some(BayesInputToken::Word(word.as_ref().to_lowercase())), - TokenType::Url(url) | TokenType::UrlNoScheme(url) => url.url_parsed.as_ref().map(|url| { - BayesInputToken::Raw(add_prefix(P_BODY_URL, url.host.sld_or_default().as_bytes())) - }), - TokenType::IpAddr(ip) => ip - .ip - .as_ref() - .map(|ip| BayesInputToken::Raw(add_prefix(P_BODY_IP, &ip_to_bytes(ip)))), - TokenType::Alphanumeric(word) | TokenType::UrlNoHost(word) => { - BayesInputToken::Raw(word.as_ref().to_lowercase().into_bytes()).into() - } - TokenType::Email(email) => BayesInputToken::Raw(add_prefix( - P_BODY_EMAIL, - email.domain_part.sld_or_default().as_bytes(), - )) - .into(), - TokenType::Other(ch) => { - let ch = ch.to_string(); - if symbols(&ch) { - Some(BayesInputToken::Raw(ch.into_bytes())) - } else { - None - } - } - TokenType::Integer(word) => number_to_tag(false, word.as_ref()).into(), - TokenType::Float(word) => number_to_tag(true, word.as_ref()).into(), - TokenType::Punctuation(_) | TokenType::Space => None, - } -} - -fn number_to_tag(is_float: bool, num: &str) -> BayesInputToken { - let t = match (is_float, num.starts_with('-')) { - (true, true) => P_FLOAT_NEG, - (true, false) => P_FLOAT_POS, - (false, true) => P_INTEGER_NEG, - (false, false) => P_INTEGER_POS, - }; - - BayesInputToken::Raw([t, num.len() as u8].to_vec()) -} - -impl AsRef for Email { - fn as_ref(&self) -> &str { - &self.address - } -} - -impl AsRef for UrlParts<'_> { - fn as_ref(&self) -> &str { - &self.url - } -} - -impl AsRef for IpParts<'_> { - fn as_ref(&self) -> &str { - &self.text - } -} diff --git a/crates/spam-filter/src/modules/classifier.rs b/crates/spam-filter/src/modules/classifier.rs index c6516ff6..456ea2bb 100644 --- a/crates/spam-filter/src/modules/classifier.rs +++ b/crates/spam-filter/src/modules/classifier.rs @@ -4,28 +4,564 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::Server; - -use crate::SpamFilterContext; +use crate::{Email, IpParts, SpamFilterContext, TextPart, analysis::url::UrlParts}; +use common::{Server, config::spamfilter::Location}; +use core::hash; +use mail_auth::DmarcResult; +use mail_parser::{DateTime, MimeHeaders}; +use nlp::{ + classifier::feature::Feature, + tokenizers::{ + stream::{WordStemTokenizer, symbols}, + types::TokenType, + }, +}; +use std::{ + borrow::Cow, + collections::{HashMap, hash_map::Entry}, + hash::{Hash, RandomState}, +}; +use store::write::{BatchBuilder, BlobLink, BlobOp, now}; +use trc::AddContext; +use types::blob_hash::BlobHash; pub trait SpamClassifier { fn spam_classify( &self, - ctx: &SpamFilterContext<'_>, - ) -> impl Future>> + Send; - - fn spam_train( - &self, - ctx: &SpamFilterContext<'_>, + ctx: &mut SpamFilterContext<'_>, ) -> impl Future> + Send; + + fn spam_train(&self) -> impl Future> + Send; } impl SpamClassifier for Server { - async fn spam_train(&self, ctx: &SpamFilterContext<'_>) -> trc::Result<()> { + async fn spam_train(&self) -> trc::Result<()> { + let todo = "parse ASN and other stuff, build context properly"; + todo!() } - async fn spam_classify(&self, ctx: &SpamFilterContext<'_>) -> trc::Result> { - todo!() + async fn spam_classify(&self, ctx: &mut SpamFilterContext<'_>) -> trc::Result<()> { + let classifier = self.inner.data.spam_classifier.load_full(); + + if classifier.is_active() { + let mut classifier_confidence = Vec::with_capacity(ctx.input.env_rcpt_to.len()); + let mut tokens = ctx.classifier_tokens().0; + let feature_builder = classifier.feature_builder(); + feature_builder.scale(&mut tokens); + + for rcpt in &ctx.input.env_rcpt_to { + let prediction = if let Some(account_id) = self + .directory() + .email_to_id(rcpt) + .await + .caused_by(trc::location!())? + { + classifier + .predict(&feature_builder.build(&tokens, account_id.into())) + .into() + } else { + None + }; + classifier_confidence.push(prediction); + } + + ctx.result.classifier_confidence = classifier_confidence; + } + + Ok(()) + } +} + +const MAX_TOKEN_LENGTH: usize = 16; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum Token<'x> { + // User types + Word { value: Cow<'x, str> }, + Number { code: [u8; 3] }, + Alphanumeric { code: [u8; 4] }, + Symbol { value: String }, + + // User and global types + Sender { value: Cow<'x, str> }, + Asn { number: [u8; 4] }, + Url { value: Cow<'x, str> }, + Email { value: Cow<'x, str> }, + Hostname { value: &'x str }, + Attachment { value: Cow<'x, str> }, +} + +#[derive(Debug)] +struct Tokens<'x>(HashMap, f32, RandomState>); + +impl<'x> SpamFilterContext<'x> { + fn classifier_tokens(&'x self) -> Tokens<'x> { + let mut tokens = Tokens::default(); + + // Add From addresses + if !matches!(self.input.dmarc_result, Some(DmarcResult::Pass)) { + tokens.insert(Token::Sender { value: "!".into() }); + } + for email in [&self.output.env_from_addr, &self.output.from.email] { + tokens.insert_email(email, true); + } + + // Add Email addresses + for email in &self.output.emails { + let is_sender = match &email.location { + Location::HeaderReplyTo | Location::HeaderDnt => true, + Location::BodyText + | Location::BodyHtml + | Location::Attachment + | Location::HeaderSubject => false, + _ => continue, + }; + + tokens.insert_email(&email.element.email, is_sender); + } + + // Add URLs + for url in &self.output.urls { + if let Some(url) = &url.element.url_parsed { + if let Some(host) = &url.host.sld { + tokens.insert(Token::Url { value: host.into() }); + if host != &url.host.fqdn { + tokens.insert(Token::Url { + value: url.host.fqdn.as_str().into(), + }); + } + } else { + tokens.insert(Token::Url { + value: url.host.fqdn.as_str().into(), + }); + } + if let Some(path) = url.parts.path_and_query() { + for token in path.as_str().split(|c: char| !c.is_alphanumeric()) { + if token.len() > 1 { + let token = truncate_word(token, MAX_TOKEN_LENGTH); + tokens.insert(Token::Url { + value: format!("_{token}").into(), + }); + } + } + } + } + } + + // Add hostnames + for domain in &self.output.domains { + if matches!( + domain.location, + Location::HeaderReceived | Location::HeaderMid | Location::Ehlo | Location::Tcp + ) { + tokens.insert(Token::Hostname { + value: &domain.element, + }); + } + } + + // Add ASN + if let Some(asn) = self.input.asn { + tokens.insert(Token::Asn { + number: asn.to_be_bytes(), + }); + } + + // Add attachment indicators + for part in &self.input.message.parts { + if let Some(name) = part.attachment_name() { + if let Some((name, ext)) = name.rsplit_once('.') { + if !ext.is_empty() { + tokens.insert(Token::Attachment { + value: lower_prefix("!", truncate_word(ext, MAX_TOKEN_LENGTH)).into(), + }); + } + for token in name.split(|c: char| !c.is_alphanumeric()) { + if token.len() > 1 { + tokens.insert(Token::Attachment { + value: lower_prefix("_", truncate_word(token, MAX_TOKEN_LENGTH)) + .into(), + }); + } + } + } + + if let Some(ct) = part.content_type() { + tokens.insert(Token::Attachment { + value: ct.c_type.as_ref().into(), + }); + if let Some(st) = &ct.c_subtype { + tokens.insert(Token::Attachment { + value: st.as_ref().into(), + }); + } + } + } + } + + // Tokenize the subject + for token in &self.output.subject_tokens { + tokens.insert_type( + &WordStemTokenizer::new(&self.output.subject_thread_lc), + token, + ); + } + + // Tokenize the text parts + let body_idx = self + .input + .message + .html_body + .first() + .or_else(|| self.input.message.text_body.first()) + .map(|idx| *idx as usize); + let mut alt_tokens = Tokens::default(); + for (idx, part) in self.output.text_parts.iter().enumerate() { + if Some(idx) == body_idx + || (!self.input.message.text_body.contains(&(idx as u32)) + && !self.input.message.html_body.contains(&(idx as u32))) + { + tokens.insert_text_part(part); + } else { + alt_tokens.insert_text_part(part); + } + } + if !alt_tokens.0.is_empty() { + for (token, count) in alt_tokens.0.into_iter() { + if let Entry::Vacant(entry) = tokens.0.entry(token) { + entry.insert(count); + } + } + } + + tokens + } +} + +impl<'x> Tokens<'x> { + fn insert_text_part(&mut self, part: &'x TextPart<'x>) { + match part { + TextPart::Plain { text_body, tokens } => { + let word_tokenizer = WordStemTokenizer::new(text_body); + + for token in tokens { + self.insert_type(&word_tokenizer, token); + } + } + TextPart::Html { + text_body, tokens, .. + } => { + let word_tokenizer = WordStemTokenizer::new(text_body); + for token in tokens { + self.insert_type(&word_tokenizer, token); + } + } + TextPart::None => (), + } + } + + fn insert_type( + &mut self, + word_tokenizer: &WordStemTokenizer, + token: &'x TokenType, Email, UrlParts<'x>, IpParts>, + ) { + match token { + TokenType::Alphabetic(word) => { + if word.chars().all(|c| c.is_lowercase() || !c.is_uppercase()) { + word_tokenizer.tokenize(word, |value| match value { + Cow::Borrowed(value) => { + self.insert(Token::Word { + value: truncate_word(value, MAX_TOKEN_LENGTH).into(), + }); + } + Cow::Owned(value) => { + if value.len() <= MAX_TOKEN_LENGTH { + self.insert(Token::Word { + value: value.into(), + }); + } else { + self.insert(Token::Word { + value: truncate_word(&value, MAX_TOKEN_LENGTH) + .to_string() + .into(), + }); + } + } + }); + } else { + let word = word.to_lowercase(); + word_tokenizer.tokenize(&word, |token| { + self.insert(Token::Word { + value: truncate_word(token.as_ref(), MAX_TOKEN_LENGTH) + .to_string() + .into(), + }); + }); + } + } + TokenType::Alphanumeric(word) => { + self.insert(Token::from_alphanumeric(word.as_ref())); + } + TokenType::UrlNoHost(url) => { + for token in url.to_lowercase().split(|c: char| !c.is_alphanumeric()) { + if token.len() > 1 { + let token = truncate_word(token, MAX_TOKEN_LENGTH); + self.insert(Token::Url { + value: format!("_{token}").into(), + }); + } + } + } + TokenType::Other(ch) => { + let value = ch.to_string(); + if symbols(&value) { + self.insert(Token::Symbol { value }); + } + } + TokenType::Integer(word) => { + self.insert(Token::from_number(false, word.as_ref())); + } + TokenType::Float(word) => { + self.insert(Token::from_number(true, word.as_ref())); + } + TokenType::Email(_) + | TokenType::Url(_) + | TokenType::UrlNoScheme(_) + | TokenType::IpAddr(_) + | TokenType::Punctuation(_) + | TokenType::Space => {} + } + } + + fn insert(&mut self, token: Token<'x>) { + *self.0.entry(token).or_insert(0.0) += 1.0; + } + + fn insert_if_missing(&mut self, token: Token<'x>) { + self.0.entry(token).or_insert(1.0); + } + + fn insert_email(&mut self, email: &'x Email, is_sender: bool) { + if is_sender { + self.insert_if_missing(Token::Sender { + value: email.address.as_str().into(), + }); + self.insert_if_missing(Token::Sender { + value: email.domain_part.fqdn.as_str().into(), + }); + if let Some(sld) = &email.domain_part.sld + && sld != &email.domain_part.fqdn + { + self.insert_if_missing(Token::Sender { value: sld.into() }); + } + } else { + self.insert_if_missing(Token::Email { + value: email.address.as_str().into(), + }); + self.insert_if_missing(Token::Email { + value: email.domain_part.fqdn.as_str().into(), + }); + if let Some(sld) = &email.domain_part.sld + && sld != &email.domain_part.fqdn + { + self.insert_if_missing(Token::Email { value: sld.into() }); + } + } + } +} + +impl Token<'static> { + fn from_alphanumeric(s: &str) -> Self { + // Character class counts + let mut upper = 0u32; + let mut lower = 0u32; + let mut digit = 0u32; + let mut len = 0; + let mut char_types = Vec::with_capacity(len); + for c in s.chars() { + let char_type = CharType::from_char(c); + char_types.push(char_type); + match char_type { + CharType::Upper => upper += 1, + CharType::Lower => lower += 1, + CharType::Digit => digit += 1, + CharType::Other => (), + } + len += 1; + } + + // Determine dominant composition + let composition = match (upper > 0, lower > 0, digit > 0) { + (true, false, false) => b'U', // UPPERCASE only + (false, true, false) => b'L', // lowercase only + (false, false, true) => b'D', // digits only + (true, true, false) => b'A', // Alphabetic mixed case + (true, false, true) => b'H', // Upper + digits (common in codes) + (false, true, true) => b'M', // lower + digits (common in identifiers) + (true, true, true) => b'X', // eXtreme mix - all three + (false, false, false) => b'E', // empty/invalid + }; + + // Length bucket (log-ish scale) + let len_code = match len { + 1 => b'1', + 2 => b'2', + 3 => b'3', + 4 => b'4', + 5..=6 => b'5', + 7..=8 => b'6', + 9..=12 => b'7', + 13..=16 => b'8', + 17..=32 => b'9', + _ => b'Z', + }; + + // Ratio encoding (which class dominates) + let max_count = upper.max(lower).max(digit); + let dominance = (max_count * 100) / len.min(1) as u32; + let ratio = match dominance { + 0..=50 => b'B', // Balanced + 51..=75 => b'P', // Partial dominance + 76..=99 => b'D', // Dominant + _ => b'O', // One class only (100%) + }; + + // Run code + let mut run_count = 0; + if len > 1 { + let mut prev_type = char_types[0]; + for ¤t_type in char_types.iter().skip(1) { + if current_type != prev_type { + run_count += 1; + prev_type = current_type; + } + } + } + let run_ratio = (run_count as f64) / ((len - 1) as f64); + let run_code = match run_ratio { + r if r <= 0.1 => b'0', // Very long runs (e.g., AAAABBBB) + r if r <= 0.3 => b'1', // Moderate runs + r if r <= 0.5 => b'2', // Balanced runs/alternation + r if r <= 0.7 => b'3', // High alternation + _ => b'4', // Near maximum alternation (e.g., A1A1A1) + }; + + Token::Alphanumeric { + code: [composition, len_code, ratio, run_code], + } + } + + fn from_number(is_float: bool, num: &str) -> Self { + Token::Number { + code: [ + u8::from(is_float), + u8::from(num.starts_with('-')), + num.as_bytes() + .iter() + .filter(|c| c.is_ascii_digit()) + .count() + .min(u8::MAX as usize) as u8, + ], + } + } +} + +fn lower_prefix(prefix: &str, value: &str) -> String { + let mut result = String::with_capacity(prefix.len() + value.len()); + result.push_str(prefix); + for ch in value.chars() { + for lower_ch in ch.to_lowercase() { + result.push(lower_ch); + } + } + result +} + +fn truncate_word(word: &str, max_len: usize) -> &str { + if word.len() <= max_len { + word + } else { + let mut pos = 0; + for (count, (idx, _)) in word.char_indices().enumerate() { + pos = idx; + if count == max_len { + break; + } + } + &word[..pos] + } +} + +impl Feature for Token<'_> { + fn prefix(&self) -> u16 { + match self { + Token::Word { .. } => 0, + Token::Number { .. } => 1, + Token::Alphanumeric { .. } => 2, + Token::Symbol { .. } => 3, + Token::Sender { .. } => 4, + Token::Asn { .. } => 5, + Token::Url { .. } => 6, + Token::Email { .. } => 7, + Token::Hostname { .. } => 8, + Token::Attachment { .. } => 9, + } + } + + fn value(&self) -> &[u8] { + match self { + Token::Word { value } => value.as_bytes(), + Token::Number { code } => code, + Token::Alphanumeric { code } => code, + Token::Symbol { value } => value.as_bytes(), + Token::Sender { value } => value.as_bytes(), + Token::Asn { number } => number, + Token::Url { value } => value.as_bytes(), + Token::Email { value } => value.as_bytes(), + Token::Hostname { value } => value.as_bytes(), + Token::Attachment { value } => value.as_bytes(), + } + } + + fn is_global_feature(&self) -> bool { + matches!( + self, + Token::Sender { .. } + | Token::Asn { .. } + | Token::Url { .. } + | Token::Email { .. } + | Token::Hostname { .. } + | Token::Attachment { .. } + ) + } + + fn is_local_feature(&self) -> bool { + true + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +enum CharType { + Upper, + Lower, + Digit, + Other, +} + +impl CharType { + fn from_char(c: char) -> CharType { + match c { + 'A'..='Z' => CharType::Upper, + 'a'..='z' => CharType::Lower, + '0'..='9' => CharType::Digit, + _ => CharType::Other, + } + } +} + +impl<'x> Default for Tokens<'x> { + fn default() -> Self { + Tokens(HashMap::with_capacity(128)) } } diff --git a/crates/store/src/write/blob.rs b/crates/store/src/write/blob.rs index 568c6ae9..cff5cb18 100644 --- a/crates/store/src/write/blob.rs +++ b/crates/store/src/write/blob.rs @@ -143,8 +143,8 @@ impl Store { let mut delete_keys = Vec::new(); let now = now(); self.iterate( - IterateParams::new(from_key, to_key).ascending().no_values(), - |key, _| { + IterateParams::new(from_key, to_key).ascending(), + |key, value| { let hash = BlobHash::try_from_hash_slice( key.get(0..BLOB_HASH_LEN) .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?, @@ -181,21 +181,35 @@ impl Store { to: BlobLink::Temporary { until }, }, )); - if account_id != u32::MAX { - delete_keys.push(( - Some(account_id), - BlobOp::Quota { - hash: last_hash.clone(), - until, - }, - )); - delete_keys.push(( - Some(account_id), - BlobOp::Undelete { - hash: last_hash.clone(), - until, - }, - )); + match value.first().copied() { + Some(BlobLink::QUOTA_LINK) => { + delete_keys.push(( + Some(account_id), + BlobOp::Quota { + hash: last_hash.clone(), + until, + }, + )); + } + Some(BlobLink::UNDELETE_LINK) => { + delete_keys.push(( + Some(account_id), + BlobOp::Undelete { + hash: last_hash.clone(), + until, + }, + )); + } + Some(BlobLink::SPAM_SAMPLE_LINK) => { + delete_keys.push(( + Some(account_id), + BlobOp::SpamSample { + hash: last_hash.clone(), + until, + }, + )); + } + _ => {} } } else { last_hash_is_linked = true; diff --git a/crates/store/src/write/key.rs b/crates/store/src/write/key.rs index 3cc48bc8..52d56f84 100644 --- a/crates/store/src/write/key.rs +++ b/crates/store/src/write/key.rs @@ -289,16 +289,6 @@ impl ValueClass { .write(if *is_insert { 7u8 } else { 8u8 }) .write(document_id) .write(index.to_u8()), - TaskQueueClass::SpamTrain { - due, - blob_hash, - learn_spam, - } => serializer - .write(due.inner()) - .write(account_id) - .write(if *learn_spam { 1u8 } else { 2u8 }) - .write(blob_hash.as_slice()) - .write(document_id), TaskQueueClass::SendAlarm { due, event_id, @@ -348,15 +338,20 @@ impl ValueClass { .write(*until), }, BlobOp::Quota { hash, until } => serializer + .write(BlobLink::QUOTA_LINK) .write(account_id) - .write(0u8) .write::<&[u8]>(hash.as_ref()) .write(*until), BlobOp::Undelete { hash, until } => serializer + .write(BlobLink::UNDELETE_LINK) .write(account_id) - .write(1u8) .write::<&[u8]>(hash.as_ref()) .write(*until), + BlobOp::SpamSample { hash, until } => serializer + .write(BlobLink::SPAM_SAMPLE_LINK) + .write(*until) + .write(account_id) + .write::<&[u8]>(hash.as_ref()), }, ValueClass::Config(key) => serializer.write(key.as_slice()), ValueClass::InMemory(lookup) => match lookup { @@ -511,6 +506,12 @@ impl ValueClass { } } +impl BlobLink { + pub const QUOTA_LINK: u8 = 0; + pub const UNDELETE_LINK: u8 = 1; + pub const SPAM_SAMPLE_LINK: u8 = 2; +} + impl + Sync + Send + Clone> Key for IndexKey { fn subspace(&self) -> u8 { SUBSPACE_INDEXES @@ -582,10 +583,10 @@ impl ValueClass { BlobOp::Quota { .. } | BlobOp::Undelete { .. } => { BLOB_HASH_LEN + U32_LEN + U64_LEN + 1 } + BlobOp::SpamSample { .. } => BLOB_HASH_LEN + U32_LEN + 2, }, ValueClass::TaskQueue(e) => match e { TaskQueueClass::UpdateIndex { .. } => (U64_LEN * 2) + 2, - TaskQueueClass::SpamTrain { .. } => BLOB_HASH_LEN + (U64_LEN * 2) + 1, TaskQueueClass::SendAlarm { .. } | TaskQueueClass::MergeThreads { .. } => { U64_LEN + (U32_LEN * 3) + 1 } @@ -642,7 +643,9 @@ impl ValueClass { ValueClass::TaskQueue { .. } => SUBSPACE_TASK_QUEUE, ValueClass::Blob(op) => match op { BlobOp::Commit { .. } | BlobOp::Link { .. } => SUBSPACE_BLOB_LINK, - BlobOp::Quota { .. } | BlobOp::Undelete { .. } => SUBSPACE_BLOB_EXTRA, + BlobOp::Quota { .. } | BlobOp::Undelete { .. } | BlobOp::SpamSample { .. } => { + SUBSPACE_BLOB_EXTRA + } }, ValueClass::Config(_) => SUBSPACE_SETTINGS, ValueClass::InMemory(lookup) => match lookup { diff --git a/crates/store/src/write/mod.rs b/crates/store/src/write/mod.rs index fc12c7bd..200ec83f 100644 --- a/crates/store/src/write/mod.rs +++ b/crates/store/src/write/mod.rs @@ -226,11 +226,6 @@ pub enum TaskQueueClass { index: SearchIndex, is_insert: bool, }, - SpamTrain { - due: TaskEpoch, - blob_hash: BlobHash, - learn_spam: bool, - }, SendAlarm { due: TaskEpoch, event_id: u16, @@ -380,6 +375,7 @@ pub enum BlobOp { Link { hash: BlobHash, to: BlobLink }, Quota { hash: BlobHash, until: u64 }, Undelete { hash: BlobHash, until: u64 }, + SpamSample { hash: BlobHash, until: u64 }, } #[derive(Debug, PartialEq, Clone, Eq, Hash)]