Nats + Redis PubSub cluster updates replacing Gossip protocol

tests/src/cluster/broadcast.rs

@@ -0,0 +1,31 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::net::IpAddr;
+
+use crate::imap::idle;
+
+use super::ClusterTest;
+
+pub async fn test(cluster: &ClusterTest) {
+    println!("Running cluster broadcast tests...");
+
+    // Run IMAP idle tests across nodes
+    let mut node1_client = cluster.imap_client("john", 1).await;
+    let mut node2_client = cluster.imap_client("john", 2).await;
+    idle::test(&mut node1_client, &mut node2_client, true).await;
+
+    // Test event broadcast
+    let server1 = cluster.server(1);
+    let server2 = cluster.server(2);
+    let test_ip: IpAddr = "8.8.8.8".parse().unwrap();
+    assert!(!server1.is_ip_blocked(&test_ip));
+    assert!(!server2.is_ip_blocked(&test_ip));
+    server1.block_ip(test_ip).await.unwrap();
+    tokio::time::sleep(std::time::Duration::from_millis(200)).await;
+    assert!(server1.is_ip_blocked(&test_ip));
+    assert!(server2.is_ip_blocked(&test_ip));
+}

tests/src/cluster/mod.rs

@@ -0,0 +1,362 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::{path::PathBuf, sync::Arc, time::Duration};
+
+use common::{
+    Caches, Core, Data, Inner, Server,
+    config::{
+        server::{Listeners, ServerProtocol},
+        telemetry::Telemetry,
+    },
+    core::BuildServer,
+    manager::{
+        boot::build_ipc,
+        config::{ConfigManager, Patterns},
+    },
+};
+use http::HttpSessionManager;
+use imap::core::ImapSessionManager;
+use imap_proto::ResponseType;
+use jmap_client::client::{Client, Credentials};
+use managesieve::core::ManageSieveSessionManager;
+use pop3::Pop3SessionManager;
+use services::{SpawnServices, broadcast::subscriber::spawn_broadcast_subscriber};
+use smtp::{SpawnQueueManager, core::SmtpSessionManager};
+use store::Stores;
+use tokio::sync::watch;
+use utils::config::Config;
+
+use crate::{
+    AssertConfig, TEST_USERS, add_test_certs,
+    directory::internal::TestInternalDirectory,
+    imap::{ImapConnection, Type},
+    jmap::enterprise::EnterpriseCore,
+};
+
+pub mod broadcast;
+pub mod stress;
+
+pub const NUM_NODES: usize = 3;
+
+#[tokio::test(flavor = "multi_thread")]
+pub async fn cluster_tests() {
+    let params = init_cluster_tests(true).await;
+    //stress::test(params.server.clone(), params.client).await;
+    broadcast::test(&params).await;
+}
+
+#[allow(dead_code)]
+pub struct ClusterTest {
+    servers: Vec<Server>,
+    shutdown_txs: Vec<watch::Sender<bool>>,
+}
+
+async fn init_cluster_tests(delete_if_exists: bool) -> ClusterTest {
+    // Load and parse config
+    let store_id = std::env::var("STORE").expect(
+        "Missing store type. Try running `STORE=<store_type> PUBSUB=<pubsub_type> cargo test`",
+    );
+    let pubsub_id = std::env::var("PUBSUB").expect(
+        "Missing store type. Try running `STORE=<store_type> PUBSUB=<pubsub_type> cargo test`",
+    );
+    let mut pubsub_config = match pubsub_id.as_str() {
+        "nats" => Config::new(SERVER_NATS).unwrap(),
+        "redis" => Config::new(SERVER_REDIS).unwrap(),
+        _ => panic!("Unsupported pubsub type: {}", pubsub_id),
+    };
+
+    // Build configs
+    let mut configs = Vec::with_capacity(NUM_NODES);
+    for node_id in 0..NUM_NODES {
+        let mut config = Config::new(
+            add_test_certs(SERVER)
+                .replace("{STORE}", &store_id)
+                .replace("{PUBSUB}", &pubsub_id)
+                .replace("{NODE_ID}", &node_id.to_string())
+                .replace(
+                    "{LEVEL}",
+                    &std::env::var("LOG").unwrap_or_else(|_| "disable".to_string()),
+                ),
+        )
+        .unwrap();
+        config.resolve_all_macros().await;
+        configs.push(config);
+    }
+
+    // Build stores
+    let stores = Stores::parse_all(configs.first_mut().unwrap(), false).await;
+
+    // Build servers
+    let mut servers = Vec::with_capacity(NUM_NODES);
+    let mut shutdown_txs = Vec::with_capacity(NUM_NODES);
+    for config in configs {
+        let mut stores = stores.clone();
+        stores.pubsub_stores = Stores::parse(&mut pubsub_config).await.pubsub_stores;
+        let (server, shutdown_tx) = build_server(config, stores).await;
+        servers.push(server);
+        shutdown_txs.push(shutdown_tx);
+    }
+
+    let store = servers.first().unwrap().store().clone();
+    if delete_if_exists {
+        store.destroy().await;
+    }
+
+    // Create test users
+    for (account, secret, name, email) in TEST_USERS {
+        let _account_id = store
+            .create_test_user(account, secret, name, &[email])
+            .await;
+    }
+
+    ClusterTest {
+        servers,
+        shutdown_txs,
+    }
+}
+
+impl ClusterTest {
+    pub async fn jmap_client(&self, login: &str, node_id: u32) -> Client {
+        Client::new()
+            .credentials(Credentials::basic(login, find_account_secret(login)))
+            .timeout(Duration::from_secs(3600))
+            .accept_invalid_certs(true)
+            .connect(&format!("https://127.0.0.1:1800{node_id}"))
+            .await
+            .unwrap()
+    }
+
+    pub async fn imap_client(&self, login: &str, node_id: u32) -> ImapConnection {
+        let mut conn = ImapConnection::connect_to(b"A1 ", format!("127.0.0.1:1900{node_id}")).await;
+        conn.assert_read(Type::Untagged, ResponseType::Ok).await;
+        conn.authenticate(login, find_account_secret(login)).await;
+        conn
+    }
+
+    pub fn server(&self, node_id: usize) -> &Server {
+        self.servers
+            .get(node_id)
+            .unwrap_or_else(|| panic!("No server found for node ID: {}", node_id))
+    }
+}
+
+fn find_account_secret(login: &str) -> &str {
+    TEST_USERS
+        .iter()
+        .find(|(account, _, _, _)| account == &login)
+        .map(|(_, secret, _, _)| secret)
+        .unwrap_or_else(|| panic!("No account found for login: {}", login))
+}
+
+async fn build_server(mut config: Config, stores: Stores) -> (Server, watch::Sender<bool>) {
+    // Parse servers
+    let mut servers = Listeners::parse(&mut config);
+
+    // Bind ports and drop privileges
+    servers.bind_and_drop_priv(&mut config);
+
+    // Parse core
+    let config_manager = ConfigManager {
+        cfg_local: Default::default(),
+        cfg_local_path: PathBuf::new(),
+        cfg_local_patterns: Patterns::parse(&mut config).into(),
+        cfg_store: config
+            .value("storage.data")
+            .and_then(|id| stores.stores.get(id))
+            .cloned()
+            .unwrap_or_default(),
+    };
+    let tracers = Telemetry::parse(&mut config, &stores);
+    let core = Core::parse(&mut config, stores, config_manager)
+        .await
+        .enable_enterprise();
+    let data = Data::parse(&mut config);
+    let cache = Caches::parse(&mut config);
+    let (ipc, mut ipc_rxs) = build_ipc(&mut config, true);
+    let inner = Arc::new(Inner {
+        shared_core: core.into_shared(),
+        data,
+        ipc,
+        cache,
+    });
+
+    // Parse acceptors
+    servers.parse_tcp_acceptors(&mut config, inner.clone());
+
+    // Enable tracing
+    tracers.enable(true);
+
+    // Start services
+    config.assert_no_errors();
+    ipc_rxs.spawn_queue_manager(inner.clone());
+    ipc_rxs.spawn_services(inner.clone());
+
+    // Spawn servers
+    let (shutdown_tx, shutdown_rx) = servers.spawn(|server, acceptor, shutdown_rx| {
+        match &server.protocol {
+            ServerProtocol::Smtp | ServerProtocol::Lmtp => server.spawn(
+                SmtpSessionManager::new(inner.clone()),
+                inner.clone(),
+                acceptor,
+                shutdown_rx,
+            ),
+            ServerProtocol::Http => server.spawn(
+                HttpSessionManager::new(inner.clone()),
+                inner.clone(),
+                acceptor,
+                shutdown_rx,
+            ),
+            ServerProtocol::Imap => server.spawn(
+                ImapSessionManager::new(inner.clone()),
+                inner.clone(),
+                acceptor,
+                shutdown_rx,
+            ),
+            ServerProtocol::Pop3 => server.spawn(
+                Pop3SessionManager::new(inner.clone()),
+                inner.clone(),
+                acceptor,
+                shutdown_rx,
+            ),
+            ServerProtocol::ManageSieve => server.spawn(
+                ManageSieveSessionManager::new(inner.clone()),
+                inner.clone(),
+                acceptor,
+                shutdown_rx,
+            ),
+        };
+    });
+
+    // Start broadcast subscriber
+    spawn_broadcast_subscriber(inner.clone(), shutdown_rx);
+
+    (inner.build_server(), shutdown_tx)
+}
+
+const SERVER: &str = r#"
+[server]
+hostname = "'server{NODE_ID}.example.org'"
+http.url = "'https://127.0.0.1:800{NODE_ID}'"
+
+[cluster]
+node-id = {NODE_ID}
+
+[server.listener.http]
+bind = ["127.0.0.1:1800{NODE_ID}"]
+protocol = "http"
+max-connections = 81920
+tls.implicit = true
+
+[server.listener.imap]
+bind = ["127.0.0.1:1900{NODE_ID}"]
+protocol = "imap"
+max-connections = 81920
+
+[server.listener.lmtp]
+bind = ['127.0.0.1:1700{NODE_ID}']
+protocol = 'lmtp'
+tls.implicit = false
+
+[server.socket]
+reuse-addr = true
+
+[server.tls]
+enable = true
+implicit = false
+certificate = "default"
+
+[session.ehlo]
+reject-non-fqdn = false
+
+[session.rcpt]
+relay = [ { if = "!is_empty(authenticated_as)", then = true }, 
+          { else = false } ]
+directory = "'{STORE}'"
+
+[session.rcpt.errors]
+total = 5
+wait = "1ms"
+
+[session.auth]
+mechanisms = "[plain, login, oauthbearer]"
+directory = "'{STORE}'"
+
+[resolver]
+type = "system"
+
+[queue.outbound]
+next-hop = [ { if = "rcpt_domain == 'example.com'", then = "'local'" }, 
+             { if = "contains(['remote.org', 'foobar.com', 'test.com', 'other_domain.com'], rcpt_domain)", then = "'mock-smtp'" },
+             { else = false } ]
+
+[store."foundationdb"]
+type = "foundationdb"
+
+[store."postgresql"]
+type = "postgresql"
+host = "localhost"
+port = 5432
+database = "stalwart"
+user = "postgres"
+password = "mysecretpassword"
+
+[store."mysql"]
+type = "mysql"
+host = "localhost"
+port = 3307
+database = "stalwart"
+user = "root"
+password = "password"
+
+[certificate.default]
+cert = "%{file:{CERT}}%"
+private-key = "%{file:{PK}}%"
+
+[storage]
+data = "{STORE}"
+fts = "{STORE}"
+blob = "{STORE}"
+lookup = "{STORE}"
+directory = "{STORE}"
+pubsub = "{PUBSUB}"
+
+[directory."{STORE}"]
+type = "internal"
+store = "{STORE}"
+
+[imap.auth]
+allow-plain-text = true
+
+[oauth]
+key = "parerga_und_paralipomena"
+
+[spam-filter]
+enable = false
+
+[tracer.console]
+type = "console"
+level = "{LEVEL}"
+multiline = false
+ansi = true
+disabled-events = ["network.*", "telemetry.webhook-error", "http.request-body", 
+                   "eval.result", "store.*", "dkim.*", "queue.*", "delivery.*",
+                   "*.raw-input", "*.raw-output" ]
+"#;
+
+const SERVER_NATS: &str = r#"
+[store."nats"]
+type = "nats"
+urls = "127.0.0.1:4444"
+"#;
+
+const SERVER_REDIS: &str = r#"
+[store."redis"]
+type = "redis"
+urls = "redis://127.0.0.1"
+redis-type = "single"
+
+"#;

tests/src/cluster/stress.rs

@@ -0,0 +1,431 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::jmap::{assert_is_empty, mailbox::destroy_all_mailboxes_no_wait, wait_for_index};
+use common::Server;
+use directory::backend::internal::manage::ManageDirectory;
+use email::{
+    cache::{MessageCacheFetch, email::MessageCacheAccess},
+    message::metadata::MessageData,
+};
+use futures::future::join_all;
+use jmap_client::{
+    client::Client,
+    core::set::{SetErrorType, SetObject},
+    mailbox::{self, Mailbox, Role},
+};
+use jmap_proto::types::{collection::Collection, id::Id};
+use std::{sync::Arc, time::Duration};
+use store::{
+    rand::{self, Rng},
+    roaring::RoaringBitmap,
+};
+
+const TEST_USER_ID: u32 = 1;
+const NUM_PASSES: usize = 1;
+
+pub async fn test(server: Server, mut client: Client) {
+    println!("Running cluster concurrency stress tests...");
+    server
+        .core
+        .storage
+        .data
+        .get_or_create_principal_id("john", directory::Type::Individual)
+        .await
+        .unwrap();
+    client.set_default_account_id(Id::from(TEST_USER_ID).to_string());
+    let client = Arc::new(client);
+    email_tests(server.clone(), client.clone()).await;
+    mailbox_tests(server.clone(), client.clone()).await;
+}
+
+async fn email_tests(server: Server, client: Arc<Client>) {
+    for pass in 0..NUM_PASSES {
+        println!(
+            "----------------- EMAIL STRESS TEST {} -----------------",
+            pass
+        );
+        let mailboxes = Arc::new(vec![
+            client
+                .mailbox_create("Stress 1", None::<String>, Role::None)
+                .await
+                .unwrap()
+                .take_id(),
+            client
+                .mailbox_create("Stress 2", None::<String>, Role::None)
+                .await
+                .unwrap()
+                .take_id(),
+            client
+                .mailbox_create("Stress 3", None::<String>, Role::None)
+                .await
+                .unwrap()
+                .take_id(),
+        ]);
+        let mut futures = Vec::new();
+
+        for num in 0..1000 {
+            match rand::rng().random_range(0..3) {
+                0 => {
+                    let client = client.clone();
+                    let mailboxes = mailboxes.clone();
+                    futures.push(tokio::spawn(async move {
+                        let mailbox_num = rand::rng().random_range::<usize, _>(0..mailboxes.len());
+                        let _message_id = client
+                            .email_import(
+                                format!(
+                                    concat!(
+                                        "From: test@test.com\n",
+                                        "To: test@test.com\r\n",
+                                        "Subject: test {}\r\n\r\ntest {}\r\n"
+                                    ),
+                                    num, num
+                                )
+                                .into_bytes(),
+                                [&mailboxes[mailbox_num]],
+                                None::<Vec<String>>,
+                                None,
+                            )
+                            .await
+                            .unwrap()
+                            .take_id();
+                        /*println!(
+                            "Inserted message {}.",
+                            Id::from_bytes(_message_id.as_bytes())
+                                .unwrap()
+                                .document_id()
+                        );*/
+                    }));
+                }
+
+                1 => {
+                    let client = client.clone();
+                    futures.push(tokio::spawn(async move {
+                        loop {
+                            let mut req = client.build();
+                            req.query_email();
+                            let ids = req.send_query_email().await.unwrap().take_ids();
+                            if !ids.is_empty() {
+                                let message_id = &ids[rand::rng().random_range(0..ids.len())];
+                                /*println!(
+                                    "Deleting message {}.",
+                                    Id::from_bytes(message_id.as_bytes()).unwrap().document_id()
+                                );*/
+                                match client.email_destroy(message_id).await {
+                                    Ok(_) => {
+                                        break;
+                                    }
+                                    Err(jmap_client::Error::Set(err)) => match err.error() {
+                                        SetErrorType::NotFound => {
+                                            break;
+                                        }
+                                        SetErrorType::Forbidden => {
+                                            // Concurrency issue, try again.
+                                            //println!("Concurrent update, trying again.");
+                                        }
+                                        _ => {
+                                            panic!("Unexpected error: {:?}", err);
+                                        }
+                                    },
+                                    Err(err) => {
+                                        panic!("Unexpected error: {:?}", err);
+                                    }
+                                }
+                            } else {
+                                break;
+                            }
+                        }
+                    }));
+                }
+                _ => {
+                    let client = client.clone();
+                    let mailboxes = mailboxes.clone();
+                    futures.push(tokio::spawn(async move {
+                        let mut req = client.build();
+                        let ref_id = req.query_email().result_reference();
+                        req.get_email()
+                            .ids_ref(ref_id)
+                            .properties([jmap_client::email::Property::MailboxIds]);
+                        let emails = req
+                            .send()
+                            .await
+                            .unwrap()
+                            .unwrap_method_responses()
+                            .pop()
+                            .unwrap()
+                            .unwrap_get_email()
+                            .unwrap()
+                            .take_list();
+
+                        if !emails.is_empty() {
+                            let message = &emails[rand::rng().random_range(0..emails.len())];
+                            let message_id = message.id().unwrap();
+                            let mailbox_ids = message.mailbox_ids();
+                            assert_eq!(mailbox_ids.len(), 1, "{:#?}", message);
+                            let mailbox_id = mailbox_ids.last().unwrap();
+                            loop {
+                                let new_mailbox_id =
+                                    &mailboxes[rand::rng().random_range(0..mailboxes.len())];
+                                if new_mailbox_id != mailbox_id {
+                                    /*println!(
+                                        "Moving message {} from {} to {}.",
+                                        Id::from_bytes(message_id.as_bytes())
+                                            .unwrap()
+                                            .document_id(),
+                                        Id::from_bytes(mailbox_id.as_bytes())
+                                            .unwrap()
+                                            .document_id(),
+                                        Id::from_bytes(new_mailbox_id.as_bytes())
+                                            .unwrap()
+                                            .document_id()
+                                    );*/
+                                    let mut req = client.build();
+                                    req.set_email()
+                                        .update(message_id)
+                                        .mailbox_ids([new_mailbox_id]);
+                                    req.send_set_email().await.unwrap();
+
+                                    break;
+                                }
+                            }
+                        }
+                    }));
+                }
+            }
+            tokio::time::sleep(Duration::from_millis(rand::rng().random_range(5..10))).await;
+        }
+
+        join_all(futures).await;
+
+        let email_ids = server
+            .get_document_ids(TEST_USER_ID, Collection::Email)
+            .await
+            .unwrap()
+            .unwrap_or_default();
+        let mailbox_ids = server
+            .get_document_ids(TEST_USER_ID, Collection::Mailbox)
+            .await
+            .unwrap()
+            .unwrap_or_default();
+        assert_eq!(mailbox_ids.len(), 8);
+
+        for mailbox in mailboxes.iter() {
+            let mailbox_id = Id::from_bytes(mailbox.as_bytes()).unwrap().document_id();
+            let email_ids_in_mailbox = RoaringBitmap::from_iter(
+                server
+                    .get_cached_messages(TEST_USER_ID)
+                    .await
+                    .unwrap()
+                    .in_mailbox(mailbox_id)
+                    .map(|m| m.document_id),
+            );
+            let mut email_ids_check = email_ids_in_mailbox.clone();
+            email_ids_check &= &email_ids;
+            assert_eq!(email_ids_in_mailbox, email_ids_check);
+
+            //println!("Emails {:?}", email_ids_in_mailbox);
+
+            for email_id in &email_ids_in_mailbox {
+                if let Some(mailbox_tags) = server
+                    .get_archive(TEST_USER_ID, Collection::Email, email_id)
+                    .await
+                    .unwrap()
+                {
+                    let mailbox_tags = mailbox_tags.deserialize::<MessageData>().unwrap().mailboxes;
+                    if mailbox_tags.len() != 1 {
+                        panic!(
+                            "Email ORM has more than one mailbox {:?}! Id {} in mailbox {} with messages {:?}",
+                            mailbox_tags, email_id, mailbox_id, email_ids_in_mailbox
+                        );
+                    }
+                    let mailbox_tag = mailbox_tags[0];
+                    assert!(mailbox_tag.uid != 0);
+                    if mailbox_tag.mailbox_id != mailbox_id {
+                        panic!(
+                            concat!(
+                                "Email ORM has an unexpected mailbox tag {:?}! Id {} in ",
+                                "mailbox {} with messages {:?}"
+                            ),
+                            mailbox_tag, email_id, mailbox_id, email_ids_in_mailbox,
+                        );
+                    }
+                } else {
+                    panic!(
+                        "Email tags not found! Id {} in mailbox {} with messages {:?}",
+                        email_id, mailbox_id, email_ids_in_mailbox
+                    );
+                }
+            }
+        }
+
+        wait_for_index(&server).await;
+        destroy_all_mailboxes_no_wait(&client).await;
+        assert_is_empty(server.clone()).await;
+    }
+}
+
+async fn mailbox_tests(server: Server, client: Arc<Client>) {
+    let mailboxes = Arc::new(vec![
+        "test/test1/test2/test3".to_string(),
+        "test1/test2/test3".to_string(),
+        "test2/test3/test4".to_string(),
+        "test3/test4/test5".to_string(),
+        "test4".to_string(),
+        "test5".to_string(),
+    ]);
+    let mut futures = Vec::new();
+
+    println!("----------------- MAILBOX STRESS TEST -----------------");
+
+    for _ in 0..1000 {
+        match rand::rng().random_range(0..=3) {
+            0 => {
+                for pos in 0..mailboxes.len() {
+                    let client = client.clone();
+                    let mailboxes = mailboxes.clone();
+                    futures.push(tokio::spawn(async move {
+                        //println!("Creating mailbox {}.", mailboxes[pos]);
+                        create_mailbox(&client, &mailboxes[pos]).await;
+                    }));
+                }
+            }
+
+            1 => {
+                let client = client.clone();
+                futures.push(tokio::spawn(async move {
+                    //print!("Querying mailboxes...");
+                    query_mailboxes(&client).await;
+                }));
+            }
+
+            2 => {
+                let client = client.clone();
+                futures.push(tokio::spawn(async move {
+                    for mailbox_id in client
+                        .mailbox_query(None::<mailbox::query::Filter>, None::<Vec<_>>)
+                        .await
+                        .unwrap()
+                        .take_ids()
+                    {
+                        let client = client.clone();
+                        tokio::spawn(async move {
+                            //println!("Deleting mailbox {}.", mailbox_id);
+                            delete_mailbox(&client, &mailbox_id).await;
+                        });
+                    }
+                }));
+            }
+
+            _ => {
+                let client = client.clone();
+                futures.push(tokio::spawn(async move {
+                    let mut ids = client
+                        .mailbox_query(None::<mailbox::query::Filter>, None::<Vec<_>>)
+                        .await
+                        .unwrap()
+                        .take_ids();
+                    if !ids.is_empty() {
+                        let id = ids.swap_remove(rand::rng().random_range(0..ids.len()));
+                        let sort_order = rand::rng().random_range(0..100);
+                        //println!("Updating mailbox {}.", id);
+                        client.mailbox_update_sort_order(&id, sort_order).await.ok();
+                    }
+                }));
+            }
+        }
+        tokio::time::sleep(Duration::from_millis(rand::rng().random_range(5..10))).await;
+    }
+
+    join_all(futures).await;
+
+    wait_for_index(&server).await;
+    for mailbox_id in client
+        .mailbox_query(None::<mailbox::query::Filter>, None::<Vec<_>>)
+        .await
+        .unwrap()
+        .take_ids()
+    {
+        let _ = client.mailbox_move(&mailbox_id, None::<String>).await;
+    }
+    for mailbox_id in client
+        .mailbox_query(None::<mailbox::query::Filter>, None::<Vec<_>>)
+        .await
+        .unwrap()
+        .take_ids()
+    {
+        let _ = client.mailbox_destroy(&mailbox_id, true).await;
+    }
+    assert_is_empty(server).await;
+}
+
+async fn create_mailbox(client: &Client, mailbox: &str) -> Vec<String> {
+    let mut request = client.build();
+    let mut create_ids: Vec<String> = Vec::new();
+    let set_request = request.set_mailbox();
+    for path_item in mailbox.split('/') {
+        let create_item = set_request.create().name(path_item);
+        if let Some(create_id) = create_ids.last() {
+            create_item.parent_id_ref(create_id);
+        }
+        create_ids.push(create_item.create_id().unwrap());
+    }
+    let mut response = request.send_set_mailbox().await.unwrap();
+    let mut ids = Vec::with_capacity(create_ids.len());
+    for create_id in create_ids {
+        if let Ok(mut id) = response.created(&create_id) {
+            ids.push(id.take_id());
+        }
+    }
+    ids
+}
+
+async fn query_mailboxes(client: &Client) -> Vec<Mailbox> {
+    let mut request = client.build();
+    let query_result = request
+        .query_mailbox()
+        .calculate_total(true)
+        .result_reference();
+    request.get_mailbox().ids_ref(query_result).properties([
+        jmap_client::mailbox::Property::Id,
+        jmap_client::mailbox::Property::Name,
+        jmap_client::mailbox::Property::IsSubscribed,
+        jmap_client::mailbox::Property::ParentId,
+        jmap_client::mailbox::Property::Role,
+        jmap_client::mailbox::Property::TotalEmails,
+        jmap_client::mailbox::Property::UnreadEmails,
+    ]);
+
+    request
+        .send()
+        .await
+        .unwrap()
+        .unwrap_method_responses()
+        .pop()
+        .unwrap()
+        .unwrap_get_mailbox()
+        .unwrap()
+        .take_list()
+}
+
+async fn delete_mailbox(client: &Client, mailbox_id: &str) {
+    for _ in 0..3 {
+        match client.mailbox_destroy(mailbox_id, true).await {
+            Ok(_) => return,
+            Err(err) => match err {
+                jmap_client::Error::Set(_) => break,
+                jmap_client::Error::Transport(_) => {
+                    let backoff = rand::rng().random_range(50..=300);
+                    tokio::time::sleep(Duration::from_millis(backoff)).await;
+                }
+                _ => panic!("Failed: {:?}", err),
+            },
+        }
+    }
+    /*println!(
+        "Warning: Too many transport errors while deleting mailbox {}.",
+        mailbox_id
+    );*/
+}