Registry testing - part 5

tests/src/utils/account.rs

@@ -117,6 +117,10 @@ impl Account {
         }
     }
 
+    pub fn update_secret(&mut self, new_secret: &'static str) {
+        self.secret = new_secret;
+    }
+
     pub fn id(&self) -> &Id {
         &self.id
     }

tests/src/utils/http.rs

@@ -0,0 +1,127 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use hyper::Method;
+use serde::{Serialize, de::DeserializeOwned};
+use std::time::Duration;
+
+pub struct HttpRequest {
+    pub port: u16,
+    pub username: Option<String>,
+    pub password: Option<String>,
+}
+
+impl Default for HttpRequest {
+    fn default() -> Self {
+        Self {
+            port: 8899,
+            username: None,
+            password: None,
+        }
+    }
+}
+
+impl HttpRequest {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    pub fn with_credentials(port: u16, username: &str, password: &str) -> Self {
+        Self {
+            port,
+            username: Some(username.to_string()),
+            password: Some(password.to_string()),
+        }
+    }
+
+    pub async fn post<T: DeserializeOwned>(
+        &self,
+        query: &str,
+        body: &impl Serialize,
+    ) -> Result<T, String> {
+        self.request_raw(
+            Method::POST,
+            query,
+            Some(serde_json::to_string(body).unwrap()),
+        )
+        .await
+        .map(|result| {
+            serde_json::from_str::<T>(&result).unwrap_or_else(|err| panic!("{err}: {result}"))
+        })
+    }
+
+    pub async fn patch<T: DeserializeOwned>(
+        &self,
+        query: &str,
+        body: &impl Serialize,
+    ) -> Result<T, String> {
+        self.request_raw(
+            Method::PATCH,
+            query,
+            Some(serde_json::to_string(body).unwrap()),
+        )
+        .await
+        .map(|result| {
+            serde_json::from_str::<T>(&result).unwrap_or_else(|err| panic!("{err}: {result}"))
+        })
+    }
+
+    pub async fn delete<T: DeserializeOwned>(&self, query: &str) -> Result<T, String> {
+        self.request_raw(Method::DELETE, query, None)
+            .await
+            .map(|result| {
+                serde_json::from_str::<T>(&result).unwrap_or_else(|err| panic!("{err}: {result}"))
+            })
+    }
+
+    pub async fn get<T: DeserializeOwned>(&self, query: &str) -> Result<T, String> {
+        self.request_raw(Method::GET, query, None)
+            .await
+            .map(|result| {
+                serde_json::from_str::<T>(&result).unwrap_or_else(|err| panic!("{err}: {result}"))
+            })
+    }
+    pub async fn request<T: DeserializeOwned>(
+        &self,
+        method: Method,
+        query: &str,
+    ) -> Result<T, String> {
+        self.request_raw(method, query, None).await.map(|result| {
+            serde_json::from_str::<T>(&result).unwrap_or_else(|err| panic!("{err}: {result}"))
+        })
+    }
+
+    async fn request_raw(
+        &self,
+        method: Method,
+        query: &str,
+        body: Option<String>,
+    ) -> Result<String, String> {
+        let mut request = reqwest::Client::builder()
+            .timeout(Duration::from_millis(500))
+            .danger_accept_invalid_certs(true)
+            .build()
+            .unwrap()
+            .request(method, format!("https://127.0.0.1:{}{query}", self.port));
+
+        if let Some(body) = body {
+            request = request.body(body);
+        }
+
+        if let (Some(username), Some(password)) = (&self.username, &self.password) {
+            request = request.basic_auth(username, Some(password));
+        }
+
+        request
+            .send()
+            .await
+            .map_err(|err| err.to_string())?
+            .bytes()
+            .await
+            .map(|bytes| String::from_utf8(bytes.to_vec()).unwrap())
+            .map_err(|err| err.to_string())
+    }
+}

tests/src/utils/imap.rs

@@ -0,0 +1,147 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use base64::{Engine, engine::general_purpose};
+use imap_proto::ResponseType;
+use std::time::Duration;
+use tokio::{
+    io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines, ReadHalf, WriteHalf},
+    net::TcpStream,
+};
+
+pub struct ImapConnection {
+    tag: &'static [u8],
+    reader: Lines<BufReader<ReadHalf<TcpStream>>>,
+    writer: WriteHalf<TcpStream>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Type {
+    Tagged,
+    Untagged,
+    Continuation,
+    Status,
+}
+
+impl ImapConnection {
+    pub async fn connect(tag: &'static [u8]) -> Self {
+        Self::connect_to(tag, "127.0.0.1:9991").await
+    }
+
+    pub async fn connect_to(tag: &'static [u8], addr: impl AsRef<str>) -> Self {
+        let (reader, writer) = tokio::io::split(TcpStream::connect(addr.as_ref()).await.unwrap());
+        ImapConnection {
+            tag,
+            reader: BufReader::new(reader).lines(),
+            writer,
+        }
+    }
+
+    pub async fn assert_read(&mut self, t: Type, rt: ResponseType) -> Vec<String> {
+        let lines = self.read(t).await;
+        let mut buf = Vec::with_capacity(10);
+        buf.extend_from_slice(match t {
+            Type::Tagged => self.tag,
+            Type::Untagged | Type::Status => b"* ",
+            Type::Continuation => b"+ ",
+        });
+        if !matches!(t, Type::Continuation | Type::Status) {
+            rt.serialize(&mut buf);
+        }
+        if lines
+            .last()
+            .unwrap()
+            .starts_with(&String::from_utf8(buf).unwrap())
+        {
+            lines
+        } else {
+            panic!("Expected {:?}/{:?} from server but got: {:?}", t, rt, lines);
+        }
+    }
+
+    pub async fn assert_disconnect(&mut self) {
+        match tokio::time::timeout(Duration::from_millis(1500), self.reader.next_line()).await {
+            Ok(Ok(None)) => {}
+            Ok(Ok(Some(line))) => {
+                panic!("Expected connection to be closed, but got {:?}", line);
+            }
+            Ok(Err(err)) => {
+                panic!("Connection broken: {:?}", err);
+            }
+            Err(_) => panic!("Timeout while waiting for server response."),
+        }
+    }
+
+    pub async fn read(&mut self, t: Type) -> Vec<String> {
+        let mut lines = Vec::new();
+        loop {
+            match tokio::time::timeout(Duration::from_millis(1500), self.reader.next_line()).await {
+                Ok(Ok(Some(line))) => {
+                    let is_done = line.starts_with(match t {
+                        Type::Tagged => std::str::from_utf8(self.tag).unwrap(),
+                        Type::Untagged | Type::Status => "* ",
+                        Type::Continuation => "+ ",
+                    });
+                    //let c = println!("<- {:?}", line);
+                    lines.push(line);
+                    if is_done {
+                        return lines;
+                    }
+                }
+                Ok(Ok(None)) => {
+                    panic!("Invalid response: {:?}.", lines);
+                }
+                Ok(Err(err)) => {
+                    panic!("Connection broken: {} ({:?})", err, lines);
+                }
+                Err(_) => panic!("Timeout while waiting for server response: {:?}", lines),
+            }
+        }
+    }
+
+    pub async fn authenticate(&mut self, user: &str, pass: &str) {
+        let creds = general_purpose::STANDARD.encode(format!("\0{user}\0{pass}"));
+        self.send(&format!(
+            "AUTHENTICATE PLAIN {{{}+}}\r\n{creds}",
+            creds.len()
+        ))
+        .await;
+        self.assert_read(Type::Tagged, ResponseType::Ok).await;
+    }
+
+    pub async fn send(&mut self, text: &str) {
+        //let c = println!("-> {}{:?}", std::str::from_utf8(self.tag).unwrap(), text);
+        self.writer.write_all(self.tag).await.unwrap();
+        self.writer.write_all(text.as_bytes()).await.unwrap();
+        self.writer.write_all(b"\r\n").await.unwrap();
+    }
+
+    pub async fn send_untagged(&mut self, text: &str) {
+        //let c = println!("-> {:?}", text);
+        self.writer.write_all(text.as_bytes()).await.unwrap();
+        self.writer.write_all(b"\r\n").await.unwrap();
+    }
+
+    pub async fn send_raw(&mut self, text: &str) {
+        //let c = println!("-> {:?}", text);
+        self.writer.write_all(text.as_bytes()).await.unwrap();
+    }
+
+    pub async fn append(&mut self, mailbox: &str, message: &str) {
+        self.send_ok(&format!(
+            "APPEND {:?} {{{}+}}\r\n{}",
+            mailbox,
+            message.len(),
+            message
+        ))
+        .await;
+    }
+
+    pub async fn send_ok(&mut self, cmd: &str) {
+        self.send(cmd).await;
+        self.assert_read(Type::Tagged, ResponseType::Ok).await;
+    }
+}

tests/src/utils/jmap.rs

@@ -616,6 +616,12 @@ pub trait JmapUtils {
         self.text_field("id")
     }
 
+    fn object_id(&self) -> Id {
+        self.id()
+            .parse()
+            .unwrap_or_else(|_| panic!("Invalid id {} in object", self.id()))
+    }
+
     fn blob_id(&self) -> &str {
         self.text_field("blobId")
     }

tests/src/utils/mod.rs

@@ -6,7 +6,11 @@
 
 pub mod account;
 pub mod cleanup;
+pub mod http;
+pub mod imap;
 pub mod jmap;
+pub mod pop3;
 pub mod registry;
 pub mod server;
+pub mod smtp;
 pub mod storage;

tests/src/utils/pop3.rs

@@ -0,0 +1,102 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use mail_send::smtp::tls::build_tls_connector;
+use rustls_pki_types::ServerName;
+use std::time::Duration;
+use tokio::{
+    io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines, ReadHalf, WriteHalf},
+    net::TcpStream,
+};
+use tokio_rustls::client::TlsStream;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum ResponseType {
+    Ok,
+    Multiline,
+    Err,
+}
+
+pub struct Pop3Connection {
+    reader: Lines<BufReader<ReadHalf<TlsStream<TcpStream>>>>,
+    writer: WriteHalf<TlsStream<TcpStream>>,
+}
+
+impl Pop3Connection {
+    pub async fn connect() -> Self {
+        let (reader, writer) = tokio::io::split(
+            build_tls_connector(true)
+                .connect(
+                    ServerName::try_from("pop3.example.org").unwrap().to_owned(),
+                    TcpStream::connect("127.0.0.1:4110").await.unwrap(),
+                )
+                .await
+                .unwrap(),
+        );
+        Pop3Connection {
+            reader: BufReader::new(reader).lines(),
+            writer,
+        }
+    }
+
+    pub async fn connect_and_login() -> Self {
+        let mut pop3 = Self::connect().await;
+        pop3.assert_read(ResponseType::Ok).await;
+        pop3.send("AUTH PLAIN AHBvcHBlckBleGFtcGxlLmNvbQBzZWNyZXQ=")
+            .await;
+        pop3.assert_read(ResponseType::Ok).await;
+        pop3
+    }
+
+    pub async fn assert_read(&mut self, rt: ResponseType) -> Vec<String> {
+        let lines = self.read(matches!(rt, ResponseType::Multiline)).await;
+        if lines.last().unwrap().starts_with(match rt {
+            ResponseType::Ok => "+OK",
+            ResponseType::Multiline => ".",
+            ResponseType::Err => "-ERR",
+        }) {
+            lines
+        } else {
+            panic!("Expected {:?} from server but got: {:?}", rt, lines);
+        }
+    }
+
+    pub async fn read(&mut self, is_multiline: bool) -> Vec<String> {
+        let mut lines = Vec::new();
+        loop {
+            match tokio::time::timeout(Duration::from_millis(1500), self.reader.next_line()).await {
+                Ok(Ok(Some(line))) => {
+                    let is_done = (!is_multiline && line.starts_with("+OK"))
+                        || (is_multiline && line == ".")
+                        || line.starts_with("-ERR");
+                    //let c = println!("<- {:?}", line);
+                    lines.push(line);
+                    if is_done {
+                        return lines;
+                    }
+                }
+                Ok(Ok(None)) => {
+                    panic!("Invalid response: {:?}.", lines);
+                }
+                Ok(Err(err)) => {
+                    panic!("Connection broken: {} ({:?})", err, lines);
+                }
+                Err(_) => panic!("Timeout while waiting for server response: {:?}", lines),
+            }
+        }
+    }
+
+    pub async fn send(&mut self, text: &str) {
+        //let c = println!("-> {:?}", text);
+        self.writer.write_all(text.as_bytes()).await.unwrap();
+        self.writer.write_all(b"\r\n").await.unwrap();
+    }
+
+    pub async fn send_raw(&mut self, text: &str) {
+        //let c = println!("-> {:?}", text);
+        self.writer.write_all(text.as_bytes()).await.unwrap();
+    }
+}

tests/src/utils/registry.rs

@@ -9,7 +9,10 @@ use crate::utils::{
     jmap::{JmapResponse, JmapSetError},
 };
 use registry::{
-    schema::prelude::ObjectType,
+    schema::{
+        prelude::{ObjectType, Property},
+        structs::Action,
+    },
     types::{EnumImpl, ObjectImpl},
 };
 use serde_json::{Value, json};
@@ -125,6 +128,29 @@ impl Account {
             .updated_id(id);
     }
 
+    pub async fn registry_update_setting<T: ObjectImpl>(
+        &self,
+        setting: T,
+        properties: &[Property],
+    ) {
+        let mut item = serde_json::to_value(setting).expect("Failed to serialize setting to JSON");
+
+        if !properties.is_empty() {
+            // Only include the specified properties in the update
+            if let Value::Object(obj) = &mut item {
+                obj.retain(|k, _| properties.iter().any(|p| p.as_str() == k));
+            }
+        }
+
+        self.registry_update(T::OBJECT, [(Id::singleton(), item)])
+            .await
+            .updated_id(Id::singleton());
+    }
+
+    pub async fn reload_settings(&self) {
+        self.registry_create_object(Action::ReloadSettings).await;
+    }
+
     pub async fn registry_update_object_expect_err(
         &self,
         object: ObjectType,
@@ -138,6 +164,19 @@ impl Account {
             .to_string();
         serde_json::from_str(&v).expect("Failed to deserialize set error")
     }
+
+    pub async fn registry_destroy_object_expect_err(
+        &self,
+        object: ObjectType,
+        id: Id,
+    ) -> JmapSetError {
+        let v = self
+            .registry_destroy(object, [id])
+            .await
+            .not_destroyed(&id.to_string())
+            .to_string();
+        serde_json::from_str(&v).expect("Failed to deserialize set error")
+    }
 }
 
 impl JmapResponse {

tests/src/utils/server.rs

@@ -103,7 +103,7 @@ impl TestServerBuilder {
             (NetworkListenerProtocol::Imap, "imaptls", 9992, true),
             (NetworkListenerProtocol::ManageSieve, "sieve", 4190, true),
             (NetworkListenerProtocol::Pop3, "pop3", 4110, true),
-            (NetworkListenerProtocol::Lmtp, "lmtp-debug", 11201, false),
+            (NetworkListenerProtocol::Lmtp, "lmtp-debug", 11200, false),
         ] {
             this = this.with_listener(protocol, name, port, use_tls).await;
         }

tests/src/utils/smtp.rs

@@ -0,0 +1,192 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::time::Duration;
+
+use tokio::{
+    io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines, ReadHalf, WriteHalf},
+    net::TcpStream,
+};
+
+pub struct SmtpConnection {
+    reader: Lines<BufReader<ReadHalf<TcpStream>>>,
+    writer: WriteHalf<TcpStream>,
+}
+
+impl SmtpConnection {
+    pub async fn ingest_with_code(
+        &mut self,
+        from: &str,
+        recipients: &[&str],
+        message: &str,
+        code: u8,
+    ) -> Vec<String> {
+        self.mail_from(from, 2).await;
+        for recipient in recipients {
+            self.rcpt_to(recipient, 2).await;
+        }
+        self.data(3).await;
+        let result = self.data_bytes(message, recipients.len(), code).await;
+        tokio::time::sleep(Duration::from_millis(500)).await;
+        result
+    }
+
+    pub async fn ingest(&mut self, from: &str, recipients: &[&str], message: &str) {
+        self.ingest_with_code(from, recipients, message, 2).await;
+    }
+
+    pub async fn ingest_chunked(
+        &mut self,
+        from: &str,
+        recipients: &[&str],
+        message: &str,
+        chunk_size: usize,
+    ) {
+        self.mail_from(from, 2).await;
+        for recipient in recipients {
+            self.rcpt_to(recipient, 2).await;
+        }
+        for chunk in message.as_bytes().chunks(chunk_size) {
+            self.bdat(std::str::from_utf8(chunk).unwrap(), 2).await;
+        }
+        self.bdat_last("", recipients.len(), 2).await;
+        tokio::time::sleep(Duration::from_millis(500)).await;
+    }
+
+    pub async fn connect() -> Self {
+        SmtpConnection::connect_port(11200).await
+    }
+
+    pub async fn connect_port(port: u16) -> Self {
+        let (reader, writer) = tokio::io::split(
+            TcpStream::connect(&format!("127.0.0.1:{port}"))
+                .await
+                .unwrap(),
+        );
+        let mut conn = SmtpConnection {
+            reader: BufReader::new(reader).lines(),
+            writer,
+        };
+        conn.read(1, 2).await;
+        conn.lhlo().await;
+        conn
+    }
+
+    pub async fn lhlo(&mut self) -> Vec<String> {
+        self.send("LHLO localhost").await;
+        self.read(1, 2).await
+    }
+
+    pub async fn mail_from(&mut self, sender: &str, code: u8) -> Vec<String> {
+        self.send(&format!("MAIL FROM:<{}>", sender)).await;
+        self.read(1, code).await
+    }
+
+    pub async fn rcpt_to(&mut self, rcpt: &str, code: u8) -> Vec<String> {
+        self.send(&format!("RCPT TO:<{}>", rcpt)).await;
+        self.read(1, code).await
+    }
+
+    pub async fn vrfy(&mut self, rcpt: &str, code: u8) -> Vec<String> {
+        self.send(&format!("VRFY {}", rcpt)).await;
+        self.read(1, code).await
+    }
+
+    pub async fn expn(&mut self, rcpt: &str, code: u8) -> Vec<String> {
+        self.send(&format!("EXPN {}", rcpt)).await;
+        self.read(1, code).await
+    }
+
+    pub async fn data(&mut self, code: u8) -> Vec<String> {
+        self.send("DATA").await;
+        self.read(1, code).await
+    }
+
+    pub async fn data_bytes(
+        &mut self,
+        message: &str,
+        num_responses: usize,
+        code: u8,
+    ) -> Vec<String> {
+        self.send_raw(message).await;
+        self.send_raw("\r\n.\r\n").await;
+        self.read(num_responses, code).await
+    }
+
+    pub async fn bdat(&mut self, chunk: &str, code: u8) -> Vec<String> {
+        self.send_raw(&format!("BDAT {}\r\n{}", chunk.len(), chunk))
+            .await;
+        self.read(1, code).await
+    }
+
+    pub async fn bdat_last(&mut self, chunk: &str, num_responses: usize, code: u8) -> Vec<String> {
+        self.send_raw(&format!("BDAT {} LAST\r\n{}", chunk.len(), chunk))
+            .await;
+        self.read(num_responses, code).await
+    }
+
+    pub async fn rset(&mut self) -> Vec<String> {
+        self.send("RSET").await;
+        self.read(1, 2).await
+    }
+
+    pub async fn noop(&mut self) -> Vec<String> {
+        self.send("NOOP").await;
+        self.read(1, 2).await
+    }
+
+    pub async fn quit(&mut self) -> Vec<String> {
+        self.send("QUIT").await;
+        self.read(1, 2).await
+    }
+
+    pub async fn read(&mut self, mut num_responses: usize, code: u8) -> Vec<String> {
+        let mut lines = Vec::new();
+        loop {
+            match tokio::time::timeout(Duration::from_millis(1500), self.reader.next_line()).await {
+                Ok(Ok(Some(line))) => {
+                    let is_done = line.as_bytes()[3] == b' ';
+                    //let c = println!("<- {:?}", line);
+                    lines.push(line);
+                    if is_done {
+                        num_responses -= 1;
+                        if num_responses != 0 {
+                            continue;
+                        }
+
+                        if code != u8::MAX {
+                            for line in &lines {
+                                if line.as_bytes()[0] - b'0' != code {
+                                    panic!("Expected completion code {}, got {:?}.", code, lines);
+                                }
+                            }
+                        }
+                        return lines;
+                    }
+                }
+                Ok(Ok(None)) => {
+                    panic!("Invalid response: {:?}.", lines);
+                }
+                Ok(Err(err)) => {
+                    panic!("Connection broken: {} ({:?})", err, lines);
+                }
+                Err(_) => panic!("Timeout while waiting for server response: {:?}", lines),
+            }
+        }
+    }
+
+    pub async fn send(&mut self, text: &str) {
+        //let c = println!("-> {:?}", text);
+        self.writer.write_all(text.as_bytes()).await.unwrap();
+        self.writer.write_all(b"\r\n").await.unwrap();
+        self.writer.flush().await.unwrap();
+    }
+
+    pub async fn send_raw(&mut self, text: &str) {
+        //let c = println!("-> {:?}", text);
+        self.writer.write_all(text.as_bytes()).await.unwrap();
+    }
+}

tests/src/utils/storage.rs

@@ -21,6 +21,7 @@ use registry::{
     },
     types::{EnumImpl, duration::Duration},
 };
+use store::U64_LEN;
 use store::{
     Deserialize, IterateParams, ValueKey,
     write::{TaskQueueClass, ValueClass},
@@ -167,8 +168,10 @@ pub async fn wait_for_index(server: &Server) {
                     ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Task { id: u64::MAX })),
                 )
                 .ascending(),
-                |_, value| {
-                    has_index_tasks = Some(Task::deserialize(value)?);
+                |key, value| {
+                    if key.len() == U64_LEN {
+                        has_index_tasks = Some(Task::deserialize(value)?);
+                    }
 
                     Ok(false)
                 },