diff --git a/Cargo.lock b/Cargo.lock index d52747ca..3761885c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2548,7 +2548,7 @@ checksum = "029d73f573d8e8d63e6d5020011d3255b28c3ba85d6cf870a07184ed23de9284" [[package]] name = "imap" -version = "0.4.2" +version = "0.5.0" dependencies = [ "ahash 0.8.6", "dashmap", @@ -2725,7 +2725,7 @@ dependencies = [ [[package]] name = "jmap" -version = "0.4.2" +version = "0.5.0" dependencies = [ "aes", "aes-gcm", @@ -3129,7 +3129,7 @@ dependencies = [ [[package]] name = "mail-server" -version = "0.4.2" +version = "0.5.0" dependencies = [ "directory", "imap", @@ -3146,7 +3146,7 @@ dependencies = [ [[package]] name = "managesieve" -version = "0.4.2" +version = "0.5.0" dependencies = [ "ahash 0.8.6", "bincode", @@ -3422,7 +3422,7 @@ dependencies = [ [[package]] name = "nlp" -version = "0.4.2" +version = "0.5.0" dependencies = [ "ahash 0.8.6", "bincode", @@ -5361,7 +5361,7 @@ checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" [[package]] name = "smtp" -version = "0.4.2" +version = "0.5.0" dependencies = [ "ahash 0.8.6", "bincode", @@ -5482,7 +5482,7 @@ dependencies = [ [[package]] name = "stalwart-cli" -version = "0.4.2" +version = "0.5.0" dependencies = [ "clap", "console", @@ -5506,7 +5506,7 @@ dependencies = [ [[package]] name = "stalwart-install" -version = "0.4.2" +version = "0.5.0" dependencies = [ "base64 0.21.5", "clap", @@ -5519,7 +5519,6 @@ dependencies = [ "rand", "reqwest", "rpassword", - "rusqlite", "tar", "zip-extract", ] @@ -6419,7 +6418,7 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "utils" -version = "0.4.2" +version = "0.5.0" dependencies = [ "ahash 0.8.6", "chrono", diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index daa64ae8..c0fb83f1 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Stalwart Labs Ltd. "] license = "AGPL-3.0-only" repository = "https://github.com/stalwartlabs/cli" homepage = "https://github.com/stalwartlabs/cli" -version = "0.4.2" +version = "0.5.0" edition = "2021" readme = "README.md" resolver = "2" diff --git a/crates/imap/Cargo.toml b/crates/imap/Cargo.toml index 79305da7..01c7b1ef 100644 --- a/crates/imap/Cargo.toml +++ b/crates/imap/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "imap" -version = "0.4.2" +version = "0.5.0" edition = "2021" resolver = "2" diff --git a/crates/imap/src/core/client.rs b/crates/imap/src/core/client.rs index e7fec781..9fbeb4ef 100644 --- a/crates/imap/src/core/client.rs +++ b/crates/imap/src/core/client.rs @@ -36,9 +36,9 @@ use super::{SelectedMailbox, Session, SessionData, State, IMAP}; impl Session { pub async fn ingest(&mut self, bytes: &[u8]) -> crate::Result { - for line in String::from_utf8_lossy(bytes).split("\r\n") { + /*for line in String::from_utf8_lossy(bytes).split("\r\n") { let c = println!("{}", line); - } + }*/ tracing::trace!(parent: &self.span, event = "read", diff --git a/crates/imap/src/core/writer.rs b/crates/imap/src/core/writer.rs index e61df02d..68ac1404 100644 --- a/crates/imap/src/core/writer.rs +++ b/crates/imap/src/core/writer.rs @@ -58,7 +58,7 @@ pub fn spawn_writer(mut stream: Event, span: tracing::Span) -> mpsc::Sender { @@ -101,7 +101,7 @@ pub fn spawn_writer(mut stream: Event, span: tracing::Span) -> mpsc::Sender { @@ -131,9 +131,9 @@ pub fn spawn_writer(mut stream: Event, span: tracing::Span) -> mpsc::Sender Session { pub async fn write_bytes(&self, bytes: impl Into>) -> crate::OpResult { let bytes = bytes.into(); - for line in String::from_utf8_lossy(bytes.as_ref()).split("\r\n") { + /*for line in String::from_utf8_lossy(bytes.as_ref()).split("\r\n") { let c = println!("{}", line); - } + }*/ if let Err(err) = self.writer.send(Event::Bytes(bytes)).await { debug!("Failed to send bytes: {}", err); @@ -147,9 +147,9 @@ impl Session { impl SessionData { pub async fn write_bytes(&self, bytes: impl Into>) -> bool { let bytes = bytes.into(); - for line in String::from_utf8_lossy(bytes.as_ref()).split("\r\n") { + /*for line in String::from_utf8_lossy(bytes.as_ref()).split("\r\n") { let c = println!("{}", line); - } + }*/ if let Err(err) = self.writer.send(Event::Bytes(bytes)).await { debug!("Failed to send bytes: {}", err); diff --git a/crates/install/Cargo.toml b/crates/install/Cargo.toml index e8c2e0a9..22f79a8f 100644 --- a/crates/install/Cargo.toml +++ b/crates/install/Cargo.toml @@ -5,14 +5,13 @@ authors = ["Stalwart Labs Ltd. "] license = "AGPL-3.0-only" repository = "https://github.com/stalwartlabs/mail-server" homepage = "https://github.com/stalwartlabs/mail-server" -version = "0.4.2" +version = "0.5.0" edition = "2021" readme = "README.md" resolver = "2" [dependencies] reqwest = { version = "0.11", default-features = false, features = ["rustls-tls-webpki-roots", "blocking"] } -rusqlite = { version = "0.30.0", features = ["bundled"] } rpassword = "7.0" indicatif = "0.17.0" dialoguer = "0.11" diff --git a/crates/install/src/main.rs b/crates/install/src/main.rs index 48557674..445af062 100644 --- a/crates/install/src/main.rs +++ b/crates/install/src/main.rs @@ -22,6 +22,7 @@ */ use std::{ + fmt::{Display, Formatter}, fs, io::Cursor, path::{Path, PathBuf}, @@ -32,9 +33,7 @@ use base64::{engine::general_purpose, Engine}; use clap::{Parser, ValueEnum}; use dialoguer::{console::Term, theme::ColorfulTheme, Input, Select}; use openssl::rsa::Rsa; -use pwhash::sha512_crypt; use rand::{distributions::Alphanumeric, thread_rng, Rng}; -use rusqlite::{Connection, OpenFlags}; const CONFIG_URL: &str = "https://get.stalw.art/resources/config.zip"; @@ -65,30 +64,47 @@ enum Component { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Backend { - SQLite, +enum Store { + RocksDB, FoundationDB, + SQLite, + PostgreSQL, + MySQL, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Blob { - Local, - MinIO, + Internal, + Filesystem, S3, - Gcs, - Azure, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Fts { + Internal, + ElasticSearch, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SpamDb { + Internal, + Redis, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Directory { - Sql, + Internal, Ldap, - None, + PostgreSQL, + MySQL, + SQLite, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum SmtpDirectory { - Sql, + PostgreSQL, + MySQL, + SQLite, Ldap, Lmtp, Imap, @@ -173,17 +189,20 @@ fn main() -> std::io::Result<()> { let mut download_url = None; // Obtain database engine - let directory = if component != Component::Smtp { - if !skip_download { - let backend = select::( - "Which database engine would you like to use?", - &[ - "SQLite (single node, replicated with Litestream)", - "FoundationDB (distributed and fault-tolerant)", - ], - Backend::SQLite, - )?; + if component != Component::Smtp { + let backend = select::( + "Which database would you like to use?", + &[ + "RocksDB (recommended for single-node setups)", + "FoundationDB (recommended for distributed environments)", + "SQLite", + "PostgreSQL", + "MySQL", + ], + Store::RocksDB, + )?; + if !skip_download { download_url = format!( concat!( "https://github.com/stalwartlabs/{}", @@ -202,46 +221,70 @@ fn main() -> std::io::Result<()> { Component::Smtp => unreachable!(), }, match backend { - Backend::SQLite => "sqlite", - Backend::FoundationDB => "foundationdb", + Store::FoundationDB => "distributed", + _ => "local", }, TARGET, PKG_EXTENSION ) .into(); } + let store = backend.to_string(); let blob = select::( - "Where would you like to store e-mails and blobs?", + "Where would you like to store e-mails and other large binaries?", &[ - "Local disk using Maildir", - "MinIO (or any S3-compatible object storage)", - "Amazon S3", - "Google Cloud Storage", - "Azure Blob Storage", + &store, + "Local file system", + "S3, MinIO or any S3-compatible object storage", ], - Blob::Local, + Blob::Internal, )?; let directory = select::( - "Do you already have a directory or database containing your accounts?", + "Do you already have a directory or database containing your user accounts?", &[ - "Yes, it's an SQL database", - "Yes, it's an LDAP directory", - "No, create a new directory for me", + &format!("No, I want Stalwart to manage my user accounts in {store}"), + "Yes, it's an LDAP server", + "Yes, it's an PostgreSQL database", + "Yes, it's an MySQL database", + "Yes, it's an SQLite database", ], - Directory::None, + Directory::Internal, + )?; + + let fts = select::( + "Where would you like to store the full-text index?", + &[&store, "ElasticSearch"], + Fts::Internal, + )?; + + let spamdb = select::( + "Where would you like to store the anti-spam database?", + &[&store, "Redis"], + SpamDb::Internal, )?; // Update settings - if blob != Blob::Local { + sed( + cfg_path.join("config.toml"), + &[ + ("__STORE__", backend.id()), + ("__DIRECTORY__", directory.id()), + ], + ); + sed( + cfg_path.join("jmap").join("store.toml"), + &[ + ("__BLOB_STORE__", blob.id().unwrap_or("%{DEFAULT_STORE}%")), + ("__FTS_STORE__", fts.id().unwrap_or("%{DEFAULT_STORE}%")), + ], + ); + if let Some(id) = spamdb.id() { sed( - cfg_path.join("jmap").join("store.toml"), - &[("\"local\"", "\"s3\"")], + cfg_path.join("common").join("sieve.toml"), + &[("%{DEFAULT_STORE}%", id)], ); } - if directory == Directory::Ldap { - sed(cfg_path.join("config.toml"), &[("/sql.toml", "/ldap.toml")]); - } sed( cfg_path.join("jmap").join("oauth.toml"), &[( @@ -254,12 +297,47 @@ fn main() -> std::io::Result<()> { )], ); - directory + // Enable stores + for store in [ + backend.id().into(), + blob.id(), + fts.id(), + spamdb.id(), + directory.sql_store_id(), + ] + .into_iter() + .flatten() + { + sed( + cfg_path.join("store").join(format!("{store}.toml")), + &[("disable = true", "disable = false")], + ); + } + + // Enable directory + if let Some(sql_id) = directory.sql_store_id() { + sed( + cfg_path.join("directory").join("sql.toml"), + &[ + ("disable = true", "disable = false"), + ("__SQL_STORE__", sql_id), + ], + ); + } else { + sed( + cfg_path + .join("directory") + .join(format!("{}.toml", directory.id())), + &[("disable = true", "disable = false")], + ); + } } else { let smtp_directory = select::( "How should your local accounts be validated?", &[ - "SQL database", + "PostgreSQL database", + "MySQL database", + "SQLite database", "LDAP directory", "LMTP server", "IMAP server", @@ -267,34 +345,61 @@ fn main() -> std::io::Result<()> { SmtpDirectory::Lmtp, )?; - if smtp_directory == SmtpDirectory::Ldap { - sed(cfg_path.join("config.toml"), &[("/sql.toml", "/ldap.toml")]); + let spamdb = select::( + "Where would you like to store the anti-spam database?", + &["Local database", "Redis"], + SpamDb::Internal, + )?; + + // Update settings + sed( + cfg_path.join("config.toml"), + &[ + ("__STORE__", "rocksdb"), + ("__DIRECTORY__", smtp_directory.id()), + ], + ); + sed( + cfg_path.join("jmap").join("store.toml"), + &[ + ("__BLOB_STORE__", "%{DEFAULT_STORE}%"), + ("__FTS_STORE__", "%{DEFAULT_STORE}%"), + ], + ); + if let Some(id) = spamdb.id() { + sed( + cfg_path.join("common").join("sieve.toml"), + &[("%{DEFAULT_STORE}%", id)], + ); } - match smtp_directory { - SmtpDirectory::Ldap => { - sed(cfg_path.join("config.toml"), &[("/sql.toml", "/ldap.toml")]); - } - SmtpDirectory::Lmtp | SmtpDirectory::Imap => { - let d_type = if smtp_directory == SmtpDirectory::Lmtp { - "lmtp" - } else { - "imap" - }; - sed( - cfg_path.join("smtp").join("queue.toml"), - &[("default", d_type)], - ); - sed( - cfg_path.join("smtp").join("session.toml"), - &[("default", d_type)], - ); - sed( - cfg_path.join("config.toml"), - &[("\"%{BASE_PATH}%/etc/directory/", format!("\"%{{BASE_PATH}}%/etc/directory/{d_type}.toml\",\n\t\"%{{BASE_PATH}}%/etc/directory/"))], - ); - } - SmtpDirectory::Sql => (), + // Enable directory + if let Some(sql_id) = smtp_directory.sql_store_id() { + sed( + cfg_path.join("directory").join("sql.toml"), + &[ + ("disable = true", "disable = false"), + ("__SQL_STORE__", sql_id), + ], + ); + } else { + sed( + cfg_path + .join("directory") + .join(format!("{}.toml", smtp_directory.id())), + &[("disable = true", "disable = false")], + ); + } + + // Enable stores + for store in [smtp_directory.sql_store_id(), spamdb.id(), "rocksdb".into()] + .into_iter() + .flatten() + { + sed( + cfg_path.join("store").join(format!("{store}.toml")), + &[("disable = true", "disable = false")], + ); } if !skip_download { @@ -307,12 +412,7 @@ fn main() -> std::io::Result<()> { ) .into(); } - match smtp_directory { - SmtpDirectory::Sql => Directory::Sql, - SmtpDirectory::Ldap => Directory::Ldap, - SmtpDirectory::Lmtp | SmtpDirectory::Imap => Directory::None, - } - }; + } // Download binary if let Some(download_url) = download_url { @@ -401,16 +501,6 @@ fn main() -> std::io::Result<()> { // Generate DKIM key and instructions let dkim_instructions = generate_dkim(&base_path, &domain, &hostname)?; - // Create authentication and spam filter SQLite databases - let admin_password = create_databases( - &base_path, - if matches!(directory, Directory::None) { - Some(&domain) - } else { - None - }, - )?; - // Update config file if args.docker { sed( @@ -440,6 +530,7 @@ fn main() -> std::io::Result<()> { ); // Write service file + if !args.docker { // Change permissions #[cfg(not(target_env = "msvc"))] @@ -542,10 +633,6 @@ fn main() -> std::io::Result<()> { eprintln!("\nšŸŽ‰ Installation completed!\n\nāœ… {dkim_instructions}\n"); - if let Some(admin_password) = admin_password { - eprintln!("šŸ”‘ The administrator account is 'admin' with password '{admin_password}'.\n",); - } - Ok(()) } @@ -690,82 +777,6 @@ fn create_directories(path: &Path) -> std::io::Result<()> { Ok(()) } -fn create_databases(base_path: &Path, domain: Option<&str>) -> std::io::Result> { - // Create Spam database - let path = PathBuf::from(base_path) - .join("data") - .join("spamfilter.sqlite3"); - let conn = Connection::open_with_flags(path, OpenFlags::default()).map_err(|err| { - std::io::Error::new( - std::io::ErrorKind::Other, - format!("Failed to open database: {}", err), - ) - })?; - for query in [ - concat!( - "CREATE TABLE IF NOT EXISTS bayes_tokens (h1 INTEGER NOT NULL, ", - "h2 INTEGER NOT NULL, ws INTEGER, wh INTEGER, PRIMARY KEY (h1, h2))", - ), - concat!( - "CREATE TABLE IF NOT EXISTS seen_ids (id STRING NOT NULL PRIMARY KEY", - ", ttl DATETIME NOT NULL)", - ), - concat!( - "CREATE TABLE IF NOT EXISTS reputation (token STRING NOT NULL PRIMARY KEY", - ", score FLOAT NOT NULL DEFAULT '0', count INT(11) NOT NULL ", - "DEFAULT '0', ttl DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP)", - ), - ] { - conn.execute(query, []).map_err(|err| { - std::io::Error::new( - std::io::ErrorKind::Other, - format!("Failed to create database: {}", err), - ) - })?; - } - - if let Some(domain) = domain { - // Create accounts database - let mut path = PathBuf::from(base_path); - path.push("data"); - if !path.exists() { - fs::create_dir_all(&path)?; - } - path.push("accounts.sqlite3"); - - let conn = Connection::open_with_flags(path, OpenFlags::default()).map_err(|err| { - std::io::Error::new( - std::io::ErrorKind::Other, - format!("Failed to open database: {}", err), - ) - })?; - let secret = thread_rng() - .sample_iter(Alphanumeric) - .take(12) - .map(char::from) - .collect::(); - let hashed_secret = sha512_crypt::hash(&secret).unwrap(); - for query in [ - concat!("CREATE TABLE IF NOT EXISTS accounts (name TEXT PRIMARY KEY, secret TEXT, description TEXT, ","type TEXT NOT NULL, quota INTEGER DEFAULT 0, active BOOLEAN DEFAULT 1)").to_string(), - concat!("CREATE TABLE IF NOT EXISTS group_members (name TEXT NOT NULL, member_of ","TEXT NOT NULL, PRIMARY KEY (name, member_of))").to_string(), - concat!("CREATE TABLE IF NOT EXISTS emails (name TEXT NOT NULL, address TEXT NOT NULL",", type TEXT, PRIMARY KEY (name, address))").to_string(), - format!("INSERT OR REPLACE INTO accounts (name, secret, description, type) VALUES ('admin', '{hashed_secret}', 'Postmaster', 'admin')"), - format!("INSERT OR REPLACE INTO emails (name, address, type) VALUES ('admin', 'postmaster@{domain}', 'primary')"), - "INSERT OR IGNORE INTO group_members (name, member_of) VALUES ('admin', 'superusers')".to_string() - ] { - conn.execute(&query, []).map_err(|err| { - std::io::Error::new( - std::io::ErrorKind::Other, - format!("Failed to create database: {}", err), - ) - })?; - } - Ok(Some(secret)) - } else { - Ok(None) - } -} - fn generate_dkim(path: &Path, domain: &str, hostname: &str) -> std::io::Result { let mut path = PathBuf::from(path); path.push("etc"); @@ -857,19 +868,25 @@ impl SelectItem for Component { } } -impl SelectItem for Backend { +impl SelectItem for Store { fn from_index(index: usize) -> Self { match index { - 0 => Self::SQLite, + 0 => Self::RocksDB, 1 => Self::FoundationDB, + 2 => Self::SQLite, + 3 => Self::PostgreSQL, + 4 => Self::MySQL, _ => unreachable!(), } } fn to_index(&self) -> usize { match self { - Self::SQLite => 0, - Self::FoundationDB => 1, + Store::RocksDB => 0, + Store::FoundationDB => 1, + Store::SQLite => 2, + Store::PostgreSQL => 3, + Store::MySQL => 4, } } } @@ -877,18 +894,22 @@ impl SelectItem for Backend { impl SelectItem for Directory { fn from_index(index: usize) -> Self { match index { - 0 => Self::Sql, + 0 => Self::Internal, 1 => Self::Ldap, - 2 => Self::None, + 2 => Self::PostgreSQL, + 3 => Self::MySQL, + 4 => Self::SQLite, _ => unreachable!(), } } fn to_index(&self) -> usize { match self { - Self::Sql => 0, - Self::Ldap => 1, - Self::None => 2, + Directory::Internal => 0, + Directory::Ldap => 1, + Directory::PostgreSQL => 2, + Directory::MySQL => 3, + Directory::SQLite => 4, } } } @@ -896,20 +917,24 @@ impl SelectItem for Directory { impl SelectItem for SmtpDirectory { fn from_index(index: usize) -> Self { match index { - 0 => Self::Sql, - 1 => Self::Ldap, - 2 => Self::Lmtp, - 3 => Self::Imap, + 0 => Self::PostgreSQL, + 1 => Self::MySQL, + 2 => Self::SQLite, + 3 => Self::Ldap, + 4 => Self::Lmtp, + 5 => Self::Imap, _ => unreachable!(), } } fn to_index(&self) -> usize { match self { - SmtpDirectory::Sql => 0, - SmtpDirectory::Ldap => 1, - SmtpDirectory::Lmtp => 2, - SmtpDirectory::Imap => 3, + SmtpDirectory::PostgreSQL => 0, + SmtpDirectory::MySQL => 1, + SmtpDirectory::SQLite => 2, + SmtpDirectory::Ldap => 3, + SmtpDirectory::Lmtp => 4, + SmtpDirectory::Imap => 5, } } } @@ -917,22 +942,52 @@ impl SelectItem for SmtpDirectory { impl SelectItem for Blob { fn from_index(index: usize) -> Self { match index { - 0 => Blob::Local, - 1 => Blob::MinIO, - 2 => Blob::S3, - 3 => Blob::Gcs, - 4 => Blob::Azure, + 0 => Self::Internal, + 1 => Self::Filesystem, + 2 => Self::S3, _ => unreachable!(), } } fn to_index(&self) -> usize { match self { - Blob::Local => 0, - Blob::MinIO => 1, + Blob::Internal => 0, + Blob::Filesystem => 1, Blob::S3 => 2, - Blob::Gcs => 3, - Blob::Azure => 4, + } + } +} + +impl SelectItem for SpamDb { + fn from_index(index: usize) -> Self { + match index { + 0 => Self::Internal, + 1 => Self::Redis, + _ => unreachable!(), + } + } + + fn to_index(&self) -> usize { + match self { + SpamDb::Internal => 0, + SpamDb::Redis => 1, + } + } +} + +impl SelectItem for Fts { + fn from_index(index: usize) -> Self { + match index { + 0 => Self::Internal, + 1 => Self::ElasticSearch, + _ => unreachable!(), + } + } + + fn to_index(&self) -> usize { + match self { + Fts::Internal => 0, + Fts::ElasticSearch => 1, } } } @@ -973,3 +1028,94 @@ impl Component { } } } + +impl Display for Store { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::RocksDB => write!(f, "RocksDB"), + Self::FoundationDB => write!(f, "FoundationDB"), + Self::SQLite => write!(f, "SQLite"), + Self::PostgreSQL => write!(f, "PostgreSQL"), + Self::MySQL => write!(f, "MySQL"), + } + } +} + +impl Store { + pub fn id(&self) -> &'static str { + match self { + Self::RocksDB => "rocksdb", + Self::FoundationDB => "foundationdb", + Self::SQLite => "sqlite", + Self::PostgreSQL => "postgresql", + Self::MySQL => "mysql", + } + } +} + +impl Directory { + pub fn id(&self) -> &'static str { + match self { + Directory::Internal => "internal", + Directory::Ldap => "ldap", + Directory::PostgreSQL | Directory::MySQL | Directory::SQLite => "sql", + } + } + + pub fn sql_store_id(&self) -> Option<&'static str> { + match self { + Directory::PostgreSQL => Some("postgresql"), + Directory::MySQL => Some("mysql"), + Directory::SQLite => Some("sqlite"), + Directory::Internal | Directory::Ldap => None, + } + } +} + +impl SmtpDirectory { + pub fn id(&self) -> &'static str { + match self { + SmtpDirectory::Ldap => "ldap", + SmtpDirectory::Lmtp => "lmtp", + SmtpDirectory::Imap => "imap", + SmtpDirectory::PostgreSQL | SmtpDirectory::MySQL | SmtpDirectory::SQLite => "sql", + } + } + + pub fn sql_store_id(&self) -> Option<&'static str> { + match self { + SmtpDirectory::PostgreSQL => Some("postgresql"), + SmtpDirectory::MySQL => Some("mysql"), + SmtpDirectory::SQLite => Some("sqlite"), + SmtpDirectory::Ldap | SmtpDirectory::Lmtp | SmtpDirectory::Imap => None, + } + } +} + +impl Blob { + pub fn id(&self) -> Option<&'static str> { + match self { + Self::Internal => None, + Self::Filesystem => "fs".into(), + Self::S3 => "s3".into(), + } + } +} + +impl Fts { + pub fn id(&self) -> Option<&'static str> { + match self { + Self::Internal => None, + Self::ElasticSearch => "elasticsearch".into(), + } + } +} + +impl SpamDb { + pub fn id(&self) -> Option<&'static str> { + match self { + Self::Internal => None, + Self::Redis => "redis".into(), + } + } +} diff --git a/crates/jmap/Cargo.toml b/crates/jmap/Cargo.toml index 14c989c6..8e675a4f 100644 --- a/crates/jmap/Cargo.toml +++ b/crates/jmap/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jmap" -version = "0.4.2" +version = "0.5.0" edition = "2021" resolver = "2" diff --git a/crates/jmap/src/mailbox/query.rs b/crates/jmap/src/mailbox/query.rs index 915224db..f589d9b1 100644 --- a/crates/jmap/src/mailbox/query.rs +++ b/crates/jmap/src/mailbox/query.rs @@ -241,7 +241,6 @@ impl JMAP { stack.push((children, it)); } } - debug_assert_eq!(response.ids.len(), paginate.ids.len(), "{tree:#?}"); response.update_results(paginate.build())?; } else { response = self diff --git a/crates/main/Cargo.toml b/crates/main/Cargo.toml index 465f5ef4..7cd3447b 100644 --- a/crates/main/Cargo.toml +++ b/crates/main/Cargo.toml @@ -7,7 +7,7 @@ homepage = "https://stalw.art" keywords = ["imap", "jmap", "smtp", "email", "mail", "server"] categories = ["email"] license = "AGPL-3.0-only" -version = "0.4.2" +version = "0.5.0" edition = "2021" resolver = "2" diff --git a/crates/managesieve/Cargo.toml b/crates/managesieve/Cargo.toml index 7920e74f..f96ac895 100644 --- a/crates/managesieve/Cargo.toml +++ b/crates/managesieve/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "managesieve" -version = "0.4.2" +version = "0.5.0" edition = "2021" resolver = "2" diff --git a/crates/nlp/Cargo.toml b/crates/nlp/Cargo.toml index d40d75bc..3d70992f 100644 --- a/crates/nlp/Cargo.toml +++ b/crates/nlp/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nlp" -version = "0.4.2" +version = "0.5.0" edition = "2021" resolver = "2" diff --git a/crates/smtp/Cargo.toml b/crates/smtp/Cargo.toml index fb2d7f72..00d31c2a 100644 --- a/crates/smtp/Cargo.toml +++ b/crates/smtp/Cargo.toml @@ -7,7 +7,7 @@ homepage = "https://stalw.art/smtp" keywords = ["smtp", "email", "mail", "server"] categories = ["email"] license = "AGPL-3.0-only" -version = "0.4.2" +version = "0.5.0" edition = "2021" resolver = "2" diff --git a/crates/store/src/backend/foundationdb/read.rs b/crates/store/src/backend/foundationdb/read.rs index b6c65b22..65875763 100644 --- a/crates/store/src/backend/foundationdb/read.rs +++ b/crates/store/src/backend/foundationdb/read.rs @@ -35,7 +35,7 @@ use crate::{ key::{DeserializeBigEndian, KeySerializer}, BitmapClass, ValueClass, }, - BitmapKey, Deserialize, IterateParams, Key, ValueKey, U32_LEN, + BitmapKey, Deserialize, IterateParams, Key, ValueKey, U32_LEN, WITH_SUBSPACE, }; use super::{FdbStore, MAX_VALUE_SIZE}; @@ -59,7 +59,7 @@ impl FdbStore { where U: Deserialize, { - let key = key.serialize(true); + let key = key.serialize(WITH_SUBSPACE); let trx = self.db.create_trx()?; match read_chunked_value(&key, &trx, true).await? { @@ -75,7 +75,7 @@ impl FdbStore { ) -> crate::Result> { #[cfg(feature = "fdb-chunked-bm")] { - read_chunked_bitmap(&key.serialize(true), &self.db.create_trx()?, true) + read_chunked_bitmap(&key.serialize(WITH_SUBSPACE), &self.db.create_trx()?, true) .await .map(Into::into) } @@ -83,9 +83,9 @@ impl FdbStore { #[cfg(not(feature = "fdb-chunked-bm"))] { let mut bm = RoaringBitmap::new(); - let begin = key.serialize(true); + let begin = key.serialize(WITH_SUBSPACE); key.block_num = u32::MAX; - let end = key.serialize(true); + let end = key.serialize(WITH_SUBSPACE); let key_len = begin.len(); let trx = self.db.create_trx()?; let mut values = trx.get_ranges( @@ -119,8 +119,8 @@ impl FdbStore { params: IterateParams, mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> crate::Result + Sync + Send, ) -> crate::Result<()> { - let begin = params.begin.serialize(true); - let end = params.end.serialize(true); + let begin = params.begin.serialize(WITH_SUBSPACE); + let end = params.end.serialize(WITH_SUBSPACE); let trx = self.db.create_trx()?; let mut iter = trx.get_ranges( @@ -156,7 +156,7 @@ impl FdbStore { &self, key: impl Into> + Sync + Send, ) -> crate::Result { - let key = key.into().serialize(true); + let key = key.into().serialize(WITH_SUBSPACE); if let Some(bytes) = self.db.create_trx()?.get(&key, true).await? { Ok(i64::from_le_bytes(bytes[..].try_into().map_err(|_| { crate::Error::InternalError("Invalid counter value.".to_string()) diff --git a/crates/store/src/backend/foundationdb/write.rs b/crates/store/src/backend/foundationdb/write.rs index 1de4d4d7..2ea9a100 100644 --- a/crates/store/src/backend/foundationdb/write.rs +++ b/crates/store/src/backend/foundationdb/write.rs @@ -40,7 +40,7 @@ use crate::{ key::KeySerializer, Batch, BitmapClass, Operation, ValueClass, ValueOp, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, }, - BitmapKey, IndexKey, Key, LogKey, ValueKey, SUBSPACE_BITMAPS, SUBSPACE_VALUES, + BitmapKey, IndexKey, Key, LogKey, ValueKey, SUBSPACE_BITMAPS, SUBSPACE_VALUES, WITH_SUBSPACE, }; use super::{ @@ -112,7 +112,7 @@ impl FdbStore { document_id, class, } - .serialize(true); + .serialize(WITH_SUBSPACE); trx.atomic_op(&key, &by.to_le_bytes()[..], MutationType::Add); } @@ -123,7 +123,7 @@ impl FdbStore { document_id, class, } - .serialize(true); + .serialize(WITH_SUBSPACE); let do_chunk = key[0] == SUBSPACE_VALUES; if let ValueOp::Set(value) = op { @@ -161,7 +161,7 @@ impl FdbStore { class: BitmapClass::DocumentIds, block_num, } - .serialize(true), + .serialize(WITH_SUBSPACE), true, ) .await @@ -192,7 +192,7 @@ impl FdbStore { field: *field, key, } - .serialize(true); + .serialize(WITH_SUBSPACE); if *set { trx.set(&key, &[]); @@ -215,7 +215,7 @@ impl FdbStore { class, block_num: DenseBitmap::block_num(document_id), } - .serialize(true), + .serialize(WITH_SUBSPACE), ) .or_insert_with(DenseBitmap::empty) .set(document_id); @@ -229,7 +229,7 @@ impl FdbStore { class, block_num: 0, } - .serialize(true), + .serialize(WITH_SUBSPACE), ) .or_insert(Vec::new()) .push(BitmapOp::new(document_id, *set)); @@ -245,7 +245,7 @@ impl FdbStore { collection: *collection, change_id: *change_id, } - .serialize(true); + .serialize(WITH_SUBSPACE); trx.set(&key, set); } Operation::AssertValue { @@ -258,7 +258,7 @@ impl FdbStore { document_id, class, } - .serialize(true); + .serialize(WITH_SUBSPACE); let matches = match read_chunked_value(&key, &trx, false).await { Ok(ChunkedValue::Single(bytes)) => assert_value.matches(bytes.as_ref()), @@ -395,7 +395,16 @@ impl FdbStore { let mut iter = trx.get_ranges( RangeOption { begin: KeySelector::first_greater_or_equal(&[SUBSPACE_BITMAPS, 0u8][..]), - end: KeySelector::first_greater_or_equal(&[SUBSPACE_BITMAPS, u8::MAX][..]), + end: KeySelector::first_greater_or_equal( + &[ + SUBSPACE_BITMAPS, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + ][..], + ), mode: options::StreamingMode::WantAll, reverse: false, ..Default::default() @@ -444,8 +453,8 @@ impl FdbStore { } pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> crate::Result<()> { - let from = from.serialize(true); - let to = to.serialize(true); + let from = from.serialize(WITH_SUBSPACE); + let to = to.serialize(WITH_SUBSPACE); let trx = self.db.create_trx()?; trx.clear_range(&from, &to); diff --git a/crates/store/src/backend/mysql/read.rs b/crates/store/src/backend/mysql/read.rs index 5258c96b..0e6e2637 100644 --- a/crates/store/src/backend/mysql/read.rs +++ b/crates/store/src/backend/mysql/read.rs @@ -44,7 +44,7 @@ impl MysqlStore { char::from(key.subspace()) )) .await?; - let key = key.serialize(false); + let key = key.serialize(0); conn.exec_first::, _, _>(&s, (key,)) .await .map_err(Into::into) @@ -61,10 +61,10 @@ impl MysqlStore { &self, mut key: BitmapKey, ) -> crate::Result> { - let begin = key.serialize(false); + let begin = key.serialize(0); key.block_num = u32::MAX; let key_len = begin.len(); - let end = key.serialize(false); + let end = key.serialize(0); let mut conn = self.conn_pool.get_conn().await?; let mut bm = RoaringBitmap::new(); @@ -86,8 +86,8 @@ impl MysqlStore { ) -> crate::Result<()> { let mut conn = self.conn_pool.get_conn().await?; let table = char::from(params.begin.subspace()); - let begin = params.begin.serialize(false); - let end = params.end.serialize(false); + let begin = params.begin.serialize(0); + let end = params.end.serialize(0); let keys = if params.values { "k, v" } else { "k" }; let s = conn @@ -144,7 +144,7 @@ impl MysqlStore { &self, key: impl Into> + Sync + Send, ) -> crate::Result { - let key = key.into().serialize(false); + let key = key.into().serialize(0); let mut conn = self.conn_pool.get_conn().await?; let s = conn.prep("SELECT v FROM c WHERE k = ?").await?; match conn.exec_first::(&s, (key,)).await { diff --git a/crates/store/src/backend/mysql/write.rs b/crates/store/src/backend/mysql/write.rs index 6e0dd52c..141974b1 100644 --- a/crates/store/src/backend/mysql/write.rs +++ b/crates/store/src/backend/mysql/write.rs @@ -105,7 +105,7 @@ impl MysqlStore { document_id, class, } - .serialize(false); + .serialize(0); if *by >= 0 { let s = trx @@ -128,7 +128,7 @@ impl MysqlStore { class, }; let table = char::from(key.subspace()); - let key = key.serialize(false); + let key = key.serialize(0); if let ValueOp::Set(value) = op { let exists = asserted_values.get(&key); @@ -170,7 +170,7 @@ impl MysqlStore { class: BitmapClass::DocumentIds, block_num: document_id, } - .serialize(false); + .serialize(0); if trx.exec_first::(&s, (key,)).await?.is_some() { trx.rollback().await?; return Ok(false); @@ -191,7 +191,7 @@ impl MysqlStore { field: *field, key, } - .serialize(false); + .serialize(0); let s = if *set { trx.prep("INSERT IGNORE INTO i (k) VALUES (?)").await? @@ -207,7 +207,7 @@ impl MysqlStore { class, block_num: document_id, } - .serialize(false); + .serialize(0); let s = if *set { if matches!(class, BitmapClass::DocumentIds) { @@ -230,7 +230,7 @@ impl MysqlStore { collection: *collection, change_id: *change_id, } - .serialize(false); + .serialize(0); let s = trx .prep("INSERT INTO l (k, v) VALUES (?, ?) ON DUPLICATE KEY UPDATE v = VALUES(v)") @@ -248,7 +248,7 @@ impl MysqlStore { class, }; let table = char::from(key.subspace()); - let key = key.serialize(false); + let key = key.serialize(0); let s = trx .prep(&format!("SELECT v FROM {} WHERE k = ? FOR UPDATE", table)) @@ -284,7 +284,7 @@ impl MysqlStore { char::from(from.subspace()), )) .await?; - conn.exec_drop(&s, (&from.serialize(false), &to.serialize(false))) + conn.exec_drop(&s, (&from.serialize(0), &to.serialize(0))) .await .map_err(Into::into) } diff --git a/crates/store/src/backend/postgres/read.rs b/crates/store/src/backend/postgres/read.rs index f13e7c1c..1b67db3a 100644 --- a/crates/store/src/backend/postgres/read.rs +++ b/crates/store/src/backend/postgres/read.rs @@ -43,7 +43,7 @@ impl PostgresStore { char::from(key.subspace()) )) .await?; - let key = key.serialize(false); + let key = key.serialize(0); conn.query_opt(&s, &[&key]) .await .map_err(Into::into) @@ -60,10 +60,10 @@ impl PostgresStore { &self, mut key: BitmapKey, ) -> crate::Result> { - let begin = key.serialize(false); + let begin = key.serialize(0); key.block_num = u32::MAX; let key_len = begin.len(); - let end = key.serialize(false); + let end = key.serialize(0); let conn = self.conn_pool.get().await?; let mut bm = RoaringBitmap::new(); @@ -90,8 +90,8 @@ impl PostgresStore { ) -> crate::Result<()> { let conn = self.conn_pool.get().await?; let table = char::from(params.begin.subspace()); - let begin = params.begin.serialize(false); - let end = params.end.serialize(false); + let begin = params.begin.serialize(0); + let end = params.end.serialize(0); let keys = if params.values { "k, v" } else { "k" }; let s = conn @@ -142,7 +142,7 @@ impl PostgresStore { &self, key: impl Into> + Sync + Send, ) -> crate::Result { - let key = key.into().serialize(false); + let key = key.into().serialize(0); let conn = self.conn_pool.get().await?; let s = conn.prepare_cached("SELECT v FROM c WHERE k = $1").await?; match conn.query_opt(&s, &[&key]).await { diff --git a/crates/store/src/backend/postgres/write.rs b/crates/store/src/backend/postgres/write.rs index 6523663a..6373ecfb 100644 --- a/crates/store/src/backend/postgres/write.rs +++ b/crates/store/src/backend/postgres/write.rs @@ -111,7 +111,7 @@ impl PostgresStore { document_id, class, } - .serialize(false); + .serialize(0); if *by >= 0 { let s = trx @@ -136,7 +136,7 @@ impl PostgresStore { class, }; let table = char::from(key.subspace()); - let key = key.serialize(false); + let key = key.serialize(0); if let ValueOp::Set(value) = op { let s = if let Some(exists) = asserted_values.get(&key) { @@ -177,7 +177,7 @@ impl PostgresStore { class: BitmapClass::DocumentIds, block_num: document_id, } - .serialize(false); + .serialize(0); if trx.query_opt(&s, &[&key]).await?.is_some() { return Ok(false); } @@ -197,7 +197,7 @@ impl PostgresStore { field: *field, key, } - .serialize(false); + .serialize(0); let s = if *set { trx.prepare_cached( @@ -216,7 +216,7 @@ impl PostgresStore { class, block_num: document_id, } - .serialize(false); + .serialize(0); let s = if *set { if matches!(class, BitmapClass::DocumentIds) { @@ -242,7 +242,7 @@ impl PostgresStore { collection: *collection, change_id: *change_id, } - .serialize(false); + .serialize(0); let s = trx .prepare_cached(concat!( @@ -263,7 +263,7 @@ impl PostgresStore { class, }; let table = char::from(key.subspace()); - let key = key.serialize(false); + let key = key.serialize(0); let s = trx .prepare_cached(&format!("SELECT v FROM {} WHERE k = $1 FOR UPDATE", table)) @@ -301,7 +301,7 @@ impl PostgresStore { char::from(from.subspace()), )) .await?; - conn.execute(&s, &[&from.serialize(false), &to.serialize(false)]) + conn.execute(&s, &[&from.serialize(0), &to.serialize(0)]) .await .map(|_| ()) .map_err(Into::into) diff --git a/crates/store/src/backend/rocksdb/read.rs b/crates/store/src/backend/rocksdb/read.rs index c0c77d30..287d3e2f 100644 --- a/crates/store/src/backend/rocksdb/read.rs +++ b/crates/store/src/backend/rocksdb/read.rs @@ -26,7 +26,7 @@ use rocksdb::{Direction, IteratorMode}; use crate::{ write::{BitmapClass, ValueClass}, - BitmapKey, Deserialize, IterateParams, Key, ValueKey, + BitmapKey, Deserialize, IterateParams, Key, ValueKey, WITHOUT_BLOCK_NUM, }; use super::{RocksDbStore, CF_BITMAPS, CF_COUNTERS}; @@ -41,7 +41,7 @@ impl RocksDbStore { db.get_pinned_cf( &db.cf_handle(std::str::from_utf8(&[key.subspace()]).unwrap()) .unwrap(), - &key.serialize(false), + &key.serialize(0), ) .map_err(Into::into) .and_then(|value| { @@ -61,21 +61,24 @@ impl RocksDbStore { ) -> crate::Result> { let db = self.db.clone(); self.spawn_worker(move || { - db.get_pinned_cf(&db.cf_handle(CF_BITMAPS).unwrap(), &key.serialize(false)) - .map_err(Into::into) - .and_then(|value| { - if let Some(value) = value { - RoaringBitmap::deserialize(&value).map(|rb| { - if !rb.is_empty() { - Some(rb) - } else { - None - } - }) - } else { - Ok(None) - } - }) + db.get_pinned_cf( + &db.cf_handle(CF_BITMAPS).unwrap(), + &key.serialize(WITHOUT_BLOCK_NUM), + ) + .map_err(Into::into) + .and_then(|value| { + if let Some(value) = value { + RoaringBitmap::deserialize(&value).map(|rb| { + if !rb.is_empty() { + Some(rb) + } else { + None + } + }) + } else { + Ok(None) + } + }) }) .await } @@ -91,8 +94,8 @@ impl RocksDbStore { let cf = db .cf_handle(std::str::from_utf8(&[params.begin.subspace()]).unwrap()) .unwrap(); - let begin = params.begin.serialize(false); - let end = params.end.serialize(false); + let begin = params.begin.serialize(0); + let end = params.end.serialize(0); let it_mode = if params.ascending { IteratorMode::From(&begin, Direction::Forward) } else { @@ -119,7 +122,7 @@ impl RocksDbStore { &self, key: impl Into> + Sync + Send, ) -> crate::Result { - let key = key.into().serialize(false); + let key = key.into().serialize(0); let db = self.db.clone(); self.spawn_worker(move || { db.get_pinned_cf(&db.cf_handle(CF_COUNTERS).unwrap(), &key) diff --git a/crates/store/src/backend/rocksdb/write.rs b/crates/store/src/backend/rocksdb/write.rs index 782963d6..27b94d44 100644 --- a/crates/store/src/backend/rocksdb/write.rs +++ b/crates/store/src/backend/rocksdb/write.rs @@ -42,7 +42,7 @@ use crate::{ write::{ Batch, BitmapClass, Operation, ValueClass, ValueOp, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, }, - BitmapKey, Deserialize, IndexKey, Key, LogKey, ValueKey, + BitmapKey, Deserialize, IndexKey, Key, LogKey, ValueKey, WITHOUT_BLOCK_NUM, }; impl RocksDbStore { @@ -101,8 +101,8 @@ impl RocksDbStore { .unwrap(); // TODO use delete_range when implemented (see https://github.com/rust-rocksdb/rust-rocksdb/issues/839) - let from = from.serialize(false); - let to = to.serialize(false); + let from = from.serialize(0); + let to = to.serialize(0); let mut delete_keys = Vec::new(); let it_mode = IteratorMode::From(&from, Direction::Forward); @@ -178,7 +178,7 @@ impl<'x> RocksDBTransaction<'x> { document_id, class, } - .serialize(false); + .serialize(0); txn.merge_cf(&self.cf_counters, &key, &by.to_le_bytes()[..])?; } @@ -189,7 +189,7 @@ impl<'x> RocksDBTransaction<'x> { document_id, class, }; - let key = key.serialize(false); + let key = key.serialize(0); if let ValueOp::Set(value) = op { txn.put_cf(&self.cf_values, &key, value)?; @@ -204,8 +204,7 @@ impl<'x> RocksDBTransaction<'x> { class: BitmapClass::DocumentIds, block_num: 0, } - .serialize(false), - //true, + .serialize(WITHOUT_BLOCK_NUM), )? .and_then(|bytes| RoaringBitmap::deserialize(&bytes).ok()) { @@ -227,7 +226,7 @@ impl<'x> RocksDBTransaction<'x> { field: *field, key, } - .serialize(false); + .serialize(0); if *set { txn.put_cf(&self.cf_indexes, &key, [])?; @@ -242,7 +241,7 @@ impl<'x> RocksDBTransaction<'x> { class, block_num: 0, } - .serialize(false); + .serialize(WITHOUT_BLOCK_NUM); let value = if *set { set_bit(document_id) @@ -262,7 +261,7 @@ impl<'x> RocksDBTransaction<'x> { collection: *collection, change_id: *change_id, } - .serialize(false); + .serialize(0); txn.put_cf(&self.cf_logs, &key, set)?; } @@ -276,7 +275,7 @@ impl<'x> RocksDBTransaction<'x> { document_id, class, }; - let key = key.serialize(false); + let key = key.serialize(0); let matches = txn .get_pinned_for_update_cf(&self.cf_values, &key, true)? .map(|value| assert_value.matches(&value)) @@ -320,7 +319,7 @@ impl<'x> RocksDBTransaction<'x> { document_id, class, } - .serialize(false); + .serialize(0); wb.merge_cf(&self.cf_counters, &key, &by.to_le_bytes()[..]); } @@ -331,7 +330,7 @@ impl<'x> RocksDBTransaction<'x> { document_id, class, }; - let key = key.serialize(false); + let key = key.serialize(0); if let ValueOp::Set(value) = op { wb.put_cf(&self.cf_values, &key, value); @@ -347,7 +346,7 @@ impl<'x> RocksDBTransaction<'x> { field: *field, key, } - .serialize(false); + .serialize(0); if *set { wb.put_cf(&self.cf_indexes, &key, []); @@ -362,7 +361,7 @@ impl<'x> RocksDBTransaction<'x> { class, block_num: 0, } - .serialize(false); + .serialize(WITHOUT_BLOCK_NUM); let value = if *set { set_bit(document_id) @@ -382,7 +381,7 @@ impl<'x> RocksDBTransaction<'x> { collection: *collection, change_id: *change_id, } - .serialize(false); + .serialize(0); wb.put_cf(&self.cf_logs, &key, set); } diff --git a/crates/store/src/backend/sqlite/read.rs b/crates/store/src/backend/sqlite/read.rs index 4dc5010d..f1fe3391 100644 --- a/crates/store/src/backend/sqlite/read.rs +++ b/crates/store/src/backend/sqlite/read.rs @@ -42,7 +42,7 @@ impl SqliteStore { "SELECT v FROM {} WHERE k = ?", char::from(key.subspace()) ))?; - let key = key.serialize(false); + let key = key.serialize(0); result .query_row([&key], |row| { U::deserialize(row.get_ref(0)?.as_bytes()?) @@ -58,10 +58,10 @@ impl SqliteStore { &self, mut key: BitmapKey, ) -> crate::Result> { - let begin = key.serialize(false); + let begin = key.serialize(0); key.block_num = u32::MAX; let key_len = begin.len(); - let end = key.serialize(false); + let end = key.serialize(0); let conn = self.conn_pool.get()?; self.spawn_worker(move || { @@ -89,8 +89,8 @@ impl SqliteStore { self.spawn_worker(move || { let table = char::from(params.begin.subspace()); - let begin = params.begin.serialize(false); - let end = params.end.serialize(false); + let begin = params.begin.serialize(0); + let end = params.end.serialize(0); let keys = if params.values { "k, v" } else { "k" }; let mut query = conn.prepare_cached(&match (params.first, params.ascending) { @@ -139,7 +139,7 @@ impl SqliteStore { &self, key: impl Into> + Sync + Send, ) -> crate::Result { - let key = key.into().serialize(false); + let key = key.into().serialize(0); let conn = self.conn_pool.get()?; self.spawn_worker(move || { match conn diff --git a/crates/store/src/backend/sqlite/write.rs b/crates/store/src/backend/sqlite/write.rs index e4acf8de..9578cb0d 100644 --- a/crates/store/src/backend/sqlite/write.rs +++ b/crates/store/src/backend/sqlite/write.rs @@ -66,7 +66,7 @@ impl SqliteStore { document_id, class, } - .serialize(false); + .serialize(0); if *by >= 0 { trx.prepare_cached(concat!( @@ -87,7 +87,7 @@ impl SqliteStore { class, }; let table = char::from(key.subspace()); - let key = key.serialize(false); + let key = key.serialize(0); if let ValueOp::Set(value) = op { trx.prepare_cached(&format!( @@ -108,7 +108,7 @@ impl SqliteStore { field: *field, key, } - .serialize(false); + .serialize(0); if *set { trx.prepare_cached("INSERT OR IGNORE INTO i (k) VALUES (?)")? @@ -125,7 +125,7 @@ impl SqliteStore { class, block_num: document_id, } - .serialize(false); + .serialize(0); if *set { trx.prepare_cached("INSERT OR IGNORE INTO b (k) VALUES (?)")? @@ -145,7 +145,7 @@ impl SqliteStore { collection: *collection, change_id: *change_id, } - .serialize(false); + .serialize(0); trx.prepare_cached("INSERT OR REPLACE INTO l (k, v) VALUES (?, ?)")? .execute([&key, set])?; @@ -161,7 +161,7 @@ impl SqliteStore { class, }; let table = char::from(key.subspace()); - let key = key.serialize(false); + let key = key.serialize(0); let matches = trx .prepare_cached(&format!("SELECT v FROM {} WHERE k = ?", table))? @@ -193,7 +193,7 @@ impl SqliteStore { "DELETE FROM {} WHERE k >= ? AND k < ?", char::from(from.subspace()), ))? - .execute([from.serialize(false), to.serialize(false)])?; + .execute([from.serialize(0), to.serialize(0)])?; Ok(()) }) diff --git a/crates/store/src/dispatch/store.rs b/crates/store/src/dispatch/store.rs index 20dafc15..7ecb8da5 100644 --- a/crates/store/src/dispatch/store.rs +++ b/crates/store/src/dispatch/store.rs @@ -180,7 +180,7 @@ impl Store { block_num: 0, class, } - .serialize(false); + .serialize(0); bitmaps.push((key, class.clone(), document_id, *set)); } _ => {} diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 0cc31dbd..f780b6fb 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -71,8 +71,12 @@ pub trait Serialize { fn serialize(self) -> Vec; } +// Key serialization flags +pub(crate) const WITH_SUBSPACE: u32 = 1; +pub(crate) const WITHOUT_BLOCK_NUM: u32 = 1 << 1; + pub trait Key: Sync + Send { - fn serialize(&self, include_subspace: bool) -> Vec; + fn serialize(&self, flags: u32) -> Vec; fn subspace(&self) -> u8; } diff --git a/crates/store/src/query/filter.rs b/crates/store/src/query/filter.rs index 3e2deae0..53e2e4e9 100644 --- a/crates/store/src/query/filter.rs +++ b/crates/store/src/query/filter.rs @@ -278,7 +278,7 @@ impl Store { collection, field, } - .serialize(false); + .serialize(0); self.iterate( IterateParams::new(begin, end).no_values().ascending(), diff --git a/crates/store/src/write/key.rs b/crates/store/src/write/key.rs index 74200496..dd1834f9 100644 --- a/crates/store/src/write/key.rs +++ b/crates/store/src/write/key.rs @@ -26,7 +26,8 @@ use utils::codec::leb128::Leb128_; use crate::{ BitmapKey, IndexKey, IndexKeyPrefix, Key, LogKey, ValueKey, BLOB_HASH_LEN, SUBSPACE_BITMAPS, - SUBSPACE_INDEXES, SUBSPACE_LOGS, SUBSPACE_VALUES, U32_LEN, U64_LEN, + SUBSPACE_INDEXES, SUBSPACE_LOGS, SUBSPACE_VALUES, U32_LEN, U64_LEN, WITHOUT_BLOCK_NUM, + WITH_SUBSPACE, }; use super::{AnyKey, BitmapClass, BlobOp, DirectoryClass, TagValue, ValueClass}; @@ -168,9 +169,9 @@ impl> ValueKey { } impl Key for IndexKeyPrefix { - fn serialize(&self, include_subspace: bool) -> Vec { + fn serialize(&self, flags: u32) -> Vec { { - if include_subspace { + if (flags & WITH_SUBSPACE) != 0 { KeySerializer::new(std::mem::size_of::() + 1) .write(crate::SUBSPACE_INDEXES) } else { @@ -199,9 +200,9 @@ impl Key for LogKey { SUBSPACE_LOGS } - fn serialize(&self, include_subspace: bool) -> Vec { + fn serialize(&self, flags: u32) -> Vec { { - if include_subspace { + if (flags & WITH_SUBSPACE) != 0 { KeySerializer::new(std::mem::size_of::() + 1).write(crate::SUBSPACE_LOGS) } else { KeySerializer::new(std::mem::size_of::()) @@ -219,8 +220,8 @@ impl + Sync + Send> Key for ValueKey { SUBSPACE_VALUES } - fn serialize(&self, include_subspace: bool) -> Vec { - let serializer = if include_subspace { + fn serialize(&self, flags: u32) -> Vec { + let serializer = if (flags & WITH_SUBSPACE) != 0 { KeySerializer::new(self.class.as_ref().serialized_size() + 2).write(self.subspace()) } else { KeySerializer::new(self.class.as_ref().serialized_size() + 1) @@ -305,10 +306,10 @@ impl + Sync + Send> Key for IndexKey { SUBSPACE_INDEXES } - fn serialize(&self, include_subspace: bool) -> Vec { + fn serialize(&self, flags: u32) -> Vec { let key = self.key.as_ref(); { - if include_subspace { + if (flags & WITH_SUBSPACE) != 0 { KeySerializer::new(std::mem::size_of::>() + key.len() + 1) .write(crate::SUBSPACE_INDEXES) } else { @@ -329,7 +330,7 @@ impl + Sync + Send> Key for BitmapKey { SUBSPACE_BITMAPS } - fn serialize(&self, include_subspace: bool) -> Vec { + fn serialize(&self, flags: u32) -> Vec { const BM_DOCUMENT_IDS: u8 = 0; const BM_TAG: u8 = 1 << 6; const BM_TEXT: u8 = 1 << 7; @@ -338,8 +339,8 @@ impl + Sync + Send> Key for BitmapKey { const TAG_TEXT: u8 = 1 << 0; const TAG_STATIC: u8 = 1 << 1; - match self.class.as_ref() { - BitmapClass::DocumentIds => if include_subspace { + let serializer = match self.class.as_ref() { + BitmapClass::DocumentIds => if (flags & WITH_SUBSPACE) != 0 { KeySerializer::new(U32_LEN + 3).write(SUBSPACE_BITMAPS) } else { KeySerializer::new(U32_LEN + 2) @@ -348,7 +349,7 @@ impl + Sync + Send> Key for BitmapKey { .write(self.collection) .write(BM_DOCUMENT_IDS), BitmapClass::Tag { field, value } => match value { - TagValue::Id(id) => if include_subspace { + TagValue::Id(id) => if (flags & WITH_SUBSPACE) != 0 { KeySerializer::new((U32_LEN * 2) + 4).write(SUBSPACE_BITMAPS) } else { KeySerializer::new((U32_LEN * 2) + 3) @@ -358,7 +359,7 @@ impl + Sync + Send> Key for BitmapKey { .write(BM_TAG | TAG_ID) .write(*field) .write_leb128(*id), - TagValue::Text(text) => if include_subspace { + TagValue::Text(text) => if (flags & WITH_SUBSPACE) != 0 { KeySerializer::new(U32_LEN + 4 + text.len()).write(SUBSPACE_BITMAPS) } else { KeySerializer::new(U32_LEN + 3 + text.len()) @@ -368,7 +369,7 @@ impl + Sync + Send> Key for BitmapKey { .write(BM_TAG | TAG_TEXT) .write(*field) .write(text.as_slice()), - TagValue::Static(id) => if include_subspace { + TagValue::Static(id) => if (flags & WITH_SUBSPACE) != 0 { KeySerializer::new(U32_LEN + 5).write(SUBSPACE_BITMAPS) } else { KeySerializer::new(U32_LEN + 4) @@ -379,7 +380,7 @@ impl + Sync + Send> Key for BitmapKey { .write(*field) .write(*id), }, - BitmapClass::Text { field, token } => if include_subspace { + BitmapClass::Text { field, token } => if (flags & WITH_SUBSPACE) != 0 { KeySerializer::new(U32_LEN + 16 + 3 + 1).write(SUBSPACE_BITMAPS) } else { KeySerializer::new(U32_LEN + 16 + 3) @@ -389,16 +390,20 @@ impl + Sync + Send> Key for BitmapKey { .write(BM_TEXT | token.len) .write(*field) .write(token.hash.as_slice()), + }; + + if (flags & WITHOUT_BLOCK_NUM) != 0 { + serializer.finalize() + } else { + serializer.write(self.block_num).finalize() } - .write(self.block_num) - .finalize() } } impl + Sync + Send> Key for AnyKey { - fn serialize(&self, include_subspace: bool) -> Vec { + fn serialize(&self, flags: u32) -> Vec { let key = self.key.as_ref(); - if include_subspace { + if (flags & WITH_SUBSPACE) != 0 { KeySerializer::new(key.len() + 1).write(self.subspace) } else { KeySerializer::new(key.len()) diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 742ce018..67709495 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "utils" -version = "0.4.2" +version = "0.5.0" edition = "2021" resolver = "2" diff --git a/resources/config.zip b/resources/config.zip index 878944dc..2b1276b0 100644 Binary files a/resources/config.zip and b/resources/config.zip differ diff --git a/resources/config/config.toml b/resources/config/config.toml index f033d3f2..b79c9f0c 100644 --- a/resources/config/config.toml +++ b/resources/config/config.toml @@ -20,8 +20,8 @@ files = [ "%{BASE_PATH}%/etc/common/server.toml", "%{BASE_PATH}%/etc/directory/lmtp.toml", "%{BASE_PATH}%/etc/directory/memory.toml", "%{BASE_PATH}%/etc/directory/sql.toml", - "%{BASE_PATH}%/etc/store/elastic.toml", - "%{BASE_PATH}%/etc/store/filesystem.toml", + "%{BASE_PATH}%/etc/store/elasticsearch.toml", + "%{BASE_PATH}%/etc/store/fs.toml", "%{BASE_PATH}%/etc/store/foundationdb.toml", "%{BASE_PATH}%/etc/store/mysql.toml", "%{BASE_PATH}%/etc/store/postgresql.toml", diff --git a/resources/config/store/elastic.toml b/resources/config/store/elasticsearch.toml similarity index 80% rename from resources/config/store/elastic.toml rename to resources/config/store/elasticsearch.toml index 8f3f8231..f5f93bce 100644 --- a/resources/config/store/elastic.toml +++ b/resources/config/store/elasticsearch.toml @@ -2,7 +2,7 @@ # ElasticSearch FTS Store configuration ############################################# -[store."elastic"] +[store."elasticsearch"] type = "elasticsearch" url = "https://localhost:9200" user = "elastic" @@ -10,9 +10,9 @@ password = "myelasticpassword" #cloud-id = "my-cloud-id" disable = true -[store."elastic".tls] +[store."elasticsearch".tls] allow-invalid-certs = true -[store."elastic".index] +[store."elasticsearch".index] shards = 3 replicas = 0 diff --git a/resources/config/store/filesystem.toml b/resources/config/store/fs.toml similarity index 100% rename from resources/config/store/filesystem.toml rename to resources/config/store/fs.toml diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 71e639be..a2c57a49 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -300,8 +300,8 @@ pub async fn jmap_tests() { thread_get::test(&mut params).await; thread_merge::test(&mut params).await; mailbox::test(&mut params).await; - delivery::test(&mut params).await; - auth_acl::test(&mut params).await;*/ + delivery::test(&mut params).await;*/ + auth_acl::test(&mut params).await; auth_limits::test(&mut params).await; auth_oauth::test(&mut params).await; event_source::test(&mut params).await;