Automated schema migration

crates/common/src/lib.rs

@@ -38,7 +38,6 @@ use std::{
     sync::{Arc, atomic::AtomicBool},
     time::Duration,
 };
-use store::roaring::RoaringBitmap;
 use tinyvec::TinyVec;
 use tokio::sync::{Notify, Semaphore, mpsc};
 use tokio_rustls::TlsConnector;
@@ -72,6 +71,8 @@ pub static USER_AGENT: &str = "Stalwart/1.0.0";
 pub static DAEMON_NAME: &str = concat!("Stalwart v", env!("CARGO_PKG_VERSION"),);
 pub static PROD_ID: &str = "-//Stalwart Labs Ltd.//Stalwart Server//EN";
 
+pub const DATABASE_SCHEMA_VERSION: u32 = 1;
+
 pub const LONG_1D_SLUMBER: Duration = Duration::from_secs(60 * 60 * 24);
 pub const LONG_1Y_SLUMBER: Duration = Duration::from_secs(60 * 60 * 24 * 365);
 
@@ -777,41 +778,6 @@ impl DavName {
     }
 }
 
-impl MessageStoreCache {
-    pub fn assign_thread_id(&self, thread_name: &[u8], message_id: &[u8]) -> u32 {
-        let mut bytes = Vec::with_capacity(thread_name.len() + message_id.len());
-        bytes.extend_from_slice(thread_name);
-        bytes.extend_from_slice(message_id);
-        let mut hash = store::gxhash::gxhash32(&bytes, 791120);
-
-        if self.emails.items.is_empty() {
-            return hash;
-        }
-
-        // Naive pass, assume hash is unique
-        let mut threads_ids = RoaringBitmap::new();
-        let mut is_unique_hash = true;
-        for item in self.emails.items.iter() {
-            if is_unique_hash && item.thread_id != hash {
-                is_unique_hash = false;
-            }
-            threads_ids.insert(item.thread_id);
-        }
-
-        if is_unique_hash {
-            hash
-        } else {
-            for _ in 0..u32::MAX {
-                hash = hash.wrapping_add(1);
-                if !threads_ids.contains(hash) {
-                    return hash;
-                }
-            }
-            hash
-        }
-    }
-}
-
 impl<T> CacheSwap<T> {
     pub fn new(value: Arc<T>) -> Self {
         Self(Arc::new(ArcSwap::new(value)))

crates/common/src/manager/boot.rs

@@ -5,6 +5,7 @@
  */
 
 use std::{
+    collections::BTreeMap,
     net::{IpAddr, Ipv4Addr},
     path::PathBuf,
     sync::Arc,
@@ -152,7 +153,6 @@ impl BootManager {
                 config.new_build_error("*", format!("Could not read configuration file: {err}"));
             }
         }
-        let cfg_local = config.keys.clone();
 
         // Resolve environment macros
         config.resolve_macros(&["env"]).await;
@@ -168,12 +168,33 @@ impl BootManager {
 
         // Load stores
         let mut stores = Stores::parse(&mut config).await;
+        let local_patterns = Patterns::parse(&mut config);
+
+        // Build local keys and warn about database keys defined in the local configuration
+        let mut cfg_local = BTreeMap::new();
+        let mut warn_keys = Vec::new();
+        for (key, value) in &config.keys {
+            if !local_patterns.is_local_key(key) {
+                warn_keys.push(key.clone());
+            }
+            cfg_local.insert(key.clone(), value.clone());
+        }
+        for warn_key in warn_keys {
+            config.new_build_warning(
+                warn_key,
+                concat!(
+                    "Database key defined in local configuration, this might cause issues. ",
+                    "See https://stalw.art/docs/configuration/overview/#loc",
+                    "al-and-database-settings"
+                ),
+            );
+        }
 
         // Build manager
         let manager = ConfigManager {
             cfg_local: ArcSwap::from_pointee(cfg_local),
             cfg_local_path,
-            cfg_local_patterns: Patterns::parse(&mut config).into(),
+            cfg_local_patterns: local_patterns.into(),
             cfg_store: config
                 .value("storage.data")
                 .and_then(|id| stores.stores.get(id))
@@ -183,10 +204,24 @@ impl BootManager {
 
         // Extend configuration with settings stored in the db
         if !manager.cfg_store.is_none() {
-            manager
-                .extend_config(&mut config, "")
+            for (key, value) in manager
+                .db_list("", false)
                 .await
-                .failed("Failed to read configuration");
+                .failed("Failed to read database configuration")
+            {
+                if manager.cfg_local_patterns.is_local_key(&key) {
+                    config.new_build_warning(
+                        &key,
+                        concat!(
+                            "Local key defined in database, this might cause issues. ",
+                            "See https://stalw.art/docs/configuration/overview/#loc",
+                            "al-and-database-settings"
+                        ),
+                    );
+                }
+
+                config.keys.entry(key).or_insert(value);
+            }
         }
 
         // Parse telemetry
@@ -213,25 +248,9 @@ impl BootManager {
                     )));
                 }
 
-                // Generate a Cluster encryption key if missing
-                if config
-                    .value("cluster.key")
-                    .filter(|v| !v.is_empty())
-                    .is_none()
-                {
-                    insert_keys.push(ConfigKey::from((
-                        "cluster.key",
-                        rng()
-                            .sample_iter(Alphanumeric)
-                            .take(64)
-                            .map(char::from)
-                            .collect::<String>(),
-                    )));
-                }
-
                 // Download Spam filter rules if missing
                 // TODO remove this check in 1.0
-                let mut update_webadmin = match config.value("version.spam-filter").and_then(|v| {
+                let update_webadmin = match config.value("version.spam-filter").and_then(|v| {
                     if !v.is_empty() {
                         Some(Semver::try_from(v))
                     } else {
@@ -248,7 +267,6 @@ impl BootManager {
                             .await;
                         let _ = manager.clear_prefix("sieve.trusted.scripts.greylist").await;
                         let _ = manager.clear_prefix("sieve.trusted.scripts.train").await;
-                        //let _ = manager.clear_prefix("session.data.script").await;
                         let _ = manager.clear("version.spam-filter").await;
 
                         match manager.fetch_spam_rules().await {
@@ -310,18 +328,6 @@ impl BootManager {
                     }
                 };
 
-                // TODO remove key migration in 1.0
-                for (old_key, new_key) in [
-                    ("lookup.default.hostname", "server.hostname"),
-                    ("lookup.default.domain", "report.domain"),
-                ] {
-                    if let (Some(old_value), None) = (config.value(old_key), config.value(new_key))
-                    {
-                        insert_keys.push(ConfigKey::from((new_key, old_value)));
-                        update_webadmin = true;
-                    }
-                }
-
                 // Download webadmin if missing
                 if let Some(blob_store) = config
                     .value("storage.blob")

crates/common/src/manager/config.rs

@@ -133,7 +133,7 @@ impl ConfigManager {
         Ok(grouped)
     }
 
-    async fn db_list(
+    pub async fn db_list(
         &self,
         prefix: &str,
         strip_prefix: bool,
@@ -530,7 +530,6 @@ impl Patterns {
                 Pattern::Include(MatchType::StartsWith(
                     "authentication.fallback-admin.".to_string(),
                 )),
-                Pattern::Exclude(MatchType::Equal("cluster.key".to_string())),
                 Pattern::Include(MatchType::StartsWith("cluster.".to_string())),
                 Pattern::Include(MatchType::Equal("storage.data".to_string())),
                 Pattern::Include(MatchType::Equal("storage.blob".to_string())),