diff --git a/crates/common/src/config/network.rs b/crates/common/src/config/network.rs index 8cc1f2f0..afeed208 100644 --- a/crates/common/src/config/network.rs +++ b/crates/common/src/config/network.rs @@ -175,44 +175,36 @@ impl Network { impl AsnGeoLookupConfig { pub fn parse(config: &mut Config) -> Option { - match config.value("server.asn.type")? { + match config.value("asn.type")? { "dns" => AsnGeoLookupConfig::Dns { - zone_ipv4: config - .value_require_non_empty("server.asn.zone.ipv4")? - .to_string(), - zone_ipv6: config - .value_require_non_empty("server.asn.zone.ipv6")? - .to_string(), - separator: config - .value_require_non_empty("server.asn.separator")? - .to_string(), - index_asn: config.property_require("server.asn.index.asn")?, - index_asn_name: config.property("server.asn.index.asn-name"), - index_country: config.property("server.asn.index.country"), + zone_ipv4: config.value_require_non_empty("asn.zone.ipv4")?.to_string(), + zone_ipv6: config.value_require_non_empty("asn.zone.ipv6")?.to_string(), + separator: config.value_require_non_empty("asn.separator")?.to_string(), + index_asn: config.property_require("asn.index.asn")?, + index_asn_name: config.property("asn.index.asn-name"), + index_country: config.property("asn.index.country"), } .into(), "resource" => { let asn_resources = config - .values("server.asn.urls.asn") + .values("asn.urls.asn") .map(|(_, v)| v.to_string()) .collect::>(); let geo_resources = config - .values("server.asn.urls.geo") + .values("asn.urls.geo") .map(|(_, v)| v.to_string()) .collect::>(); if asn_resources.is_empty() && geo_resources.is_empty() { - config.new_build_error("server.asn.urls", "No resources found"); + config.new_build_error("asn.urls", "No resources found"); return None; } AsnGeoLookupConfig::Resource { - headers: parse_http_headers(config, "server.asn"), - expires: config.property_or_default::("server.asn.expires", "1d")?, - timeout: config.property_or_default::("server.asn.timeout", "5m")?, - max_size: config - .property("server.asn.max-size") - .unwrap_or(100 * 1024 * 1024), + headers: parse_http_headers(config, "asn"), + expires: config.property_or_default::("asn.expires", "1d")?, + timeout: config.property_or_default::("asn.timeout", "5m")?, + max_size: config.property("asn.max-size").unwrap_or(100 * 1024 * 1024), asn_resources, geo_resources, } @@ -220,7 +212,7 @@ impl AsnGeoLookupConfig { } "disable" | "disabled" | "none" | "false" => AsnGeoLookupConfig::Disabled.into(), _ => { - config.new_build_error("server.asn.type", "Invalid value"); + config.new_build_error("asn.type", "Invalid value"); None } } diff --git a/crates/common/src/manager/boot.rs b/crates/common/src/manager/boot.rs index e698ecb1..17b52837 100644 --- a/crates/common/src/manager/boot.rs +++ b/crates/common/src/manager/boot.rs @@ -249,34 +249,32 @@ impl BootManager { } }) { Some(Err(_)) => { - if std::env::var("DO_NOT_MIGRATE").is_err() { - let _ = manager.clear_prefix("lookup.spam-").await; - let _ = manager - .clear_prefix("sieve.trusted.scripts.spam-filter") - .await; - let _ = manager - .clear_prefix("sieve.trusted.scripts.track-replies") - .await; - let _ = manager.clear_prefix("sieve.trusted.scripts.greylist").await; - let _ = manager.clear_prefix("sieve.trusted.scripts.train").await; - let _ = manager.clear_prefix("session.data.script").await; - let _ = manager.clear("version.spam-filter").await; + let _ = manager.clear_prefix("lookup.spam-").await; + let _ = manager + .clear_prefix("sieve.trusted.scripts.spam-filter") + .await; + let _ = manager + .clear_prefix("sieve.trusted.scripts.track-replies") + .await; + let _ = manager.clear_prefix("sieve.trusted.scripts.greylist").await; + let _ = manager.clear_prefix("sieve.trusted.scripts.train").await; + //let _ = manager.clear_prefix("session.data.script").await; + let _ = manager.clear("version.spam-filter").await; - match manager.fetch_spam_rules().await { - Ok(external_config) => { - trc::event!( - Config(trc::ConfigEvent::ImportExternal), - Version = external_config.version.to_string(), - Id = "spam-filter" - ); - insert_keys.extend(external_config.keys); - } - Err(err) => { - config.new_build_error( - "*", - format!("Failed to fetch spam filter: {err}"), - ); - } + match manager.fetch_spam_rules().await { + Ok(external_config) => { + trc::event!( + Config(trc::ConfigEvent::ImportExternal), + Version = external_config.version.to_string(), + Id = "spam-filter" + ); + insert_keys.extend(external_config.keys); + } + Err(err) => { + config.new_build_error( + "*", + format!("Failed to fetch spam filter: {err}"), + ); } } @@ -414,7 +412,7 @@ impl BootManager { .property_or_default::("spam-filter.auto-update", "false") .unwrap_or_default() { - if let Err(err) = core.storage.config.update_spam_rules(false).await { + if let Err(err) = core.storage.config.update_spam_rules(false, false).await { trc::event!( Resource(trc::ResourceEvent::Error), Details = "Failed to update spam-filter", diff --git a/crates/common/src/manager/config.rs b/crates/common/src/manager/config.rs index f1daf1a5..79b82724 100644 --- a/crates/common/src/manager/config.rs +++ b/crates/common/src/manager/config.rs @@ -329,24 +329,57 @@ impl ConfigManager { }) } - pub async fn update_spam_rules(&self, overwrite: bool) -> trc::Result> { - let external = self.fetch_spam_rules().await.map_err(|reason| { + pub async fn update_spam_rules( + &self, + force_update: bool, + overwrite: bool, + ) -> trc::Result> { + let current_version = self + .get("version.spam-filter") + .await? + .and_then(|v| Semver::try_from(v.as_str()).ok()); + let is_update = current_version.is_some(); + + let mut external = self.fetch_spam_rules().await.map_err(|reason| { trc::EventType::Config(trc::ConfigEvent::FetchError) .caused_by(trc::location!()) .details("Failed to update spam filter rules") .ctx(trc::Key::Reason, reason) })?; - if self.get("version.spam-filter").await?.map_or(true, |v| { - v.as_str().try_into().map_or(true, |v| external.version > v) - }) { - // Delete previous STWT_* rules - for prefix in [ - "spam-filter.rule.stwt_", - "spam-filter.dnsbl.server.stwt_", - "http-lookup.stwt_", - ] { - self.clear_prefix(prefix).await?; + if current_version.map_or(true, |v| external.version > v || force_update) { + if is_update { + // Delete previous STWT_* rules + let mut rule_settings = AHashMap::new(); + for prefix in [ + "spam-filter.rule.stwt_", + "spam-filter.dnsbl.server.stwt_", + "http-lookup.stwt_", + ] { + for (key, value) in self.list(prefix, false).await? { + if key.ends_with(".enable") { + rule_settings.insert(key, value); + } + } + + self.clear_prefix(prefix).await?; + } + + // Update keys + if !rule_settings.is_empty() { + for key in &mut external.keys { + if let Some(value) = rule_settings.remove(&key.key) { + key.value = value; + } + } + } + + if !overwrite { + // Do not overwrite ASN or LLM settings + external.keys.retain(|key| { + !key.key.starts_with("spam-filter.llm.") && !key.key.starts_with("asn.") + }); + } } self.set(external.keys, overwrite).await?; @@ -391,16 +424,9 @@ impl ConfigManager { } else if key.starts_with("spam-filter.") || key.starts_with("http-lookup.") || (key.starts_with("lookup.") && !key.starts_with("lookup.default.")) - || key.starts_with("server.asn.") + || key.starts_with("asn.") { external.keys.push(ConfigKey::from((key, value))); - } else { - trc::event!( - Config(trc::ConfigEvent::ExternalKeyIgnored), - Key = key, - Value = value, - Id = "spam-filter", - ); } } diff --git a/crates/jmap/src/api/management/reload.rs b/crates/jmap/src/api/management/reload.rs index 1040d863..b8d00469 100644 --- a/crates/jmap/src/api/management/reload.rs +++ b/crates/jmap/src/api/management/reload.rs @@ -125,15 +125,17 @@ impl ManageReload for Server { (Some("spam-filter"), &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::SpamFilterUpdate)?; + let params = UrlParams::new(req.uri().query()); - let overwrite = UrlParams::new(req.uri().query()).has_key("overwrite"); + let overwrite = params.has_key("overwrite"); + let force = params.has_key("force"); Ok(JsonResponse::new(json!({ "data": self .core .storage .config - .update_spam_rules(overwrite) + .update_spam_rules(force, overwrite) .await? .map(|v| v.to_string()), })) diff --git a/crates/trc/src/event/description.rs b/crates/trc/src/event/description.rs index c7fc3e2b..2314fa25 100644 --- a/crates/trc/src/event/description.rs +++ b/crates/trc/src/event/description.rs @@ -1319,7 +1319,6 @@ impl ConfigEvent { ConfigEvent::ParseWarning => "Configuration parse warning", ConfigEvent::BuildWarning => "Configuration build warning", ConfigEvent::ImportExternal => "Importing external configuration", - ConfigEvent::ExternalKeyIgnored => "External configuration key ignored", ConfigEvent::AlreadyUpToDate => "Configuration already up to date", } } @@ -1337,7 +1336,6 @@ impl ConfigEvent { ConfigEvent::ParseWarning => "A warning occurred while parsing the configuration", ConfigEvent::BuildWarning => "A warning occurred while building the configuration", ConfigEvent::ImportExternal => "An external configuration is being imported", - ConfigEvent::ExternalKeyIgnored => "An external configuration key is ignored", ConfigEvent::AlreadyUpToDate => "The configuration is already up to date", } } diff --git a/crates/trc/src/event/level.rs b/crates/trc/src/event/level.rs index d0a47e93..9c6633df 100644 --- a/crates/trc/src/event/level.rs +++ b/crates/trc/src/event/level.rs @@ -240,8 +240,7 @@ impl EventType { ConfigEvent::DefaultApplied | ConfigEvent::MissingSetting | ConfigEvent::UnusedSetting - | ConfigEvent::AlreadyUpToDate - | ConfigEvent::ExternalKeyIgnored => Level::Debug, + | ConfigEvent::AlreadyUpToDate => Level::Debug, ConfigEvent::ParseWarning | ConfigEvent::BuildWarning => Level::Warn, ConfigEvent::ImportExternal => Level::Info, }, diff --git a/crates/trc/src/lib.rs b/crates/trc/src/lib.rs index 3927caf0..d2e7d9fc 100644 --- a/crates/trc/src/lib.rs +++ b/crates/trc/src/lib.rs @@ -732,7 +732,6 @@ pub enum ConfigEvent { ParseWarning, BuildWarning, ImportExternal, - ExternalKeyIgnored, AlreadyUpToDate, } diff --git a/crates/trc/src/serializers/binary.rs b/crates/trc/src/serializers/binary.rs index bd41386b..a2c987f4 100644 --- a/crates/trc/src/serializers/binary.rs +++ b/crates/trc/src/serializers/binary.rs @@ -362,7 +362,6 @@ impl EventType { EventType::Config(ConfigEvent::BuildError) => 54, EventType::Config(ConfigEvent::BuildWarning) => 55, EventType::Config(ConfigEvent::DefaultApplied) => 56, - EventType::Config(ConfigEvent::ExternalKeyIgnored) => 57, EventType::Config(ConfigEvent::FetchError) => 58, EventType::Config(ConfigEvent::ImportExternal) => 59, EventType::Config(ConfigEvent::MacroError) => 60, @@ -916,7 +915,6 @@ impl EventType { 45 => Some(EventType::Cluster(ClusterEvent::PeerBackOnline)), 46 => Some(EventType::Cluster(ClusterEvent::PeerDiscovered)), 47 => Some(EventType::Cluster(ClusterEvent::PeerHasChanges)), - 48 => Some(EventType::Cluster(ClusterEvent::PeerHasChanges)), // TODO: recycle 49 => Some(EventType::Cluster(ClusterEvent::PeerLeaving)), 50 => Some(EventType::Cluster(ClusterEvent::PeerOffline)), 51 => Some(EventType::Cluster(ClusterEvent::PeerSuspected)), @@ -925,7 +923,6 @@ impl EventType { 54 => Some(EventType::Config(ConfigEvent::BuildError)), 55 => Some(EventType::Config(ConfigEvent::BuildWarning)), 56 => Some(EventType::Config(ConfigEvent::DefaultApplied)), - 57 => Some(EventType::Config(ConfigEvent::ExternalKeyIgnored)), 58 => Some(EventType::Config(ConfigEvent::FetchError)), 59 => Some(EventType::Config(ConfigEvent::ImportExternal)), 60 => Some(EventType::Config(ConfigEvent::MacroError)), @@ -1471,7 +1468,7 @@ impl EventType { } } -// 147 148 335 336 376 458 459 +// 57 48 147 148 335 336 376 458 459 impl Key { fn code(&self) -> u64 {