filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Better spam rules auto-updates

Browse Source
This commit is contained in:
mdecimus
2025-01-04 09:56:41 +01:00
parent 2370043594
commit 434d4fdbe8
8 changed files with 93 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
crates/common/src/config/network.rs
View File

@@ -175,44 +175,36 @@ impl Network {
impl AsnGeoLookupConfig {
pub fn parse(config: &mut Config) -> Option<Self> {
match config.value("server.asn.type")? {
match config.value("asn.type")? {
"dns" => AsnGeoLookupConfig::Dns {
zone_ipv4: config
.value_require_non_empty("server.asn.zone.ipv4")?
.to_string(),
zone_ipv6: config
.value_require_non_empty("server.asn.zone.ipv6")?
.to_string(),
separator: config
.value_require_non_empty("server.asn.separator")?
.to_string(),
index_asn: config.property_require("server.asn.index.asn")?,
index_asn_name: config.property("server.asn.index.asn-name"),
index_country: config.property("server.asn.index.country"),
zone_ipv4: config.value_require_non_empty("asn.zone.ipv4")?.to_string(),
zone_ipv6: config.value_require_non_empty("asn.zone.ipv6")?.to_string(),
separator: config.value_require_non_empty("asn.separator")?.to_string(),
index_asn: config.property_require("asn.index.asn")?,
index_asn_name: config.property("asn.index.asn-name"),
index_country: config.property("asn.index.country"),
}
.into(),
"resource" => {
let asn_resources = config
.values("server.asn.urls.asn")
.values("asn.urls.asn")
.map(|(_, v)| v.to_string())
.collect::<Vec<_>>();
let geo_resources = config
.values("server.asn.urls.geo")
.values("asn.urls.geo")
.map(|(_, v)| v.to_string())
.collect::<Vec<_>>();
if asn_resources.is_empty() && geo_resources.is_empty() {
config.new_build_error("server.asn.urls", "No resources found");
config.new_build_error("asn.urls", "No resources found");
return None;
}
AsnGeoLookupConfig::Resource {
headers: parse_http_headers(config, "server.asn"),
expires: config.property_or_default::<Duration>("server.asn.expires", "1d")?,
timeout: config.property_or_default::<Duration>("server.asn.timeout", "5m")?,
max_size: config
.property("server.asn.max-size")
.unwrap_or(100 * 1024 * 1024),
headers: parse_http_headers(config, "asn"),
expires: config.property_or_default::<Duration>("asn.expires", "1d")?,
timeout: config.property_or_default::<Duration>("asn.timeout", "5m")?,
max_size: config.property("asn.max-size").unwrap_or(100 * 1024 * 1024),
asn_resources,
geo_resources,
}
@@ -220,7 +212,7 @@ impl AsnGeoLookupConfig {
}
"disable" | "disabled" | "none" | "false" => AsnGeoLookupConfig::Disabled.into(),
_ => {
config.new_build_error("server.asn.type", "Invalid value");
config.new_build_error("asn.type", "Invalid value");
None
}
}

54
crates/common/src/manager/boot.rs
View File

@@ -249,34 +249,32 @@ impl BootManager {
}
}) {
Some(Err(_)) => {
if std::env::var("DO_NOT_MIGRATE").is_err() {
let _ = manager.clear_prefix("lookup.spam-").await;
let _ = manager
.clear_prefix("sieve.trusted.scripts.spam-filter")
.await;
let _ = manager
.clear_prefix("sieve.trusted.scripts.track-replies")
.await;
let _ = manager.clear_prefix("sieve.trusted.scripts.greylist").await;
let _ = manager.clear_prefix("sieve.trusted.scripts.train").await;
let _ = manager.clear_prefix("session.data.script").await;
let _ = manager.clear("version.spam-filter").await;
let _ = manager.clear_prefix("lookup.spam-").await;
let _ = manager
.clear_prefix("sieve.trusted.scripts.spam-filter")
.await;
let _ = manager
.clear_prefix("sieve.trusted.scripts.track-replies")
.await;
let _ = manager.clear_prefix("sieve.trusted.scripts.greylist").await;
let _ = manager.clear_prefix("sieve.trusted.scripts.train").await;
//let _ = manager.clear_prefix("session.data.script").await;
let _ = manager.clear("version.spam-filter").await;
match manager.fetch_spam_rules().await {
Ok(external_config) => {
trc::event!(
Config(trc::ConfigEvent::ImportExternal),
Version = external_config.version.to_string(),
Id = "spam-filter"
);
insert_keys.extend(external_config.keys);
}
Err(err) => {
config.new_build_error(
"*",
format!("Failed to fetch spam filter: {err}"),
);
}
match manager.fetch_spam_rules().await {
Ok(external_config) => {
trc::event!(
Config(trc::ConfigEvent::ImportExternal),
Version = external_config.version.to_string(),
Id = "spam-filter"
);
insert_keys.extend(external_config.keys);
}
Err(err) => {
config.new_build_error(
"*",
format!("Failed to fetch spam filter: {err}"),
);
}
}
@@ -414,7 +412,7 @@ impl BootManager {
.property_or_default::<bool>("spam-filter.auto-update", "false")
.unwrap_or_default()
{
if let Err(err) = core.storage.config.update_spam_rules(false).await {
if let Err(err) = core.storage.config.update_spam_rules(false, false).await {
trc::event!(
Resource(trc::ResourceEvent::Error),
Details = "Failed to update spam-filter",

66
crates/common/src/manager/config.rs
View File

@@ -329,24 +329,57 @@ impl ConfigManager {
})
}
pub async fn update_spam_rules(&self, overwrite: bool) -> trc::Result<Option<Semver>> {
let external = self.fetch_spam_rules().await.map_err(|reason| {
pub async fn update_spam_rules(
&self,
force_update: bool,
overwrite: bool,
) -> trc::Result<Option<Semver>> {
let current_version = self
.get("version.spam-filter")
.await?
.and_then(|v| Semver::try_from(v.as_str()).ok());
let is_update = current_version.is_some();
let mut external = self.fetch_spam_rules().await.map_err(|reason| {
trc::EventType::Config(trc::ConfigEvent::FetchError)
.caused_by(trc::location!())
.details("Failed to update spam filter rules")
.ctx(trc::Key::Reason, reason)
})?;
if self.get("version.spam-filter").await?.map_or(true, |v| {
v.as_str().try_into().map_or(true, |v| external.version > v)
}) {
// Delete previous STWT_* rules
for prefix in [
"spam-filter.rule.stwt_",
"spam-filter.dnsbl.server.stwt_",
"http-lookup.stwt_",
] {
self.clear_prefix(prefix).await?;
if current_version.map_or(true, |v| external.version > v || force_update) {
if is_update {
// Delete previous STWT_* rules
let mut rule_settings = AHashMap::new();
for prefix in [
"spam-filter.rule.stwt_",
"spam-filter.dnsbl.server.stwt_",
"http-lookup.stwt_",
] {
for (key, value) in self.list(prefix, false).await? {
if key.ends_with(".enable") {
rule_settings.insert(key, value);
}
}
self.clear_prefix(prefix).await?;
}
// Update keys
if !rule_settings.is_empty() {
for key in &mut external.keys {
if let Some(value) = rule_settings.remove(&key.key) {
key.value = value;
}
}
}
if !overwrite {
// Do not overwrite ASN or LLM settings
external.keys.retain(|key| {
!key.key.starts_with("spam-filter.llm.") && !key.key.starts_with("asn.")
});
}
}
self.set(external.keys, overwrite).await?;
@@ -391,16 +424,9 @@ impl ConfigManager {
} else if key.starts_with("spam-filter.")
|| key.starts_with("http-lookup.")
|| (key.starts_with("lookup.") && !key.starts_with("lookup.default."))
|| key.starts_with("server.asn.")
|| key.starts_with("asn.")
{
external.keys.push(ConfigKey::from((key, value)));
} else {
trc::event!(
Config(trc::ConfigEvent::ExternalKeyIgnored),
Key = key,
Value = value,
Id = "spam-filter",
);
}
}