filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Spam filter combined tests passing

Browse Source
This commit is contained in:
mdecimus
2023-10-20 19:18:18 +02:00
parent b5409a0b00
commit f9b37a3c99
25 changed files with 229 additions and 357 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
tests/resources/create_test_env.sh
View File

@@ -1,6 +1,7 @@
#!/bin/sh
#!/bin/bash
BASE_DIR = "/tmp/stalwart-test"
BASE_DIR="/tmp/stalwart-test"
DOMAIN="example.org"
# Delete previous tests
rm -rf $BASE_DIR
@@ -15,3 +16,37 @@ cp -r resources/config $BASE_DIR/etc
cp -r tests/resources/tls_cert.pem $BASE_DIR/etc
cp -r tests/resources/tls_privatekey.pem $BASE_DIR/etc
# Replace settings
sed -i '' -e "s/__DOMAIN__/$DOMAIN/g" -e "s/__HOST__/mail.$DOMAIN/g" -e 's/sql.toml/memory.toml/g' -e "s|__BASE_PATH__|$BASE_DIR|g" "$BASE_DIR/etc/config.toml"
sed -i '' -e "s|__CERT_PATH__|$BASE_DIR/etc/tls_cert.pem|g" -e "s|__PK_PATH__|$BASE_DIR/etc/tls_privatekey.pem|g" "$BASE_DIR/etc/common/tls.toml"
sed -i '' -e 's/method = "log"/method = "stdout"/g' -e 's/level = "info"/level = "trace"/g' "$BASE_DIR/etc/common/tracing.toml"
sed -i '' -e 's/user = "stalwart-mail"//g' -e 's/group = "stalwart-mail"//g' "$BASE_DIR/etc/common/server.toml"
# Generate DKIM key
mkdir -p $BASE_DIR/etc/dkim
openssl genpkey -algorithm RSA -out $BASE_DIR/etc/dkim/$DOMAIN.key
# Create antispam tables
sqlite3 $BASE_DIR/data/spamfilter.sqlite3 <<EOF
CREATE TABLE IF NOT EXISTS bayes_tokens (
h1 INTEGER NOT NULL,
h2 INTEGER NOT NULL,
ws INTEGER,
wh INTEGER,
PRIMARY KEY (h1, h2)
);
CREATE TABLE IF NOT EXISTS seen_ids (
id STRING NOT NULL PRIMARY KEY,
ttl DATETIME NOT NULL
);
CREATE TABLE IF NOT EXISTS reputation (
token STRING NOT NULL PRIMARY KEY,
score FLOAT NOT NULL DEFAULT '0',
count INT(11) NOT NULL DEFAULT '0',
ttl DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
EOF
#cargo run --manifest-path=crates/main/Cargo.toml -- --config=/tmp/stalwart-test/etc/config.toml

10
tests/resources/smtp/antispam/combined.test
View File

@@ -6,7 +6,9 @@ spf.result none
spf_ehlo.result none
dmarc.result none
remote_ip 195.210.29.48
expect hfilter_helo_nores_a_or_mx once_received mid_rhs_match_from r_spf_na has_data_uri arc_na subject_has_exclaim subject_ends_exclaim rdns_dnsfail mime_html_only html_short_link_img_1 to_dn_none rcpt_count_one to_match_envrcpt_all hfilter_fromhost_nores_a_or_mx rcvd_count_zero from_eq_envfrom r_dkim_na rcvd_no_tls_last from_has_dn date_in_past
expect_header X-Spam-Status Yes, score=7.
expect_header X-Spam-Result
expect auth_na dmarc_na hfilter_helo_nores_a_or_mx once_received mid_rhs_match_from r_spf_na has_data_uri arc_na subject_has_exclaim subject_ends_exclaim rdns_dnsfail mime_html_only html_short_link_img_1 to_dn_none rcpt_count_one to_match_envrcpt_all hfilter_fromhost_nores_a_or_mx rcvd_count_zero from_eq_envfrom r_dkim_na rcvd_no_tls_last from_has_dn date_in_past
From: Client Services <noreply@tetheer.com>
To: licensing@stalw.art
@@ -48,6 +50,8 @@ dkim.domains tenthrevolution.com
dmarc.result pass
remote_ip 185.58.86.181
tls.version TLSv1.3
expect_header X-Spam-Status No, score=4.
expect_header X-Spam-Result
expect from_eq_envfrom from_has_dn hfilter_helo_nores_a_or_mx forged_rcvd_trail date_in_past arc_na uri_count_odd dkim_signed has_attachment r_spf_allow rcvd_tls_last rcpt_count_one mime_good subject_ends_spaces hfilter_fromhost_nores_a_or_mx to_dn_eq_addr_all r_dkim_allow dmarc_policy_allow rcvd_count_three to_match_envrcpt_all
DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=tenthrevolution.com;
@@ -621,7 +625,9 @@ dmarc.result fail
dmarc.policy reject
remote_ip 51.89.165.39
tls.version TLS1_2
expect has_replyto once_received r_parts_differ mid_rhs_match_from hfilter_fromhost_nores_a_or_mx from_has_dn r_dkim_allow date_in_past to_match_envrcpt_all html_short_link_img_1 rcpt_count_one arc_na hfilter_helo_nores_a_or_mx r_spf_softfail rcvd_tls_last rcvd_count_zero replyto_dom_eq_from_dom to_dn_none has_list_unsub dkim_signed rdns_none from_eq_envfrom dmarc_policy_reject
expect_header X-Spam-Status Yes, score=13.
expect_header X-Spam-Result
expect has_replyto violated_direct_spf replyto_addr_eq_from once_received r_parts_differ mid_rhs_match_from hfilter_fromhost_nores_a_or_mx from_has_dn r_dkim_allow date_in_past to_match_envrcpt_all html_short_link_img_1 rcpt_count_one arc_na hfilter_helo_nores_a_or_mx r_spf_softfail rcvd_tls_last rcvd_count_zero replyto_dom_eq_from_dom to_dn_none has_list_unsub dkim_signed rdns_none from_eq_envfrom dmarc_policy_reject
DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; s=sectionalism; d=grupokonecta.net;
h=To:Subject:Message-ID:Date:From:Reply-To:MIME-Version:List-Unsubscribe:

236
tests/resources/test_config.toml
View File

@@ -1,236 +0,0 @@
[server]
hostname = "test.example.org"
[server.listener.jmap]
bind = ["0.0.0.0:9990"]
url = "https://0.0.0.0:9990"
protocol = "jmap"
max-connections = 8192
[server.listener.imap]
bind = ["0.0.0.0:9991"]
protocol = "imap"
max-connections = 8192
[server.listener.imaptls]
bind = ["0.0.0.0:9992"]
protocol = "imap"
max-connections = 8192
tls.implicit = true
[server.listener.sieve]
bind = ["0.0.0.0:9993"]
protocol = "managesieve"
max-connections = 8192
tls.implicit = true
[server.listener.smtps]
bind = ['0.0.0.0:9994']
greeting = 'Test SMTP instance'
protocol = 'smtp'
tls.implicit = true
[server.listener.smtp]
bind = ['0.0.0.0:9995']
greeting = 'Test SMTP instance'
protocol = 'smtp'
tls.implicit = false
[server.socket]
reuse-addr = true
[server.tls]
enable = true
implicit = false
certificate = "default"
[global.tracing]
method = "stdout"
#level = "trace"
level = "info"
[session.ehlo]
reject-non-fqdn = false
[session.rcpt]
relay = [ { if = "authenticated-as", ne = "", then = true },
{ else = false } ]
directory = "local"
[session.rcpt.errors]
total = 5
wait = "1ms"
[queue]
path = "/tmp/stalwart-test"
hash = 64
[report]
path = "/tmp/stalwart-test"
hash = 64
[resolver]
type = "system"
[queue.outbound]
next-hop = [ { if = "rcpt-domain", in-list = "local/domains", then = "local" },
{ else = false } ]
[remote."mock-smtp"]
address = "localhost"
port = 9999
protocol = "smtp"
[remote."mock-smtp".tls]
implicit = false
allow-invalid-certs = true
[session.extensions]
future-release = [ { if = "authenticated-as", ne = "", then = "99999999d"},
{ else = false } ]
[store]
db.path = "/tmp/stalwart-test/sqlite.db"
[store.blob]
type = "local"
[store.blob.local]
path = "/tmp/stalwart-test"
[certificate.default]
cert = "file://./tests/resources/tls_cert.pem"
private-key = "file://./tests/resources/tls_privatekey.pem"
[jmap]
directory = "local"
[jmap.http]
headers = ["Access-Control-Allow-Origin: *",
"Access-Control-Allow-Methods: POST, GET, HEAD, OPTIONS",
"Access-Control-Allow-Headers: *"]
[jmap.protocol]
set.max-objects = 100000
[jmap.protocol.request]
max-concurrent = 8
[jmap.protocol.upload]
max-size = 5000000
max-concurrent = 4
ttl = "1m"
[jmap.protocol.upload.quota]
files = 3
size = 50000
[jmap.rate-limit]
account = "1000/1m"
authentication = "100/2s"
anonymous = "100/1m"
[jmap.event-source]
throttle = "500ms"
[jmap.web-sockets]
throttle = "500ms"
[jmap.push]
throttle = "500ms"
attempts.interval = "500ms"
[directory."local"]
type = "memory"
[directory."local".options]
catch-all = true
subaddressing = true
[directory."local".lookup]
domains = ["example.org"]
[[directory."local".users]]
name = "admin"
description = "Superadmin"
secret = "secret"
member-of = ["superusers"]
[[directory."local".users]]
name = "john"
description = "John Doe"
secret = "12345"
#secret = "$argon2id$v=19$m=16,t=2,p=1$Ym1GMmMwd210YXpGWUF2Ng$MKUyI28a4OADfd7r2iHulQ"
email = ["john@example.org", "jdoe@example.org", "john.doe@example.org"]
email-list = ["info@example.org"]
member-of = ["sales"]
[[directory."local".users]]
name = "jane"
description = "Jane Doe"
secret = "12345"
email = "jane@example.org"
email-list = ["info@example.org"]
member-of = ["sales", "support"]
[[directory."local".users]]
name = "bill"
description = "Bill Foobar"
secret = "12345"
quota = 500000
email = "bill@example.org"
email-list = ["info@example.org"]
[[directory."local".groups]]
name = "sales"
email = "sales@example.org"
description = "Sales Team"
[[directory."local".groups]]
name = "support"
email = "support@example.org"
description = "Support Team"
[oauth]
key = "parerga_und_paralipomena"
[oauth.auth]
max-attempts = 1
[oauth.expiry]
user-code = "1s"
token = "1s"
refresh-token = "3s"
refresh-token-renew = "2s"
[imap.auth]
allow-plain-text = true
[imap.rate-limit]
requests = "90000/1s"
concurrent = 9000
[signature."ed25519"]
public-key = "-----BEGIN PUBLIC KEY-----
MCowBQYDK2VwAyEAUA3S0BdVG7LeA1agv7ZtqLkQMn+/AoYx1VPyGmLWEIM=
-----END PUBLIC KEY-----"
private-key = "-----BEGIN PRIVATE KEY-----
MC4CAQAwBQYDK2VwBCIEIPdRszAzLvx4JaSIE4dQdZtN9y2XW+55K+YkCJI6lcn8
-----END PRIVATE KEY-----"
domain = "example.org"
selector = "stalwart_ed"
headers = ["From", "To", "Date", "Subject", "Message-ID"]
algorithm = "ed25519-sha256"
canonicalization = "simple/simple"
set-body-length = true
report = true
[signature."ed25519-2"]
public-key = "11qYAYKxCrfVS/7TyWQHOg7hcvPapiMlrwIaaPcHURo="
private-key = "nWGxne/9WmC6hEr0kuwsxERJxWl7MmkZcDusAxyuf2A="
domain = "example.org"
selector = "stalwart_ed2"
headers = ["From", "To", "Date", "Subject", "Message-ID"]
algorithm = "ed25519-sha256"
canonicalization = "simple/simple"
set-body-length = true
report = true

66
tests/src/smtp/inbound/antispam.rs
View File

@@ -18,7 +18,7 @@ use smtp::{
inbound::AuthResult,
scripts::{
functions::html::{get_attribute, html_attr_tokens, html_img_area, html_to_tokens},
ScriptResult,
ScriptModification, ScriptResult,
},
};
use tokio::runtime::Handle;
@@ -122,7 +122,7 @@ values = ["spamtrap@*"]
[directory."spam".lookup."scores"]
type = "map"
values = ["SPAM_TRAP discard"]
values = "file://%CFG_PATH%/maps/scores.map"
[resolver]
public-suffix = "file://%LIST_PATH%/public-suffix.dat"
@@ -192,6 +192,13 @@ async fn antispam() {
];
let mut core = SMTP::test();
let qr = core.init_test_queue("smtp_antispam_test");
let base_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.parent()
.unwrap()
.to_path_buf()
.join("resources")
.join("config")
.join("spamfilter");
let mut config = CONFIG
.replace("%PATH%", qr._temp_dir.temp_dir.as_path().to_str().unwrap())
.replace(
@@ -202,15 +209,9 @@ async fn antispam() {
.join("lists")
.to_str()
.unwrap(),
);
let base_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.parent()
.unwrap()
.to_path_buf()
.join("resources")
.join("config")
.join("spamfilter")
.join("scripts");
)
.replace("%CFG_PATH%", base_path.as_path().to_str().unwrap());
let base_path = base_path.join("scripts");
let script_config = fs::read_to_string(base_path.join("config.sieve")).unwrap();
let script_prelude = fs::read_to_string(base_path.join("prelude.sieve")).unwrap();
let mut all_scripts = script_config.clone() + "\n" + script_prelude.as_str();
@@ -235,6 +236,13 @@ async fn antispam() {
"{test_name} = '''{script_config}\n{script_prelude}\n{script}\n'''\n"
));
}
for test_name in ["composites", "scores", "epilogue"] {
all_scripts = all_scripts
+ "\n"
+ fs::read_to_string(base_path.join(format!("{test_name}.sieve")))
.unwrap()
.as_str();
}
config.push_str(&format!("combined = '''{all_scripts}\n'''\n"));
@@ -331,6 +339,7 @@ async fn antispam() {
let mut in_params = true;
let mut variables: HashMap<String, Variable> = HashMap::new();
let mut expected_variables = AHashMap::new();
let mut expected_headers = AHashMap::new();
// Build session
let mut session = Session::test(core.clone());
@@ -420,6 +429,14 @@ async fn antispam() {
.unwrap_or((v.to_lowercase(), Variable::Integer(1)))
}));
}
"expect_header" => {
if let Some((header, value)) = value.split_once(' ') {
expected_headers
.insert(header.to_string(), value.trim().to_string());
} else {
expected_headers.insert(value.to_string(), String::new());
}
}
"score" | "final_score" => {
variables
.insert(param.to_string(), value.parse::<f64>().unwrap().into());
@@ -466,7 +483,32 @@ async fn antispam() {
.await
.unwrap()
{
ScriptResult::Accept { .. } => {}
ScriptResult::Accept { modifications } => {
if modifications.len() != expected_headers.len() {
panic!(
"Expected {:?} headers, got {:?}",
expected_headers, modifications
);
}
for modification in modifications {
if let ScriptModification::AddHeader { name, value } = modification {
if let Some(expected_value) = expected_headers.remove(name.as_str()) {
if !expected_value.is_empty()
&& !value.starts_with(expected_value.as_str())
{
panic!(
"Expected header {:?} to be {:?}, got {:?}",
name, expected_value, value
);
}
} else {
panic!("Unexpected header {:?}", name);
}
} else {
panic!("Unexpected modification {:?}", modification);
}
}
}
ScriptResult::Reject(message) => panic!("{}", message),
ScriptResult::Replace {
message,