CLI account management + Directory refactoring

2023-12-18 22:25:42 +01:00
parent 566a2a0ab8
commit ea94de6d77
91 changed files with 3531 additions and 1846 deletions
--- a/resources/config/directory/internal.toml
+++ b/resources/config/directory/internal.toml
@@ -0,0 +1,17 @@
+#############################################
+# Internal Directory configuration
+#############################################
+
+[directory."internal"]
+type = "internal"
+store = "sqlite"
+
+[directory."internal".options]
+catch-all = true
+#catch-all = { map = "(.+)@(.+)$", to = "info@${2}" }
+subaddressing = true
+#subaddressing = { map = "^([^.]+)\.([^.]+)@(.+)$", to = "${2}@${3}" }
+
+[directory."internal".cache]
+entries = 500
+ttl = {positive = '1h', negative = '10m'}
--- a/resources/config/directory/ldap.toml
+++ b/resources/config/directory/ldap.toml
@@ -2,46 +2,51 @@
 # LDAP Directory configuration
 #############################################

-[directory."default"]
+[directory."ldap"]
 type = "ldap"
 address = "ldap://localhost:389"
 base-dn = "dc=example,dc=org"

-[directory."default".bind]
+[directory."ldap".bind]
 dn = "cn=serviceuser,ou=svcaccts,dc=example,dc=org"
 secret = "mysecret"

-[directory."default".cache]
+[directory."ldap".bind.auth]
+enable = false
+dn = "cn=?,ou=svcaccts,dc=example,dc=org"
+
+[directory."ldap".tls]
+enable = false
+allow-invalid-certs = false
+
+[directory."ldap".cache]
 entries = 500
 ttl = {positive = '1h', negative = '10m'}

-[directory."default".options]
+[directory."ldap".options]
 catch-all = true
 #catch-all = { map = "(.+)@(.+)$", to = "info@${2}" }
 subaddressing = true
 #subaddressing = { map = "^([^.]+)\.([^.]+)@(.+)$", to = "${2}@${3}" }
-superuser-group = "superusers"

-[directory."default".pool]
+[directory."ldap".pool]
 max-connections = 10
-min-connections = 0
-max-lifetime = "30m"
-idle-timeout = "10m"
-connect-timeout = "30s"

-[directory."default".filter]
+[directory."ldap".pool.timeout]
+create = "30s"
+wait = "30s"
+recycle = "30s"
+
+[directory."ldap".filter]
 name = "(&(|(objectClass=posixAccount)(objectClass=posixGroup))(uid=?))"
 email = "(&(|(objectClass=posixAccount)(objectClass=posixGroup))(|(mail=?)(mailAlias=?)(mailList=?)))"
 verify = "(&(|(objectClass=posixAccount)(objectClass=posixGroup))(|(mail=*?*)(mailAlias=*?*)))"
 expand = "(&(|(objectClass=posixAccount)(objectClass=posixGroup))(mailList=?))"
 domains = "(&(|(objectClass=posixAccount)(objectClass=posixGroup))(|(mail=*@?)(mailAlias=*@?)))"

-[directory."default".object-classes]
-user = "posixAccount"
-group = "posixGroup"
-
-[directory."default".attributes]
+[directory."ldap".attributes]
 name = "uid"
+type = "objectClass"
 description = ["principalName", "description"]
 secret = "userPassword"
 groups = ["memberOf", "otherGroups"]
--- a/resources/config/directory/memory.toml
+++ b/resources/config/directory/memory.toml
@@ -2,54 +2,55 @@
 # In-Memory Directory configuration
 #############################################

-[directory."default"]
+[directory."memory"]
 type = "memory"

-[directory."default".options]
+[directory."memory".options]
 catch-all = true
 #catch-all = { map = "(.+)@(.+)$", to = "info@${2}" }
 subaddressing = true
 #subaddressing = { map = "^([^.]+)\.([^.]+)@(.+)$", to = "${2}@${3}" }
-superuser-group = "superusers"

-[[directory."default".principals]]
+[[directory."memory".principals]]
 name = "admin"
+type = "admin"
 description = "Superuser"
 secret = "changeme"
 email = ["postmaster@%{DEFAULT_DOMAIN}%"]
-member-of = ["superusers"]

-[[directory."default".principals]]
+[[directory."memory".principals]]
 name = "john"
+type = "individual"
 description = "John Doe"
 secret = "12345"
 email = ["john@%{DEFAULT_DOMAIN}%", "jdoe@%{DEFAULT_DOMAIN}%", "john.doe@%{DEFAULT_DOMAIN}%"]
 email-list = ["info@%{DEFAULT_DOMAIN}%"]
 member-of = ["sales"]

-[[directory."default".principals]]
+[[directory."memory".principals]]
 name = "jane"
+type = "individual"
 description = "Jane Doe"
 secret = "abcde"
 email = ["jane@%{DEFAULT_DOMAIN}%", "jane.doe@%{DEFAULT_DOMAIN}%"]
 email-list = ["info@%{DEFAULT_DOMAIN}%"]
 member-of = ["sales", "support"]

-[[directory."default".principals]]
+[[directory."memory".principals]]
 name = "bill"
+type = "individual"
 description = "Bill Foobar"
 secret = "$2y$05$bvIG6Nmid91Mu9RcmmWZfO5HJIMCT8riNW0hEp8f6/FuA2/mHZFpe"
 quota = 50000000
 email = ["bill@%{DEFAULT_DOMAIN}%", "bill.foobar@%{DEFAULT_DOMAIN}%"]
 email-list = ["info@%{DEFAULT_DOMAIN}%"]

-[[directory."default".principals]]
+[[directory."memory".principals]]
 name = "sales"
+type = "group"
 description = "Sales Team"

-[[directory."default".principals]]
+[[directory."memory".principals]]
 name = "support"
+type = "group"
 description = "Support Team"
-
-[directory."default".lookup]
-domains = ["%{DEFAULT_DOMAIN}%"]
--- a/resources/config/directory/sql.toml
+++ b/resources/config/directory/sql.toml
@@ -2,39 +2,22 @@
 # SQL Directory configuration
 #############################################

-[directory."default"]
+[directory."sql"]
 type = "sql"
-address = "sqlite://%{BASE_PATH}%/data/accounts.sqlite3?mode=rwc"
+store = "sqlite"

-[directory."default".options]
+[directory."sql".options]
 catch-all = true
 #catch-all = { map = "(.+)@(.+)$", to = "info@${2}" }
 subaddressing = true
 #subaddressing = { map = "^([^.]+)\.([^.]+)@(.+)$", to = "${2}@${3}" }
-superuser-group = "superusers"

-[directory."default".pool]
-max-connections = 10
-min-connections = 0
-#idle-timeout = "10m"
-
-[directory."default".cache]
+[directory."sql".cache]
 entries = 500
 ttl = {positive = '1h', negative = '10m'}

-[directory."default".query]
-name = "SELECT name, type, secret, description, quota FROM accounts WHERE name = ? AND active = true"
-members = "SELECT member_of FROM group_members WHERE name = ?"
-recipients = "SELECT name FROM emails WHERE address = ?"
-emails = "SELECT address FROM emails WHERE name = ? AND type != 'list' ORDER BY type DESC, address ASC"
-verify = "SELECT address FROM emails WHERE address LIKE '%' || ? || '%' AND type = 'primary' ORDER BY address LIMIT 5"
-expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = ? AND l.type = 'list' ORDER BY p.address LIMIT 50"
-domains = "SELECT 1 FROM emails WHERE address LIKE '%@' || ? LIMIT 1"
-
-[directory."default".columns]
-name = "name"
-description = "description"
-secret = "secret"
-email = "address"
-quota = "quota"
+[directory."sql".columns]
 type = "type"
+secret = "secret"
+description = "description"
+quota = "quota"
--- a/resources/config/jmap/auth.toml
+++ b/resources/config/jmap/auth.toml
@@ -8,3 +8,6 @@ directory = "default"
 [jmap.session.cache]
 ttl = "1h"
 size = 100
+
+[jmap.session.purge]
+frequency = "15 * *"
--- a/resources/config/jmap/store.toml
+++ b/resources/config/jmap/store.toml
@@ -2,31 +2,10 @@
 # JMAP server store configuration
 #############################################

-[store.db]
-path = "%{BASE_PATH}%/data/index.sqlite3"
-
-[store.db.pool]
-max-connections = 10
-#workers = 8
-
-[store.db.cache]
-size = 1000
-
-[store.blob]
-type = "local"
-
-[store.blob.local]
-path = "%{BASE_PATH}%/data/blobs"
-
-[store.blob.s3]
-bucket = "stalwart"
-region = "eu-central-1"
-access-key = "minioadmin"
-secret-key = "minioadmin"
-#endpoint = ""
-#security-token = ""
-#profile = ""
-timeout = "30s"
+[jmap.store]
+data = "rocksdb"
+fts = "rocksdb"
+blob = "rocksdb"

 [jmap.encryption]
 enable = true
@@ -35,7 +14,5 @@ append = false
 [jmap.fts]
 default-language = "en"

-[jmap.purge.schedule]
-db = "0 3 *"
-blobs = "30 3 *"
-sessions = "15 * *"
+[jmap.cluster]
+node-id = 1
--- a/resources/config/smtp/spamfilter.toml
+++ b/resources/config/smtp/spamfilter.toml
@@ -2,83 +2,57 @@
 # SMTP Spam & Phishing filter configuration
 #############################################

-[directory."spamdb"]
-type = "sql"
-address = "sqlite://%{BASE_PATH}%/data/spamfilter.sqlite3?mode=rwc"
-
-[directory."spamdb".pool]
-max-connections = 10
-min-connections = 0
-idle-timeout = "5m"
-
-[directory."spamdb".lookup]
-token-insert = "INSERT INTO bayes_tokens (h1, h2, ws, wh) VALUES (?, ?, ?, ?) 
-                ON CONFLICT(h1, h2) 
-                DO UPDATE SET ws = ws + excluded.ws, wh = wh + excluded.wh"
-token-lookup = "SELECT ws, wh FROM bayes_tokens WHERE h1 = ? AND h2 = ?"
-id-insert = "INSERT INTO seen_ids (id, ttl) VALUES (?, datetime('now', ? || ' seconds'))"
-id-lookup = "SELECT 1 FROM seen_ids WHERE id = ? AND ttl > CURRENT_TIMESTAMP"
-reputation-insert = "INSERT INTO reputation (token, score, count, ttl) VALUES (?, ?, 1, datetime('now', '30 days')) 
-                     ON CONFLICT(token) 
-                     DO UPDATE SET score = (count + 1) * (excluded.score + 0.98 * score) / (0.98 * count + 1), count = count + 1, ttl = excluded.ttl"
-reputation-lookup = "SELECT score, count FROM reputation WHERE token = ?"
-
-[directory."spamdb".schedule]
-query = ["DELETE FROM seen_ids WHERE ttl < CURRENT_TIMESTAMP", 
-         "DELETE FROM reputation WHERE ttl < CURRENT_TIMESTAMP"]
-frequency = "0 3 *"
-
-[directory."spam"]
+[store."spam"]
 type = "memory"

-[directory."spam".lookup."free-domains"]
+[store."spam".lookup."free-domains"]
 type = "glob"
 comment = '#'
 values = ["https://get.stalw.art/resources/config/spamfilter/maps/domains_free.list", 
          "file+fallback://%{BASE_PATH}%/etc/spamfilter/maps/domains_free.list"]

-[directory."spam".lookup."disposable-domains"]
+[store."spam".lookup."disposable-domains"]
 type = "glob"
 comment = '#'
 values = ["https://get.stalw.art/resources/config/spamfilter/maps/domains_disposable.list", 
          "file+fallback://%{BASE_PATH}%/etc/spamfilter/maps/domains_disposable.list"]

-[directory."spam".lookup."redirectors"]
+[store."spam".lookup."redirectors"]
 type = "glob"
 comment = '#'
 values = ["https://get.stalw.art/resources/config/spamfilter/maps/url_redirectors.list", 
          "file+fallback://%{BASE_PATH}%/etc/spamfilter/maps/url_redirectors.list"]

-[directory."spam".lookup."domains-allow"]
+[store."spam".lookup."domains-allow"]
 type = "glob"
 comment = '#'
 values = ["https://get.stalw.art/resources/config/spamfilter/maps/allow_domains.list", 
          "file+fallback://%{BASE_PATH}%/etc/spamfilter/maps/allow_domains.list"]

-[directory."spam".lookup."dmarc-allow"]
+[store."spam".lookup."dmarc-allow"]
 type = "glob"
 comment = '#'
 values = ["https://get.stalw.art/resources/config/spamfilter/maps/allow_dmarc.list", 
          "file+fallback://%{BASE_PATH}%/etc/spamfilter/maps/allow_dmarc.list"]

-[directory."spam".lookup."spf-dkim-allow"]
+[store."spam".lookup."spf-dkim-allow"]
 type = "glob"
 comment = '#'
 values = ["https://get.stalw.art/resources/config/spamfilter/maps/allow_spf_dkim.list", 
          "file+fallback://%{BASE_PATH}%/etc/spamfilter/maps/allow_spf_dkim.list"]

-[directory."spam".lookup."mime-types"]
+[store."spam".lookup."mime-types"]
 type = "map"
 comment = '#'
 values = ["https://get.stalw.art/resources/config/spamfilter/maps/mime_types.map", 
          "file+fallback://%{BASE_PATH}%/etc/spamfilter/maps/mime_types.map"]

-[directory."spam".lookup."trap-address"]
+[store."spam".lookup."trap-address"]
 type = "glob"
 comment = '#'
 values = "file://%{BASE_PATH}%/etc/spamfilter/maps/spam_trap.list"

-[directory."spam".lookup."scores"]
+[store."spam".lookup."scores"]
 type = "map"
 values = "file://%{BASE_PATH}%/etc/spamfilter/maps/scores.map"

--- a/resources/config/store/elastic.toml
+++ b/resources/config/store/elastic.toml
@@ -0,0 +1,17 @@
+#############################################
+# ElasticSearch FTS Store configuration
+#############################################
+
+[store."elastic"]
+type = "elasticsearch"
+url = "https://localhost:9200"
+user = "elastic"
+password = "myelasticpassword"
+#cloud-id = "my-cloud-id"
+
+[store."elastic".tls]
+allow-invalid-certs = true
+
+[store."elastic".index]
+shards = 3
+replicas = 0
--- a/resources/config/store/filesystem.toml
+++ b/resources/config/store/filesystem.toml
@@ -0,0 +1,11 @@
+#############################################
+# File System Blob Store configuration
+#############################################
+
+[store."fs"]
+type = "fs"
+path = "%{BASE_PATH}%/data/blobs"
+depth = 2
+
+[store."fs".purge]
+frequency = "0 3 *"
--- a/resources/config/store/foundationdb.toml
+++ b/resources/config/store/foundationdb.toml
@@ -0,0 +1,17 @@
+#############################################
+# FoundationDB Store configuration
+#############################################
+
+[store."foundationdb"]
+type = "foundationdb"
+#path = "/etc/foundationdb/fdb.cluster"
+
+#[store."foundationdb".transaction]
+#timeout = "5s"
+#retry-limit = 10
+#max-retry-delay = "1s"
+#machine-id = "stalwart"
+#data-center-id = "my-datacenter"
+
+[store."foundationdb".purge]
+frequency = "0 3 *"
--- a/resources/config/store/mysql.toml
+++ b/resources/config/store/mysql.toml
@@ -0,0 +1,37 @@
+#############################################
+# MySQL Store configuration
+#############################################
+
+[store."mysql"]
+type = "mysql"
+host = "localhost"
+port = 3307
+database = "stalwart"
+user = "root"
+password = "password"
+
+[store."mysql".timeout]
+wait = "15s"
+
+#[store."postgresql".pool]
+#max-connections = 10
+#min-connections = 5
+
+#[store."mysql".init]
+#execute = [
+#    "CREATE TABLE IF NOT EXISTS accounts (name VARCHAR(32) PRIMARY KEY, secret VARCHAR(1024), description VARCHAR(1024), type VARCHAR(32) NOT NULL, quota INTEGER DEFAULT 0, active BOOLEAN DEFAULT 1)",
+#    "CREATE TABLE IF NOT EXISTS group_members (name VARCHAR(32) NOT NULL, member_of VARCHAR(32) NOT NULL, PRIMARY KEY (name, member_of))",
+#    "CREATE TABLE IF NOT EXISTS emails (name VARCHAR(32) NOT NULL, address VARCHAR(128) NOT NULL, type VARCHAR(32), PRIMARY KEY (name, address))"
+#]
+
+[store."mysql".query]
+name = "SELECT name, type, secret, description, quota FROM accounts WHERE name = ? AND active = true"
+members = "SELECT member_of FROM group_members WHERE name = ?"
+recipients = "SELECT name FROM emails WHERE address = ? ORDER BY name ASC"
+emails = "SELECT address FROM emails WHERE name = ? AND type != 'list' ORDER BY type DESC, address ASC"
+verify = "SELECT address FROM emails WHERE address LIKE CONCAT('%', ?, '%') AND type = 'primary' ORDER BY address LIMIT 5"
+expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = ? AND l.type = 'list' ORDER BY p.address LIMIT 50"
+domains = "SELECT 1 FROM emails WHERE address LIKE CONCAT('%@', ?) LIMIT 1"
+
+[store."mysql".purge]
+frequency = "0 3 *"
--- a/resources/config/store/postgresql.toml
+++ b/resources/config/store/postgresql.toml
@@ -0,0 +1,40 @@
+#############################################
+# PostgreSQL Store configuration
+#############################################
+
+[store."postgresql"]
+type = "postgresql"
+host = "localhost"
+port = 5432
+database = "stalwart"
+user = "postgres"
+password = "mysecretpassword"
+
+[store."postgresql".timeout]
+connect = "15s"
+
+[store."postgresql".tls]
+enable = false
+allow-invalid-certs = false
+
+#[store."postgresql".pool]
+#max-connections = 10
+
+#[store."postgresql".init]
+#execute = [
+#    "CREATE TABLE IF NOT EXISTS accounts (name TEXT PRIMARY KEY, secret TEXT, description TEXT, type TEXT NOT NULL, quota INTEGER DEFAULT 0, active BOOLEAN DEFAULT 1)",
+#    "CREATE TABLE IF NOT EXISTS group_members (name TEXT NOT NULL, member_of TEXT NOT NULL, PRIMARY KEY (name, member_of))",
+#    "CREATE TABLE IF NOT EXISTS emails (name TEXT NOT NULL, address TEXT NOT NULL, type TEXT, PRIMARY KEY (name, address))"
+#]
+
+[store."postgresql".query]
+name = "SELECT name, type, secret, description, quota FROM accounts WHERE name = $1 AND active = true"
+members = "SELECT member_of FROM group_members WHERE name = $1"
+recipients = "SELECT name FROM emails WHERE address = $1 ORDER BY name ASC"
+emails = "SELECT address FROM emails WHERE name = $1 AND type != 'list' ORDER BY type DESC, address ASC"
+verify = "SELECT address FROM emails WHERE address LIKE '%' || $1 || '%' AND type = 'primary' ORDER BY address LIMIT 5"
+expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = $1 AND l.type = 'list' ORDER BY p.address LIMIT 50"
+domains = "SELECT 1 FROM emails WHERE address LIKE '%@' || $1 LIMIT 1"
+
+[store."postgresql".purge]
+frequency = "0 3 *"
--- a/resources/config/store/redis.toml
+++ b/resources/config/store/redis.toml
@@ -0,0 +1,15 @@
+#############################################
+# Redis Lookup Store configuration
+#############################################
+
+[store."redis"]
+type = "redis"
+url = "redis://127.0.0.1"
+#urls = ["redis://192.168.1.1", "redis://192.168.1.1"] # for Redis cluster
+username = "my_username"
+password = "secretpassword"
+timeout = "10s"
+#retries = 3
+#max-retry-wait = "1s"
+#min-retry-wait = "500ms"
+#read-from-replicas = false
--- a/resources/config/store/rocksdb.toml
+++ b/resources/config/store/rocksdb.toml
@@ -0,0 +1,17 @@
+#############################################
+# RocksDB Store configuration
+#############################################
+
+[store."rocksdb"]
+type = "rocksdb"
+path = "%{BASE_PATH}%/data"
+
+[store."rocksdb".settings]
+min-blob-size = 16834
+write-buffer-size = 134217728
+
+#[store."rocksdb".pool]
+#workers = 10
+
+[store."rocksdb".purge]
+frequency = "0 3 *"
--- a/resources/config/store/s3.toml
+++ b/resources/config/store/s3.toml
@@ -0,0 +1,17 @@
+#############################################
+# S3/MinIO Blob Store configuration
+#############################################
+
+[store."s3"]
+type = "s3"
+bucket = "stalwart"
+region = "eu-central-1"
+access-key = "minioadmin"
+secret-key = "minioadmin"
+#endpoint = ""
+#security-token = ""
+#profile = ""
+timeout = "30s"
+
+[store."s3".purge]
+frequency = "0 3 *"
--- a/resources/config/store/sqlite.toml
+++ b/resources/config/store/sqlite.toml
@@ -0,0 +1,30 @@
+#############################################
+# SQLite Store configuration
+#############################################
+
+[store."sqlite"]
+type = "sqlite"
+path = "%{BASE_PATH}%/data/index.sqlite3"
+
+#[store."sqlite".pool]
+#max-connections = 10
+#workers = 10
+
+#[store."sqlite".init]
+#execute = [
+#    "CREATE TABLE IF NOT EXISTS accounts (name TEXT PRIMARY KEY, secret TEXT, description TEXT, type TEXT NOT NULL, quota INTEGER DEFAULT 0, active BOOLEAN DEFAULT 1)",
+#    "CREATE TABLE IF NOT EXISTS group_members (name TEXT NOT NULL, member_of TEXT NOT NULL, PRIMARY KEY (name, member_of))",
+#    "CREATE TABLE IF NOT EXISTS emails (name TEXT NOT NULL, address TEXT NOT NULL, type TEXT, PRIMARY KEY (name, address))"
+#]
+
+[store."sqlite".query]
+name = "SELECT name, type, secret, description, quota FROM accounts WHERE name = ? AND active = true"
+members = "SELECT member_of FROM group_members WHERE name = ?"
+recipients = "SELECT name FROM emails WHERE address = ?"
+emails = "SELECT address FROM emails WHERE name = ? AND type != 'list' ORDER BY type DESC, address ASC"
+verify = "SELECT address FROM emails WHERE address LIKE '%' || ? || '%' AND type = 'primary' ORDER BY address LIMIT 5"
+expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = ? AND l.type = 'list' ORDER BY p.address LIMIT 50"
+domains = "SELECT 1 FROM emails WHERE address LIKE '%@' || ? LIMIT 1"
+
+[store."sqlite".purge]
+frequency = "0 3 *"