RocksDB optimizations

tests/resources/scripts/create_test_env.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+BASE_DIR="/tmp/stalwart-test"
+DOMAIN="example.org"
+
+# Stores
+#STORE="foundationdb"
+#FTS_STORE="foundationdb"
+#BLOB_STORE="foundationdb"
+STORE="rocksdb"
+FTS_STORE="rocksdb"
+BLOB_STORE="rocksdb"
+
+# Directories
+DIRECTORY="internal"
+SQL_STORE="sqlite"
+
+# Delete previous tests
+rm -rf $BASE_DIR
+
+# Create directories
+mkdir -p $BASE_DIR $BASE_DIR/data $BASE_DIR/data/blobs $BASE_DIR/logs $BASE_DIR/reports $BASE_DIR/queue
+
+# Copy config files
+cp -r resources/config $BASE_DIR/etc
+
+# Copy self-signed certs
+cp -r tests/resources/tls_cert.pem $BASE_DIR/etc
+cp -r tests/resources/tls_privatekey.pem $BASE_DIR/etc
+
+# Replace stores and directories
+sed -i '' -e "s|__SQL_STORE__|$SQL_STORE|g" "$BASE_DIR/etc/directory/sql.toml"
+sed -i '' -e 's/disable = true//g' "$BASE_DIR/etc/directory/$DIRECTORY.toml"
+sed -i '' -e 's/disable = true//g' "$BASE_DIR/etc/store/$STORE.toml"
+sed -i '' -e 's/disable = true//g' "$BASE_DIR/etc/store/$FTS_STORE.toml"
+sed -i '' -e 's/disable = true//g' "$BASE_DIR/etc/store/$BLOB_STORE.toml"
+sed -i '' -e "s/__FTS_STORE__/$FTS_STORE/g" \
+          -e "s/__BLOB_STORE__/$BLOB_STORE/g" "$BASE_DIR/etc/jmap/store.toml"
+
+# Replace settings
+sed -i ''  -e "s/__STORE__/$STORE/g" \
+          -e "s/__DIRECTORY__/$DIRECTORY/g" \
+          -e "s/__DOMAIN__/$DOMAIN/g" \
+          -e "s/__HOST__/mail.$DOMAIN/g" \
+          -e "s|__BASE_PATH__|$BASE_DIR|g" "$BASE_DIR/etc/config.toml"
+sed -i '' -e "s|__CERT_PATH__|$BASE_DIR/etc/tls_cert.pem|g" \
+          -e "s|__PK_PATH__|$BASE_DIR/etc/tls_privatekey.pem|g" "$BASE_DIR/etc/common/tls.toml"
+sed -i '' -e 's/method = "log"/method = "stdout"/g' \
+          -e 's/level = "info"/level = "info"/g' "$BASE_DIR/etc/common/tracing.toml"
+sed -i '' -e 's/%{HOST}%/127.0.0.1/g' "$BASE_DIR/etc/jmap/listener.toml"
+sed -i '' -e 's/allow-plain-text = false/allow-plain-text = true/g' \
+          -e 's/2000\/1m/9999999\/100m/g' \
+          -e 's/concurrent = 4/concurrent = 90000/g' "$BASE_DIR/etc/imap/settings.toml"
+sed -i '' -e 's/user = "stalwart-mail"//g' \
+          -e 's/group = "stalwart-mail"//g' "$BASE_DIR/etc/common/server.toml"
+
+# Generate DKIM key
+mkdir -p $BASE_DIR/etc/dkim
+openssl genpkey -algorithm RSA -out $BASE_DIR/etc/dkim/$DOMAIN.key
+
+# Create admin user
+SET_ADMIN_USER="admin" SET_ADMIN_PASS="secret" cargo run -p mail-server --no-default-features --features "foundationdb postgres mysql rocks elastic s3 redis" -- --config=/tmp/stalwart-test/etc/config.toml
+cargo run -p mail-server --no-default-features --features "foundationdb postgres mysql rocks elastic s3 redis" -- --config=/tmp/stalwart-test/etc/config.toml

tests/resources/scripts/create_test_users.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret domain create example.org
+cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret account create john 12345 -d "John Doe" -a john@example.org -a john.doe@example.org
+#cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret account create jane abcde -d "Jane Doe" -a jane@example.org
+#cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret account create bill xyz12 -d "Bill Foobar" -a bill@example.org
+#cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret group create sales -d "Sales Department"
+#cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret group create support -d "Technical Support"
+#cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret account add-to-group john sales support
+#cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret account remove-from-group john support
+#cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret account add-email jane jane.doe@example.org
+#cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret list create everyone everyone@example.org
+#cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret list add-members everyone jane john bill
+#cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret account list
+#cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret import messages --format mbox john _ignore/dovecot-crlf 
+

tests/resources/scripts/imap_import.py

@@ -0,0 +1,48 @@
+import imaplib
+import socket
+import time
+import threading
+from email.message import Message
+
+def append_message(thread_id, start, end):
+    conn = imaplib.IMAP4('localhost')
+    conn.login('john', '12345')
+    conn.socket().setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+    start_time = time.time()
+
+    for n in range(start, end):
+        msg = Message()
+        msg['From'] = 'somebody@some.where'
+        msg['To'] = 'john@example.org'
+        msg['Message-Id'] = f'unique.message.id.{n}@nowhere'
+        msg['Subject'] = f"This is message #{n}"
+        msg.set_payload('...nothing...')
+
+        response_code, response_details = conn.append('INBOX', '', None, str(msg).encode('utf-8'))
+        if response_code != 'OK':
+            print(f'Thread {thread_id}: Error while appending message #{n}: {response_code} {response_details}')
+            break
+        if n != 0 and n % 100 == 0:
+          elapsed_time = (time.time() - start_time) * 1000 
+          print(f'Thread {thread_id}: Inserting batch {n} took {elapsed_time} ms.', flush=True)
+          start_time = time.time()
+
+    conn.logout()
+
+num_threads = 5
+num_messages = 10000
+messages_per_thread = num_messages // num_threads
+
+threads = []
+for i in range(num_threads):
+    start = i * messages_per_thread
+    end = start + messages_per_thread
+    thread = threading.Thread(target=append_message, args=(i, start, end))
+    threads.append(thread)
+    thread.start()
+
+for thread in threads:
+    thread.join()
+
+print("All messages appended.")
+