filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Registry crate implementation

Browse Source
This commit is contained in:
mdecimus
2026-01-23 18:44:11 +01:00
parent 2266634f36
commit acecd32c8e
43 changed files with 2258 additions and 912 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
crates/common/Cargo.toml
View File

@@ -10,6 +10,7 @@ nlp = { path = "../nlp" }
store = { path = "../store" }
trc = { path = "../trc" }
directory = { path = "../directory" }
coordinator = { path = "../coordinator" }
types = { path = "../types" }
jmap_proto = { path = "../jmap-proto" }
imap_proto = { path = "../imap-proto" }

31
crates/common/src/config/mod.rs
View File

@@ -13,6 +13,7 @@ use crate::{
listener::tls::AcmeProviders, manager::config::ConfigManager,
};
use arc_swap::ArcSwap;
use coordinator::Coordinator;
use directory::{Directories, Directory};
use groupware::GroupwareConfig;
use hyper::HeaderMap;
@@ -132,21 +133,21 @@ impl Core {
}
})
.unwrap_or_default();
let pubsub = config
.value("cluster.coordinator")
.map(|id| id.to_string())
.and_then(|id| {
if let Some(store) = stores.pubsub_stores.get(&id) {
store.clone().into()
} else {
config.new_parse_error(
"cluster.coordinator",
format!("Coordinator backend {id:?} not found"),
);
None
}
})
.unwrap_or_default();
let pubsub = Coordinator::None; /*config
.value("cluster.coordinator")
.map(|id| id.to_string())
.and_then(|id| {
if let Some(store) = stores.pubsub_stores.get(&id) {
store.clone().into()
} else {
config.new_parse_error(
"cluster.coordinator",
format!("Coordinator backend {id:?} not found"),
);
None
}
})
.unwrap_or_default();*/
let mut directories =
Directories::parse(config, &stores, data.clone(), is_enterprise).await;
let directory = config

8
crates/common/src/config/storage.rs
View File

@@ -4,11 +4,11 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use std::sync::Arc;
use ahash::AHashMap;
use coordinator::Coordinator;
use directory::Directory;
use store::{BlobStore, SearchStore, InMemoryStore, PubSubStore, PurgeSchedule, Store};
use std::sync::Arc;
use store::{BlobStore, InMemoryStore, PurgeSchedule, SearchStore, Store};
use crate::manager::config::ConfigManager;
@@ -18,7 +18,7 @@ pub struct Storage {
pub blob: BlobStore,
pub fts: SearchStore,
pub lookup: InMemoryStore,
pub pubsub: PubSubStore,
pub pubsub: Coordinator,
pub directory: Arc<Directory>,
pub directories: AHashMap<String, Arc<Directory>>,
pub purge_schedules: Vec<PurgeSchedule>,

1
crates/common/src/manager/reload.rs
View File

@@ -76,7 +76,6 @@ impl Server {
blob_stores: self.core.storage.blobs.clone(),
search_stores: self.core.storage.ftss.clone(),
in_memory_stores: self.core.storage.lookups.clone(),
pubsub_stores: Default::default(),
purge_schedules: Default::default(),
};
stores.parse_stores(&mut config).await;

23
crates/coordinator/Cargo.toml Normal file
View File

@@ -0,0 +1,23 @@
[package]
name = "coordinator"
version = "0.15.4"
edition = "2024"
[dependencies]
utils = { path = "../utils" }
store = { path = "../store" }
trc = { path = "../trc" }
futures = { version = "0.3", optional = true }
tokio = { version = "1.47", features = ["sync", "fs", "io-util"] }
async-nats = { version = "0.44", default-features = false, features = ["server_2_10", "server_2_11", "ring"], optional = true }
zenoh = { version = "1.3.4", default-features = false, features = ["auth_pubkey", "transport_multilink", "transport_compression", "transport_quic", "transport_tcp", "transport_tls", "transport_udp"], optional = true }
rdkafka = { version = "0.38", features = ["cmake-build"], optional = true }
redis = { version = "0.32", features = [ "tokio-comp", "tokio-rustls-comp", "tls-rustls-insecure", "tls-rustls-webpki-roots", "cluster-async"], optional = true }
[features]
nats = ["async-nats"]
zenoh = ["dep:zenoh"]
kafka = ["rdkafka"]
redis = ["dep:redis", "futures"]
enterprise = []
test_mode = []

0
crates/store/src/backend/kafka/mod.rs → crates/coordinator/src/backend/kafka/mod.rs
View File

0
crates/store/src/backend/kafka/pubsub.rs → crates/coordinator/src/backend/kafka/pubsub.rs
View File

14
crates/coordinator/src/backend/mod.rs Normal file
View File

@@ -0,0 +1,14 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
#[cfg(feature = "kafka")]
pub mod kafka;
#[cfg(feature = "nats")]
pub mod nats;
#[cfg(feature = "redis")]
pub mod redis;
#[cfg(feature = "zenoh")]
pub mod zenoh;

0
crates/store/src/backend/nats/mod.rs → crates/coordinator/src/backend/nats/mod.rs
View File

0
crates/store/src/backend/nats/pubsub.rs → crates/coordinator/src/backend/nats/pubsub.rs
View File

105
crates/coordinator/src/backend/redis/mod.rs Normal file
View File

@@ -0,0 +1,105 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::{Msg, PubSubStream};
use futures::StreamExt;
use redis::{AsyncCommands, PushInfo, cluster::ClusterConfig, cluster_async::ClusterConnection};
use std::fmt::Display;
use store::backend::redis::{RedisPool, RedisStore};
use tokio::sync::mpsc::UnboundedReceiver;
pub struct RedisPubSubStream {
stream: redis::aio::PubSubStream,
}
pub struct RedisClusterPubSubStream {
_conn: ClusterConnection,
rx: UnboundedReceiver<PushInfo>,
}
pub(crate) async fn redis_publish(
redis: &RedisStore,
topic: &'static str,
message: Vec<u8>,
) -> trc::Result<()> {
match &redis.pool {
RedisPool::Single(pool) => pool
.get()
.await
.map_err(into_error)?
.as_mut()
.publish(topic, message)
.await
.map_err(into_error),
RedisPool::Cluster(pool) => pool
.get()
.await
.map_err(into_error)?
.as_mut()
.publish(topic, message)
.await
.map_err(into_error),
}
}
pub(crate) async fn redis_subscribe(
redis: &RedisStore,
topic: &'static str,
) -> trc::Result<PubSubStream> {
match &redis.pool {
RedisPool::Single(pool) => {
let mut pubsub = pool
.manager()
.client
.get_async_pubsub()
.await
.map_err(into_error)?;
pubsub.subscribe(topic).await.map_err(into_error)?;
Ok(PubSubStream::Redis(RedisPubSubStream {
stream: pubsub.into_on_message(),
}))
}
RedisPool::Cluster(pool) => {
let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
let mut _conn = pool
.manager()
.client
.get_async_connection_with_config(ClusterConfig::default().set_push_sender(tx))
.await
.map_err(into_error)?;
_conn.subscribe(topic).await.map_err(into_error)?;
Ok(PubSubStream::RedisCluster(RedisClusterPubSubStream {
_conn,
rx,
}))
}
}
}
impl RedisPubSubStream {
pub async fn next(&mut self) -> Option<Msg> {
self.stream.next().await.map(Msg::Redis)
}
}
impl RedisClusterPubSubStream {
pub async fn next(&mut self) -> Option<Msg> {
loop {
if let Some(msg) = redis::Msg::from_push_info(self.rx.recv().await?) {
return Some(Msg::Redis(msg));
}
}
}
}
#[inline(always)]
fn into_error(err: impl Display) -> trc::Error {
trc::StoreEvent::RedisError.reason(err)
}

0
crates/store/src/backend/zenoh/mod.rs → crates/coordinator/src/backend/zenoh/mod.rs
View File

0
crates/store/src/backend/zenoh/pubsub.rs → crates/coordinator/src/backend/zenoh/pubsub.rs
View File

58
crates/store/src/dispatch/pubsub.rs → crates/coordinator/src/dispatch.rs
View File

@@ -4,68 +4,44 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::PubSubStore;
pub enum PubSubStream {
#[cfg(feature = "redis")]
Redis(crate::backend::redis::pubsub::RedisPubSubStream),
#[cfg(feature = "redis")]
RedisCluster(crate::backend::redis::pubsub::RedisClusterPubSubStream),
#[cfg(feature = "nats")]
Nats(crate::backend::nats::pubsub::NatsPubSubStream),
#[cfg(feature = "zenoh")]
Zenoh(crate::backend::zenoh::pubsub::ZenohPubSubStream),
#[cfg(feature = "kafka")]
Kafka(crate::backend::kafka::pubsub::KafkaPubSubStream),
#[cfg(not(any(feature = "redis", feature = "nats")))]
Unimplemented,
}
pub enum Msg {
#[cfg(feature = "redis")]
Redis(redis::Msg),
#[cfg(feature = "nats")]
Nats(async_nats::Message),
#[cfg(feature = "zenoh")]
Zenoh(Vec<u8>),
#[cfg(feature = "kafka")]
Kafka(Vec<u8>),
#[cfg(not(any(feature = "redis", feature = "nats")))]
Unimplemented,
}
#[cfg(feature = "redis")]
use crate::PubSubStream;
use crate::{Coordinator, Msg};
#[allow(unused_variables)]
impl PubSubStore {
impl Coordinator {
pub async fn publish(&self, topic: &'static str, message: Vec<u8>) -> trc::Result<()> {
match self {
#[cfg(feature = "redis")]
PubSubStore::Redis(store) => store.publish(topic, message).await,
Coordinator::Redis(store) => {
crate::backend::redis::redis_publish(store, topic, message).await
}
#[cfg(feature = "nats")]
PubSubStore::Nats(store) => store.publish(topic, message).await,
Coordinator::Nats(store) => store.publish(topic, message).await,
#[cfg(feature = "zenoh")]
PubSubStore::Zenoh(store) => store.publish(topic, message).await,
Coordinator::Zenoh(store) => store.publish(topic, message).await,
#[cfg(feature = "kafka")]
PubSubStore::Kafka(store) => store.publish(topic, message).await,
PubSubStore::None => Err(trc::StoreEvent::NotSupported.into_err()),
Coordinator::Kafka(store) => store.publish(topic, message).await,
Coordinator::None => Err(trc::StoreEvent::NotSupported.into_err()),
}
}
pub async fn subscribe(&self, topic: &'static str) -> trc::Result<PubSubStream> {
match self {
#[cfg(feature = "redis")]
PubSubStore::Redis(store) => store.subscribe(topic).await,
Coordinator::Redis(store) => crate::backend::redis::redis_subscribe(store, topic).await,
#[cfg(feature = "nats")]
PubSubStore::Nats(store) => store.subscribe(topic).await,
Coordinator::Nats(store) => store.subscribe(topic).await,
#[cfg(feature = "zenoh")]
PubSubStore::Zenoh(store) => store.subscribe(topic).await,
Coordinator::Zenoh(store) => store.subscribe(topic).await,
#[cfg(feature = "kafka")]
PubSubStore::Kafka(store) => store.subscribe(topic).await,
PubSubStore::None => Err(trc::StoreEvent::NotSupported.into_err()),
Coordinator::Kafka(store) => store.subscribe(topic).await,
Coordinator::None => Err(trc::StoreEvent::NotSupported.into_err()),
}
}
pub fn is_none(&self) -> bool {
matches!(self, PubSubStore::None)
matches!(self, Coordinator::None)
}
}

52
crates/coordinator/src/lib.rs Normal file
View File

@@ -0,0 +1,52 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use std::sync::Arc;
pub mod backend;
pub mod dispatch;
#[derive(Clone, Default)]
pub enum Coordinator {
#[cfg(feature = "redis")]
Redis(Arc<store::backend::redis::RedisStore>),
#[cfg(feature = "nats")]
Nats(Arc<backend::nats::NatsPubSub>),
#[cfg(feature = "zenoh")]
Zenoh(Arc<backend::zenoh::ZenohPubSub>),
#[cfg(feature = "kafka")]
Kafka(Arc<backend::kafka::KafkaPubSub>),
#[default]
None,
}
pub enum PubSubStream {
#[cfg(feature = "redis")]
Redis(crate::backend::redis::RedisPubSubStream),
#[cfg(feature = "redis")]
RedisCluster(crate::backend::redis::RedisClusterPubSubStream),
#[cfg(feature = "nats")]
Nats(crate::backend::nats::pubsub::NatsPubSubStream),
#[cfg(feature = "zenoh")]
Zenoh(crate::backend::zenoh::pubsub::ZenohPubSubStream),
#[cfg(feature = "kafka")]
Kafka(crate::backend::kafka::pubsub::KafkaPubSubStream),
#[cfg(not(any(feature = "redis", feature = "nats")))]
Unimplemented,
}
pub enum Msg {
#[cfg(feature = "redis")]
Redis(redis::Msg),
#[cfg(feature = "nats")]
Nats(async_nats::Message),
#[cfg(feature = "zenoh")]
Zenoh(Vec<u8>),
#[cfg(feature = "kafka")]
Kafka(Vec<u8>),
#[cfg(not(any(feature = "redis", feature = "nats")))]
Unimplemented,
}

9
crates/main/Cargo.toml
View File

@@ -16,6 +16,7 @@ path = "src/main.rs"
[dependencies]
store = { path = "../store" }
coordinator = { path = "../coordinator" }
jmap = { path = "../jmap" }
types = { path = "../types" }
smtp = { path = "../smtp" }
@@ -47,11 +48,11 @@ postgres = ["store/postgres"]
mysql = ["store/mysql"]
rocks = ["store/rocks"]
s3 = ["store/s3"]
redis = ["store/redis"]
nats = ["store/nats"]
redis = ["store/redis", "coordinator/redis"]
azure = ["store/azure"]
zenoh = ["store/zenoh"]
kafka = ["store/kafka"]
nats = ["coordinator/nats"]
zenoh = ["coordinator/zenoh"]
kafka = ["coordinator/kafka"]
enterprise = [ "jmap/enterprise",
"smtp/enterprise",
"common/enterprise",

16
crates/registry/Cargo.toml Normal file
View File

@@ -0,0 +1,16 @@
[package]
name = "registry"
version = "0.15.4"
edition = "2024"
[dependencies]
utils = { path = "../utils" }
trc = { path = "../trc" }
types = { path = "../types" }
serde = { version = "1.0", features = ["derive"]}
serde_json = "1.0"
hashify = "0.2.7"
[features]
test_mode = []
enterprise = []

9
crates/registry/src/lib.rs Normal file
View File

@@ -0,0 +1,9 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
pub mod pickle;
pub mod schema;
pub mod types;

204
crates/registry/src/pickle.rs Normal file
View File

@@ -0,0 +1,204 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::types::EnumType;
use std::collections::HashMap;
pub trait Pickle: Sized {
fn pickle(&self, out: &mut Vec<u8>);
fn unpickle(stream: &mut PickledStream<'_>) -> Option<Self>;
}
pub struct PickledStream<'x> {
data: &'x [u8],
pos: usize,
}
impl<'x> PickledStream<'x> {
pub fn new(data: &'x [u8]) -> Self {
PickledStream { data, pos: 0 }
}
pub fn read(&mut self) -> Option<u8> {
let byte = *self.data.get(self.pos)?;
self.pos += 1;
Some(byte)
}
pub fn read_bytes(&mut self, len: usize) -> Option<&'x [u8]> {
let bytes = self.data.get(self.pos..self.pos + len)?;
self.pos += len;
Some(bytes)
}
pub fn eof(&self) -> bool {
self.pos >= self.data.len()
}
}
impl Pickle for u16 {
fn pickle(&self, out: &mut Vec<u8>) {
out.extend_from_slice(&self.to_le_bytes());
}
fn unpickle(stream: &mut PickledStream<'_>) -> Option<Self> {
let mut arr = [0u8; 2];
arr.copy_from_slice(stream.read_bytes(2)?);
Some(u16::from_le_bytes(arr))
}
}
impl Pickle for u64 {
fn pickle(&self, out: &mut Vec<u8>) {
out.extend_from_slice(&self.to_le_bytes());
}
fn unpickle(stream: &mut PickledStream<'_>) -> Option<Self> {
let mut arr = [0u8; 8];
arr.copy_from_slice(stream.read_bytes(8)?);
Some(u64::from_le_bytes(arr))
}
}
impl Pickle for i64 {
fn pickle(&self, out: &mut Vec<u8>) {
out.extend_from_slice(&self.to_le_bytes());
}
fn unpickle(stream: &mut PickledStream<'_>) -> Option<Self> {
let mut arr = [0u8; 8];
arr.copy_from_slice(stream.read_bytes(8)?);
Some(i64::from_le_bytes(arr))
}
}
impl Pickle for f64 {
fn pickle(&self, out: &mut Vec<u8>) {
out.extend_from_slice(&self.to_le_bytes());
}
fn unpickle(stream: &mut PickledStream<'_>) -> Option<Self> {
let mut arr = [0u8; 8];
arr.copy_from_slice(stream.read_bytes(8)?);
Some(f64::from_le_bytes(arr))
}
}
impl Pickle for bool {
fn pickle(&self, out: &mut Vec<u8>) {
out.push(if *self { 1 } else { 0 });
}
fn unpickle(stream: &mut PickledStream<'_>) -> Option<Self> {
match stream.read()? {
0 => Some(false),
1 => Some(true),
_ => None,
}
}
}
impl Pickle for String {
fn pickle(&self, out: &mut Vec<u8>) {
out.extend_from_slice(&(self.len() as u32).to_le_bytes());
out.extend_from_slice(self.as_bytes());
}
fn unpickle(stream: &mut PickledStream<'_>) -> Option<Self> {
let mut len_arr = [0u8; 4];
len_arr.copy_from_slice(stream.read_bytes(4)?);
let bytes = stream.read_bytes(u32::from_le_bytes(len_arr) as usize)?;
String::from_utf8(bytes.to_vec()).ok()
}
}
impl<T: EnumType> Pickle for T {
fn pickle(&self, out: &mut Vec<u8>) {
out.extend_from_slice(&self.to_id().to_le_bytes());
}
fn unpickle(stream: &mut PickledStream<'_>) -> Option<Self> {
let mut id_arr = [0u8; 2];
id_arr.copy_from_slice(stream.read_bytes(2)?);
Self::from_id(u16::from_le_bytes(id_arr))
}
}
impl<T> Pickle for Option<T>
where
T: Pickle,
{
fn pickle(&self, out: &mut Vec<u8>) {
match self {
Some(value) => {
out.push(1);
value.pickle(out);
}
None => {
out.push(0);
}
}
}
fn unpickle(stream: &mut PickledStream<'_>) -> Option<Self> {
match stream.read()? {
0 => Some(None),
1 => T::unpickle(stream).map(Some),
_ => None,
}
}
}
impl<T> Pickle for Vec<T>
where
T: Pickle,
{
fn pickle(&self, out: &mut Vec<u8>) {
out.extend_from_slice(&(self.len() as u32).to_le_bytes());
for item in self {
item.pickle(out);
}
}
fn unpickle(stream: &mut PickledStream<'_>) -> Option<Self> {
let mut len_arr = [0u8; 4];
len_arr.copy_from_slice(stream.read_bytes(4)?);
let len = u32::from_le_bytes(len_arr) as usize;
let mut vec = Vec::with_capacity(len);
for _ in 0..len {
vec.push(T::unpickle(stream)?);
}
Some(vec)
}
}
impl<K, V, S> Pickle for HashMap<K, V, S>
where
K: Pickle + std::hash::Hash + Eq,
V: Pickle,
S: std::hash::BuildHasher + Default,
{
fn pickle(&self, out: &mut Vec<u8>) {
out.extend_from_slice(&(self.len() as u32).to_le_bytes());
for (key, value) in self {
key.pickle(out);
value.pickle(out);
}
}
fn unpickle(stream: &mut PickledStream<'_>) -> Option<Self> {
let mut len_arr = [0u8; 4];
len_arr.copy_from_slice(stream.read_bytes(4)?);
let len = u32::from_le_bytes(len_arr) as usize;
let mut map = HashMap::with_capacity_and_hasher(len, S::default());
for _ in 0..len {
let key = K::unpickle(stream)?;
let value = V::unpickle(stream)?;
map.insert(key, value);
}
Some(map)
}
}

16
crates/registry/src/schema/mod.rs Normal file
View File

@@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
#[allow(clippy::derivable_impls)]
pub mod enums;
pub mod enums_impl;
pub mod prelude;
pub mod properties;
pub mod properties_impl;
#[allow(clippy::large_enum_variant)]
pub mod structs;
#[allow(clippy::derivable_impls)]
pub mod structs_impl;

21
crates/registry/src/schema/prelude.rs Normal file
View File

@@ -0,0 +1,21 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
pub use crate::pickle::Pickle;
pub use crate::schema::enums::*;
pub use crate::schema::properties::*;
pub use crate::schema::structs::*;
pub use crate::types::EnumType;
pub use crate::types::datetime::UTCDateTime;
pub use crate::types::duration::Duration;
pub use crate::types::error::*;
pub use crate::types::id::Id;
pub use crate::types::ipaddr::IpAddr;
pub use crate::types::ipmask::IpAddrOrMask;
pub use crate::types::socketaddr::SocketAddr;
pub use serde::{Deserialize, Serialize};
pub use std::collections::HashMap;
pub use std::str::FromStr;

284
crates/registry/src/types/datetime.rs Normal file
View File

@@ -0,0 +1,284 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::pickle::{Pickle, PickledStream};
use std::{fmt::Display, str::FromStr};
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
#[repr(transparent)]
pub struct UTCDateTime(i64);
struct DateTime {
pub year: u16,
pub month: u8,
pub day: u8,
pub hour: u8,
pub minute: u8,
pub second: u8,
pub tz_before_gmt: bool,
pub tz_hour: u8,
pub tz_minute: u8,
}
impl FromStr for UTCDateTime {
type Err = ();
fn from_str(s: &str) -> Result<Self, Self::Err> {
// 2004 - 06 - 28 T 23 : 43 : 45 . 000 Z
// 1969 - 02 - 13 T 23 : 32 : 00 - 03 : 30
// 0 1 2 3 4 5 6 7
let mut pos = 0;
let mut parts = [0u32; 8];
let mut parts_sizes = [
4u32, // Year (0)
2u32, // Month (1)
2u32, // Day (2)
2u32, // Hour (3)
2u32, // Minute (4)
2u32, // Second (5)
2u32, // TZ Hour (6)
2u32, // TZ Minute (7)
];
let mut skip_digits = false;
let mut is_plus = true;
for ch in s.as_bytes() {
match ch {
b'0'..=b'9' => {
if !skip_digits {
if parts_sizes[pos] > 0 {
parts_sizes[pos] -= 1;
parts[pos] += (ch - b'0') as u32 * u32::pow(10, parts_sizes[pos]);
} else {
break;
}
}
}
b'-' => {
if pos <= 1 {
pos += 1;
} else if pos == 5 {
pos += 1;
is_plus = false;
skip_digits = false;
} else {
break;
}
}
b'T' => {
if pos == 2 {
pos += 1;
} else {
break;
}
}
b':' => {
if [3, 4, 6].contains(&pos) {
pos += 1;
} else {
break;
}
}
b'+' => {
if pos == 5 {
pos += 1;
skip_digits = false;
} else {
break;
}
}
b'.' => {
if pos == 5 {
skip_digits = true;
} else {
break;
}
}
b'Z' | b'z' => (),
_ => {
break;
}
}
}
let dt = DateTime {
year: parts[0] as u16,
month: parts[1] as u8,
day: parts[2] as u8,
hour: parts[3] as u8,
minute: parts[4] as u8,
second: parts[5] as u8,
tz_hour: parts[6] as u8,
tz_minute: parts[7] as u8,
tz_before_gmt: !is_plus,
};
if pos >= 5 && dt.is_valid() {
Ok(UTCDateTime(dt.timestamp()))
} else {
Err(())
}
}
}
impl UTCDateTime {
pub fn from_timestamp(timestamp: i64) -> Self {
UTCDateTime(timestamp)
}
pub fn timestamp(&self) -> i64 {
self.0
}
pub fn is_valid(&self) -> bool {
self.0 != i64::MAX
}
}
impl DateTime {
pub fn from_timestamp(timestamp: i64) -> Self {
// Ported from http://howardhinnant.github.io/date_algorithms.html#civil_from_days
let (z, seconds) = ((timestamp / 86400) + 719468, timestamp % 86400);
let era: i64 = (if z >= 0 { z } else { z - 146096 }) / 146097;
let doe: u64 = (z - era * 146097) as u64; // [0, 146096]
let yoe: u64 = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; // [0, 399]
let y: i64 = (yoe as i64) + era * 400;
let doy: u64 = doe - (365 * yoe + yoe / 4 - yoe / 100); // [0, 365]
let mp = (5 * doy + 2) / 153; // [0, 11]
let d: u64 = doy - (153 * mp + 2) / 5 + 1; // [1, 31]
let m: u64 = if mp < 10 { mp + 3 } else { mp - 9 }; // [1, 12]
let (h, mn, s) = (seconds / 3600, (seconds / 60) % 60, seconds % 60);
DateTime {
year: (y + i64::from(m <= 2)) as u16,
month: m as u8,
day: d as u8,
hour: h as u8,
minute: mn as u8,
second: s as u8,
tz_before_gmt: false,
tz_hour: 0,
tz_minute: 0,
}
}
#[inline(always)]
pub fn is_valid(&self) -> bool {
(0..=23).contains(&self.tz_hour)
&& (1970..=3000).contains(&self.year)
&& (0..=59).contains(&self.tz_minute)
&& (1..=12).contains(&self.month)
&& (1..=31).contains(&self.day)
&& (0..=23).contains(&self.hour)
&& (0..=59).contains(&self.minute)
&& (0..=59).contains(&self.second)
}
pub fn timestamp(&self) -> i64 {
// Ported from https://github.com/protocolbuffers/upb/blob/22182e6e/upb/json_decode.c#L982-L992
let month = self.month as u32;
let year_base = 4800; /* Before min year, multiple of 400. */
let m_adj = month.wrapping_sub(3); /* March-based month. */
let carry = i64::from(m_adj > month);
let adjust = if carry > 0 { 12 } else { 0 };
let y_adj = self.year as i64 + year_base - carry;
let month_days = ((m_adj.wrapping_add(adjust)) * 62719 + 769) / 2048;
let leap_days = y_adj / 4 - y_adj / 100 + y_adj / 400;
(y_adj * 365 + leap_days + month_days as i64 + (self.day as i64 - 1) - 2472632) * 86400
+ self.hour as i64 * 3600
+ self.minute as i64 * 60
+ self.second as i64
+ ((self.tz_hour as i64 * 3600 + self.tz_minute as i64 * 60)
* if self.tz_before_gmt { 1 } else { -1 })
}
}
impl Display for UTCDateTime {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let dt = DateTime::from_timestamp(self.0);
write!(
f,
"{:04}-{:02}-{:02}T{:02}:{:02}:{:02}Z",
dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second,
)
}
}
impl Default for UTCDateTime {
fn default() -> Self {
UTCDateTime(i64::MAX)
}
}
impl serde::Serialize for UTCDateTime {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_str(self.to_string().as_str())
}
}
impl<'de> serde::Deserialize<'de> for UTCDateTime {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
UTCDateTime::from_str(<&str>::deserialize(deserializer)?)
.map_err(|_| serde::de::Error::custom("invalid DateTime"))
}
}
impl Pickle for UTCDateTime {
fn pickle(&self, out: &mut Vec<u8>) {
out.extend_from_slice(&self.0.to_le_bytes());
}
fn unpickle(data: &mut PickledStream<'_>) -> Option<Self> {
let mut arr = [0u8; 8];
arr.copy_from_slice(data.read_bytes(8)?);
Some(UTCDateTime(i64::from_le_bytes(arr)))
}
}
impl From<u64> for UTCDateTime {
fn from(value: u64) -> Self {
UTCDateTime(value as i64)
}
}
#[cfg(test)]
mod tests {
use std::str::FromStr;
use crate::types::datetime::UTCDateTime;
#[test]
fn parse_jmap_date() {
for (input, _) in [
("1997-11-21T09:55:06-06:00", "1997-11-21T09:55:06-06:00"),
("1997-11-21T09:55:06+00:00", "1997-11-21T09:55:06Z"),
("2021-01-01T09:55:06+02:00", "2021-01-01T09:55:06+02:00"),
("2004-06-28T23:43:45.000Z", "2004-06-28T23:43:45Z"),
("1997-11-21T09:55:06.123+00:00", "1997-11-21T09:55:06Z"),
(
"2021-01-01T09:55:06.4567+02:00",
"2021-01-01T09:55:06+02:00",
),
] {
let date = UTCDateTime::from_str(input).unwrap();
//assert_eq!(date.to_string(), expected_result);
let timestamp = date.timestamp();
assert_eq!(
UTCDateTime::from_timestamp(timestamp).timestamp(),
timestamp
);
}
}
}

123
crates/registry/src/types/duration.rs Normal file
View File

@@ -0,0 +1,123 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use std::{fmt::Display, str::FromStr};
use crate::pickle::{Pickle, PickledStream};
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct Duration(pub std::time::Duration);
impl Duration {
pub fn from_millis(millis: u64) -> Self {
Duration(std::time::Duration::from_millis(millis))
}
pub fn into_inner(self) -> std::time::Duration {
self.0
}
pub fn is_valid(&self) -> bool {
self.0.as_millis() > 0
}
}
impl Default for Duration {
fn default() -> Self {
Duration(std::time::Duration::from_millis(0))
}
}
impl Display for Duration {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.0.as_millis())
}
}
impl serde::Serialize for Duration {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_str(self.to_string().as_str())
}
}
impl<'de> serde::Deserialize<'de> for Duration {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
<u64>::deserialize(deserializer)
.map(std::time::Duration::from_millis)
.map(Duration)
.map_err(|_| serde::de::Error::custom("invalid Duration"))
}
}
impl AsRef<std::time::Duration> for Duration {
fn as_ref(&self) -> &std::time::Duration {
&self.0
}
}
impl PartialOrd for Duration {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl Ord for Duration {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.0.cmp(&other.0)
}
}
impl FromStr for Duration {
type Err = String;
fn from_str(value: &str) -> Result<Self, Self::Err> {
let mut digits = String::new();
let mut multiplier = String::new();
for ch in value.chars() {
if ch.is_ascii_digit() {
digits.push(ch);
} else if !ch.is_ascii_whitespace() {
multiplier.push(ch.to_ascii_lowercase());
}
}
let multiplier = match multiplier.as_str() {
"d" => 24 * 60 * 60 * 1000,
"h" => 60 * 60 * 1000,
"m" => 60 * 1000,
"s" => 1000,
"ms" | "" => 1,
_ => return Err(format!("Invalid duration value {:?}.", value)),
};
digits
.parse::<u64>()
.ok()
.map(|num| std::time::Duration::from_millis(num * multiplier))
.map(Duration)
.ok_or_else(|| format!("Invalid duration value {:?}.", value))
}
}
impl Pickle for Duration {
fn pickle(&self, out: &mut Vec<u8>) {
out.extend_from_slice(&(self.0.as_millis() as u64).to_le_bytes());
}
fn unpickle(data: &mut PickledStream<'_>) -> Option<Self> {
let mut arr = [0u8; 8];
arr.copy_from_slice(data.read_bytes(8)?);
Some(Duration(std::time::Duration::from_millis(
u64::from_le_bytes(arr),
)))
}
}

86
crates/registry/src/types/error.rs Normal file
View File

@@ -0,0 +1,86 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::schema::prelude::Property;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ValidationErrorType {
Invalid,
Required,
MinItems(usize),
MaxItems(usize),
MaxLength(usize),
MinLength(usize),
MaxValue(i64),
MinValue(i64),
}
pub struct ValidationError {
pub property: Property,
pub typ: ValidationErrorType,
}
impl ValidationError {
pub fn new(property: Property, typ: ValidationErrorType) -> Self {
Self { property, typ }
}
pub fn required(property: Property) -> Self {
Self {
property,
typ: ValidationErrorType::Required,
}
}
pub fn invalid(property: Property) -> Self {
Self {
property,
typ: ValidationErrorType::Invalid,
}
}
pub fn min_items(property: Property, value: usize) -> Self {
Self {
property,
typ: ValidationErrorType::MinItems(value),
}
}
pub fn max_items(property: Property, value: usize) -> Self {
Self {
property,
typ: ValidationErrorType::MaxItems(value),
}
}
pub fn max_length(property: Property, value: usize) -> Self {
Self {
property,
typ: ValidationErrorType::MaxLength(value),
}
}
pub fn min_length(property: Property, value: usize) -> Self {
Self {
property,
typ: ValidationErrorType::MinLength(value),
}
}
pub fn max_value(property: Property, value: i64) -> Self {
Self {
property,
typ: ValidationErrorType::MaxValue(value),
}
}
pub fn min_value(property: Property, value: i64) -> Self {
Self {
property,
typ: ValidationErrorType::MinValue(value),
}
}
}

155
crates/registry/src/types/id.rs Normal file
View File

@@ -0,0 +1,155 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::{
pickle::{Pickle, PickledStream},
schema::prelude::Object,
};
use std::str::FromStr;
use utils::codec::base32_custom::{BASE32_ALPHABET, BASE32_INVERSE};
#[derive(Debug, PartialEq, Clone, Copy, Eq, Hash)]
#[repr(transparent)]
pub struct Id(u64);
impl Id {
pub fn new(object: Object, id: u64) -> Self {
Id(id & (u64::MAX >> 16) | ((object as u64) << 48))
}
pub fn id(&self) -> u64 {
self.0
}
pub fn is_valid(&self) -> bool {
self.0 != u64::MAX
}
// From https://github.com/archer884/crockford by J/A <archer884@gmail.com>
// License: MIT/Apache 2.0
pub fn as_string(&self) -> String {
match self.0 {
0 => "a".to_string(),
mut n => {
// Used for the initial shift.
const QUAD_SHIFT: usize = 60;
const QUAD_RESET: usize = 4;
// Used for all subsequent shifts.
const FIVE_SHIFT: usize = 59;
const FIVE_RESET: usize = 5;
// After we clear the four most significant bits, the four least significant bits will be
// replaced with 0001. We can then know to stop once the four most significant bits are,
// likewise, 0001.
const STOP_BIT: u64 = 1 << QUAD_SHIFT;
let mut buf = String::with_capacity(7);
// Start by getting the most significant four bits. We get four here because these would be
// leftovers when starting from the least significant bits. In either case, tag the four least
// significant bits with our stop bit.
match (n >> QUAD_SHIFT) as usize {
// Eat leading zero-bits. This should not be done if the first four bits were non-zero.
// Additionally, we *must* do this in increments of five bits.
0 => {
n <<= QUAD_RESET;
n |= 1;
n <<= n.leading_zeros() / 5 * 5;
}
// Write value of first four bytes.
i => {
n <<= QUAD_RESET;
n |= 1;
buf.push(char::from(BASE32_ALPHABET[i]));
}
}
// From now until we reach the stop bit, take the five most significant bits and then shift
// left by five bits.
while n != STOP_BIT {
buf.push(char::from(BASE32_ALPHABET[(n >> FIVE_SHIFT) as usize]));
n <<= FIVE_RESET;
}
buf
}
}
}
}
impl Object {
pub fn id(&self, id: u64) -> Id {
Id::new(*self, id)
}
pub fn singleton(&self) -> Id {
Id::new(*self, u64::MAX)
}
}
impl FromStr for Id {
type Err = ();
fn from_str(s: &str) -> Result<Self, Self::Err> {
let mut id = 0;
for &ch in s.as_bytes() {
let i = BASE32_INVERSE[ch as usize];
if i != u8::MAX {
id = (id << 5) | i as u64;
} else {
return Err(());
}
}
Ok(Id(id))
}
}
impl Default for Id {
fn default() -> Self {
Id(u64::MAX)
}
}
impl serde::Serialize for Id {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_str(self.as_string().as_str())
}
}
impl<'de> serde::Deserialize<'de> for Id {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
Id::from_str(<&str>::deserialize(deserializer)?)
.map_err(|_| serde::de::Error::custom("invalid Registry ID"))
}
}
impl std::fmt::Display for Id {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.as_string())
}
}
impl Pickle for Id {
fn pickle(&self, out: &mut Vec<u8>) {
out.extend_from_slice(&self.0.to_le_bytes());
}
fn unpickle(data: &mut PickledStream<'_>) -> Option<Self> {
let mut arr = [0u8; 8];
arr.copy_from_slice(data.read_bytes(8)?);
Some(Id(u64::from_le_bytes(arr)))
}
}

114
crates/registry/src/types/ipaddr.rs Normal file
View File

@@ -0,0 +1,114 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use std::{fmt::Display, net::Ipv4Addr, str::FromStr};
use crate::pickle::{Pickle, PickledStream};
#[derive(Debug, Clone, PartialEq)]
pub struct IpAddr(pub std::net::IpAddr);
impl IpAddr {
pub fn into_inner(self) -> std::net::IpAddr {
self.0
}
pub fn is_valid(&self) -> bool {
!matches!(
self.0,
std::net::IpAddr::V4(addr) if addr == Ipv4Addr::UNSPECIFIED
)
}
}
impl FromStr for IpAddr {
type Err = String;
fn from_str(s: &str) -> Result<Self, Self::Err> {
s.parse::<std::net::IpAddr>()
.map(IpAddr)
.map_err(|err| err.to_string())
}
}
impl Display for IpAddr {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.0)
}
}
impl serde::Serialize for IpAddr {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_str(self.to_string().as_str())
}
}
impl<'de> serde::Deserialize<'de> for IpAddr {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
IpAddr::from_str(<&str>::deserialize(deserializer)?)
.map_err(|_| serde::de::Error::custom("invalid IpAddr"))
}
}
impl AsRef<std::net::IpAddr> for IpAddr {
fn as_ref(&self) -> &std::net::IpAddr {
&self.0
}
}
impl Default for IpAddr {
fn default() -> Self {
IpAddr(std::net::IpAddr::V4(Ipv4Addr::UNSPECIFIED))
}
}
impl Pickle for std::net::IpAddr {
fn pickle(&self, out: &mut Vec<u8>) {
match self {
std::net::IpAddr::V4(addr) => {
out.push(4);
out.extend_from_slice(&addr.octets());
}
std::net::IpAddr::V6(addr) => {
out.push(6);
out.extend_from_slice(&addr.octets());
}
}
}
fn unpickle(data: &mut PickledStream<'_>) -> Option<Self> {
let kind = data.read()?;
match kind {
4 => {
let mut arr = [0u8; 4];
arr.copy_from_slice(data.read_bytes(4)?);
Some(std::net::IpAddr::V4(Ipv4Addr::from(arr)))
}
6 => {
let mut arr = [0u8; 16];
arr.copy_from_slice(data.read_bytes(16)?);
Some(std::net::IpAddr::V6(std::net::Ipv6Addr::from(arr)))
}
_ => None,
}
}
}
impl Pickle for IpAddr {
fn pickle(&self, out: &mut Vec<u8>) {
self.0.pickle(out);
}
fn unpickle(data: &mut PickledStream<'_>) -> Option<Self> {
std::net::IpAddr::unpickle(data).map(IpAddr)
}
}

244
crates/registry/src/types/ipmask.rs Normal file
View File

@@ -0,0 +1,244 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use std::{
fmt::{Display, Formatter},
net::{IpAddr, Ipv4Addr, Ipv6Addr},
str::FromStr,
};
use crate::pickle::{Pickle, PickledStream};
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum IpAddrOrMask {
V4 { addr: Ipv4Addr, mask: u32 },
V6 { addr: Ipv6Addr, mask: u128 },
}
impl IpAddrOrMask {
pub fn is_valid(&self) -> bool {
!matches!(
self,
IpAddrOrMask::V4 { addr, mask: _ } if addr == &Ipv4Addr::UNSPECIFIED
)
}
pub fn matches(&self, remote: &IpAddr) -> bool {
match self {
IpAddrOrMask::V4 { addr, mask } => match *mask {
u32::MAX => match remote {
IpAddr::V4(remote) => addr == remote,
IpAddr::V6(remote) => {
if let Some(remote) = remote.to_ipv4_mapped() {
addr == &remote
} else {
false
}
}
},
0 => {
matches!(remote, IpAddr::V4(_))
}
_ => {
u32::from_be_bytes(match remote {
IpAddr::V4(ip) => ip.octets(),
IpAddr::V6(ip) => {
if let Some(ip) = ip.to_ipv4() {
ip.octets()
} else {
return false;
}
}
}) & mask
== u32::from_be_bytes(addr.octets()) & mask
}
},
IpAddrOrMask::V6 { addr, mask } => match *mask {
u128::MAX => match remote {
IpAddr::V6(remote) => remote == addr,
IpAddr::V4(remote) => &remote.to_ipv6_mapped() == addr,
},
0 => {
matches!(remote, IpAddr::V6(_))
}
_ => {
u128::from_be_bytes(match remote {
IpAddr::V6(ip) => ip.octets(),
IpAddr::V4(ip) => ip.to_ipv6_mapped().octets(),
}) & mask
== u128::from_be_bytes(addr.octets()) & mask
}
},
}
}
}
impl FromStr for IpAddrOrMask {
type Err = String;
fn from_str(value: &str) -> Result<Self, Self::Err> {
if let Some((addr, mask)) = value.rsplit_once('/') {
if let (Ok(addr), Ok(mask)) =
(addr.trim().parse::<IpAddr>(), mask.trim().parse::<u32>())
{
match addr {
IpAddr::V4(addr) if (8..=32).contains(&mask) => {
return Ok(IpAddrOrMask::V4 {
addr,
mask: u32::MAX << (32 - mask),
});
}
IpAddr::V6(addr) if (8..=128).contains(&mask) => {
return Ok(IpAddrOrMask::V6 {
addr,
mask: u128::MAX << (128 - mask),
});
}
_ => (),
}
}
} else {
match value.trim().parse::<IpAddr>() {
Ok(IpAddr::V4(addr)) => {
return Ok(IpAddrOrMask::V4 {
addr,
mask: u32::MAX,
});
}
Ok(IpAddr::V6(addr)) => {
return Ok(IpAddrOrMask::V6 {
addr,
mask: u128::MAX,
});
}
_ => (),
}
}
Err(format!("Invalid IP address {:?}", value,))
}
}
impl Display for IpAddrOrMask {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
IpAddrOrMask::V4 { addr, mask } => {
if (*mask) == u32::MAX {
write!(f, "{}", addr)
} else {
let prefix = mask.count_ones();
write!(f, "{}/{}", addr, prefix)
}
}
IpAddrOrMask::V6 { addr, mask } => {
if (*mask) == u128::MAX {
write!(f, "{}", addr)
} else {
let prefix = mask.count_ones();
write!(f, "{}/{}", addr, prefix)
}
}
}
}
}
impl serde::Serialize for IpAddrOrMask {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_str(self.to_string().as_str())
}
}
impl<'de> serde::Deserialize<'de> for IpAddrOrMask {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
IpAddrOrMask::from_str(<&str>::deserialize(deserializer)?)
.map_err(|_| serde::de::Error::custom("invalid IpAddrOrMask"))
}
}
impl Default for IpAddrOrMask {
fn default() -> Self {
IpAddrOrMask::V4 {
addr: Ipv4Addr::UNSPECIFIED,
mask: u32::MAX,
}
}
}
impl Pickle for IpAddrOrMask {
fn pickle(&self, out: &mut Vec<u8>) {
match self {
IpAddrOrMask::V4 { addr, mask } => {
out.push(4);
out.extend_from_slice(&addr.octets());
out.extend_from_slice(&mask.to_le_bytes());
}
IpAddrOrMask::V6 { addr, mask } => {
out.push(6);
out.extend_from_slice(&addr.octets());
out.extend_from_slice(&mask.to_le_bytes());
}
}
}
fn unpickle(data: &mut PickledStream<'_>) -> Option<Self> {
match data.read()? {
4 => {
let mut addr_arr = [0u8; 4];
addr_arr.copy_from_slice(data.read_bytes(4)?);
let mut mask_arr = [0u8; 4];
mask_arr.copy_from_slice(data.read_bytes(4)?);
Some(IpAddrOrMask::V4 {
addr: Ipv4Addr::from(addr_arr),
mask: u32::from_le_bytes(mask_arr),
})
}
6 => {
let mut addr_arr = [0u8; 16];
addr_arr.copy_from_slice(data.read_bytes(16)?);
let mut mask_arr = [0u8; 16];
mask_arr.copy_from_slice(data.read_bytes(16)?);
Some(IpAddrOrMask::V6 {
addr: Ipv6Addr::from(addr_arr),
mask: u128::from_le_bytes(mask_arr),
})
}
_ => None,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_ipaddrmask() {
for (mask, ip) in [
("10.0.0.0/8", "10.30.20.11"),
("10.0.0.0/8", "10.0.13.73"),
("192.168.1.1", "192.168.1.1"),
] {
let mask = IpAddrOrMask::from_str(mask).unwrap();
let ip = ip.parse::<IpAddr>().unwrap();
assert!(mask.matches(&ip));
}
for (mask, ip) in [
("10.0.0.0/8", "11.30.20.11"),
("192.168.1.1", "193.168.1.1"),
] {
let mask = IpAddrOrMask::from_str(mask).unwrap();
let ip = ip.parse::<IpAddr>().unwrap();
assert!(!mask.matches(&ip));
}
}
}

20
crates/registry/src/types/mod.rs Normal file
View File

@@ -0,0 +1,20 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
pub mod datetime;
pub mod duration;
pub mod error;
pub mod id;
pub mod ipaddr;
pub mod ipmask;
pub mod socketaddr;
pub trait EnumType: Sized {
fn parse(s: &str) -> Option<Self>;
fn as_str(&self) -> &'static str;
fn from_id(id: u16) -> Option<Self>;
fn to_id(&self) -> u16;
}

84
crates/registry/src/types/socketaddr.rs Normal file
View File

@@ -0,0 +1,84 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use std::{fmt::Display, str::FromStr};
use crate::pickle::{Pickle, PickledStream};
#[derive(Debug, Clone, PartialEq)]
pub struct SocketAddr(pub std::net::SocketAddr);
impl SocketAddr {
pub fn into_inner(self) -> std::net::SocketAddr {
self.0
}
pub fn is_valid(&self) -> bool {
!self.0.ip().is_unspecified()
}
}
impl FromStr for SocketAddr {
type Err = String;
fn from_str(s: &str) -> Result<Self, Self::Err> {
s.parse::<std::net::SocketAddr>()
.map(SocketAddr)
.map_err(|err| err.to_string())
}
}
impl Display for SocketAddr {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.0)
}
}
impl serde::Serialize for SocketAddr {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_str(self.to_string().as_str())
}
}
impl<'de> serde::Deserialize<'de> for SocketAddr {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
SocketAddr::from_str(<&str>::deserialize(deserializer)?)
.map_err(|_| serde::de::Error::custom("invalid SocketAddr"))
}
}
impl Default for SocketAddr {
fn default() -> Self {
SocketAddr(std::net::SocketAddr::from(([0, 0, 0, 0], 0)))
}
}
impl AsRef<std::net::SocketAddr> for SocketAddr {
fn as_ref(&self) -> &std::net::SocketAddr {
&self.0
}
}
impl Pickle for SocketAddr {
fn pickle(&self, out: &mut Vec<u8>) {
self.0.ip().pickle(out);
out.extend_from_slice(&self.0.port().to_le_bytes());
}
fn unpickle(data: &mut PickledStream<'_>) -> Option<Self> {
let ip = std::net::IpAddr::unpickle(data)?;
let mut port_bytes = [0u8; 2];
port_bytes.copy_from_slice(data.read_bytes(2)?);
let port = u16::from_le_bytes(port_bytes);
Some(SocketAddr(std::net::SocketAddr::new(ip, port)))
}
}

9
crates/store/Cargo.toml
View File

@@ -8,12 +8,12 @@ utils = { path = "../utils" }
types = { path = "../types" }
nlp = { path = "../nlp" }
trc = { path = "../trc" }
registry = { path = "../registry" }
rocksdb = { version = "0.24", optional = true, features = ["multi-threaded-cf"] }
foundationdb = { version = "0.9.2", features = ["embedded-fdb-include", "fdb-7_3"], optional = true }
rusqlite = { version = "0.37", features = ["bundled"], optional = true }
#rust-s3 = { version = "0.37", default-features = false, features = ["tokio-rustls-tls"], optional = true }
rust-s3 = { version = "0.35", default-features = false, features = ["tokio-rustls-tls", "no-verify-ssl"], optional = true }
async-nats = { version = "0.44", default-features = false, features = ["server_2_10", "server_2_11", "ring"], optional = true }
azure_core = { version = "0.21.0", optional = true }
azure_storage = { version = "0.21.0", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"], optional = true }
azure_storage_blobs = { version = "0.21.0", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"], optional = true }
@@ -51,8 +51,6 @@ bitpacking = "0.9.2"
memchr = { version = "2.7" }
rkyv = { version = "0.8.10", features = ["little_endian"] }
compact_str = "0.9.0"
zenoh = { version = "1.3.4", default-features = false, features = ["auth_pubkey", "transport_multilink", "transport_compression", "transport_quic", "transport_tcp", "transport_tls", "transport_udp"], optional = true }
rdkafka = { version = "0.38", features = ["cmake-build"], optional = true }
rustls_021 = { package = "rustls", version = "0.21", default-features = false, features = ["dangerous_configuration"], optional = true }
[dev-dependencies]
@@ -74,10 +72,5 @@ azure = ["azure_core", "azure_storage", "azure_storage_blobs"]
# In-memory stores
redis = ["dep:redis", "deadpool", "futures"]
# Pubsub
nats = ["async-nats"]
zenoh = ["dep:zenoh"]
kafka = ["rdkafka"]
enterprise = []
test_mode = []

6
crates/store/src/backend/mod.rs
View File

@@ -11,14 +11,10 @@ pub mod elastic;
pub mod foundationdb;
pub mod fs;
pub mod http;
#[cfg(feature = "kafka")]
pub mod kafka;
pub mod meili;
pub mod memory;
#[cfg(feature = "mysql")]
pub mod mysql;
#[cfg(feature = "nats")]
pub mod nats;
#[cfg(feature = "postgres")]
pub mod postgres;
#[cfg(feature = "redis")]
@@ -29,8 +25,6 @@ pub mod rocksdb;
pub mod s3;
#[cfg(feature = "sqlite")]
pub mod sqlite;
#[cfg(feature = "zenoh")]
pub mod zenoh;
// SPDX-SnippetBegin
// SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>

16
crates/store/src/backend/redis/mod.rs
View File

@@ -4,8 +4,6 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use std::{fmt::Display, time::Duration};
use deadpool::{
Runtime,
managed::{Manager, Pool},
@@ -14,28 +12,28 @@ use redis::{
Client, ProtocolVersion,
cluster::{ClusterClient, ClusterClientBuilder},
};
use std::{fmt::Display, time::Duration};
use utils::config::{Config, utils::AsKey};
pub mod lookup;
pub mod pool;
pub mod pubsub;
#[derive(Debug)]
pub struct RedisStore {
pool: RedisPool,
pub pool: RedisPool,
}
struct RedisConnectionManager {
client: Client,
pub struct RedisConnectionManager {
pub client: Client,
timeout: Duration,
}
struct RedisClusterConnectionManager {
client: ClusterClient,
pub struct RedisClusterConnectionManager {
pub client: ClusterClient,
timeout: Duration,
}
enum RedisPool {
pub enum RedisPool {
Single(Pool<RedisConnectionManager>),
Cluster(Pool<RedisClusterConnectionManager>),
}

94
crates/store/src/backend/redis/pubsub.rs
View File

@@ -1,94 +0,0 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use super::{RedisPool, RedisStore, into_error};
use crate::dispatch::pubsub::{Msg, PubSubStream};
use futures::StreamExt;
use redis::{AsyncCommands, PushInfo, cluster::ClusterConfig, cluster_async::ClusterConnection};
use tokio::sync::mpsc::UnboundedReceiver;
pub struct RedisPubSubStream {
stream: redis::aio::PubSubStream,
}
pub struct RedisClusterPubSubStream {
_conn: ClusterConnection,
rx: UnboundedReceiver<PushInfo>,
}
impl RedisStore {
pub async fn publish(&self, topic: &'static str, message: Vec<u8>) -> trc::Result<()> {
match &self.pool {
RedisPool::Single(pool) => pool
.get()
.await
.map_err(into_error)?
.as_mut()
.publish(topic, message)
.await
.map_err(into_error),
RedisPool::Cluster(pool) => pool
.get()
.await
.map_err(into_error)?
.as_mut()
.publish(topic, message)
.await
.map_err(into_error),
}
}
pub async fn subscribe(&self, topic: &'static str) -> trc::Result<PubSubStream> {
match &self.pool {
RedisPool::Single(pool) => {
let mut pubsub = pool
.manager()
.client
.get_async_pubsub()
.await
.map_err(into_error)?;
pubsub.subscribe(topic).await.map_err(into_error)?;
Ok(PubSubStream::Redis(RedisPubSubStream {
stream: pubsub.into_on_message(),
}))
}
RedisPool::Cluster(pool) => {
let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
let mut _conn = pool
.manager()
.client
.get_async_connection_with_config(ClusterConfig::default().set_push_sender(tx))
.await
.map_err(into_error)?;
_conn.subscribe(topic).await.map_err(into_error)?;
Ok(PubSubStream::RedisCluster(RedisClusterPubSubStream {
_conn,
rx,
}))
}
}
}
}
impl RedisPubSubStream {
pub async fn next(&mut self) -> Option<Msg> {
self.stream.next().await.map(Msg::Redis)
}
}
impl RedisClusterPubSubStream {
pub async fn next(&mut self) -> Option<Msg> {
loop {
if let Some(msg) = redis::Msg::from_push_info(self.rx.recv().await?) {
return Some(Msg::Redis(msg));
}
}
}
}

8
crates/store/src/config.rs
View File

@@ -227,11 +227,11 @@ impl Stores {
{
self.in_memory_stores
.insert(store_id.clone(), InMemoryStore::Redis(db.clone()));
self.pubsub_stores
.insert(store_id, crate::PubSubStore::Redis(db));
//self.pubsub_stores
// .insert(store_id, crate::PubSubStore::Redis(db));
}
}
#[cfg(feature = "nats")]
/*#[cfg(feature = "nats")]
"nats" => {
if let Some(db) = crate::backend::nats::NatsPubSub::open(config, prefix)
.await
@@ -260,7 +260,7 @@ impl Stores {
self.pubsub_stores
.insert(store_id, crate::PubSubStore::Kafka(db));
}
}
}*/
// SPDX-SnippetBegin
// SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
// SPDX-License-Identifier: LicenseRef-SEL

5
crates/store/src/dispatch/mod.rs
View File

@@ -4,13 +4,12 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use roaring::RoaringBitmap;
use crate::Store;
use roaring::RoaringBitmap;
pub mod blob;
pub mod lookup;
pub mod pubsub;
pub mod registry;
pub mod search;
pub mod store;

23
crates/store/src/dispatch/registry.rs Normal file
View File

@@ -0,0 +1,23 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use registry::{schema::prelude::Registry, types::id::Id};
use crate::RegistryStore;
impl RegistryStore {
pub async fn get(&self, id: Id) -> trc::Result<Option<Registry>> {
todo!()
}
pub async fn get_or_default(&self, id: Id) -> trc::Result<Registry> {
todo!()
}
pub async fn delete(&self, id: Id) -> trc::Result<()> {
todo!()
}
}

20
crates/store/src/lib.rs
View File

@@ -8,6 +8,7 @@ pub mod backend;
pub mod config;
pub mod dispatch;
pub mod query;
pub mod registry;
pub mod search;
pub mod write;
@@ -21,7 +22,7 @@ pub use xxhash_rust;
use ahash::AHashMap;
use backend::{fs::FsStore, http::HttpStore, memory::StaticMemoryStore};
use std::{borrow::Cow, sync::Arc};
use std::{borrow::Cow, path::PathBuf, sync::Arc};
use utils::config::cron::SimpleCron;
use write::ValueClass;
@@ -129,7 +130,6 @@ pub struct Stores {
pub blob_stores: AHashMap<String, BlobStore>,
pub search_stores: AHashMap<String, SearchStore>,
pub in_memory_stores: AHashMap<String, InMemoryStore>,
pub pubsub_stores: AHashMap<String, PubSubStore>,
pub purge_schedules: Vec<PurgeSchedule>,
}
@@ -205,18 +205,10 @@ pub enum InMemoryStore {
// SPDX-SnippetEnd
}
#[derive(Clone, Default)]
pub enum PubSubStore {
#[cfg(feature = "redis")]
Redis(Arc<backend::redis::RedisStore>),
#[cfg(feature = "nats")]
Nats(Arc<backend::nats::NatsPubSub>),
#[cfg(feature = "zenoh")]
Zenoh(Arc<backend::zenoh::ZenohPubSub>),
#[cfg(feature = "kafka")]
Kafka(Arc<backend::kafka::KafkaPubSub>),
#[default]
None,
#[derive(Clone)]
pub enum RegistryStore {
Remote(Store),
Local(PathBuf),
}
#[cfg(feature = "sqlite")]

5
crates/store/src/registry/mod.rs Normal file
View File

@@ -0,0 +1,5 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/

14
crates/store/src/write/mod.rs
View File

@@ -278,6 +278,20 @@ pub enum DirectoryClass {
UsedQuota(u32),
}
#[derive(Debug, PartialEq, Clone, Eq, Hash)]
pub enum RegistryClass {
Item(u64),
Relation {
from: u64,
to: u64,
},
Index {
index_id: u16,
item_id: u64,
key: Vec<u8>,
},
}
#[derive(Debug, PartialEq, Clone, Eq, Hash)]
pub enum QueueClass {
Message(u64),