How I stopped worrying and learned to love zero-copy deserialization

crates/utils/Cargo.toml

@@ -37,6 +37,7 @@ quick_cache = "0.6.9"
 downcast-rs = "2.0.1"
 fast-float = "0.2.0"
 erased-serde = "0.4.5"
+rkyv = { version = "0.8.10", features = ["little_endian"] }
 
 [target.'cfg(unix)'.dependencies]
 privdrop = "0.5.3"

crates/utils/src/lib.rs

@@ -28,8 +28,20 @@ pub use erased_serde;
 
 pub const BLOB_HASH_LEN: usize = 32;
 
-#[derive(Clone, Debug, Default, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
-pub struct BlobHash([u8; BLOB_HASH_LEN]);
+#[derive(
+    rkyv::Archive,
+    rkyv::Deserialize,
+    rkyv::Serialize,
+    Clone,
+    Debug,
+    Default,
+    PartialEq,
+    Eq,
+    Hash,
+    serde::Serialize,
+    serde::Deserialize,
+)]
+pub struct BlobHash(pub [u8; BLOB_HASH_LEN]);
 
 impl BlobHash {
     pub fn new_max() -> Self {
@@ -59,6 +71,12 @@ impl From<&[u8]> for BlobHash {
     }
 }
 
+impl From<&ArchivedBlobHash> for BlobHash {
+    fn from(value: &ArchivedBlobHash) -> Self {
+        value.0.as_slice().into()
+    }
+}
+
 impl From<Vec<u8>> for BlobHash {
     fn from(value: Vec<u8>) -> Self {
         value.as_slice().into()

crates/utils/src/map/bitmap.rs

@@ -7,11 +7,25 @@
 use std::ops::Deref;
 
 #[derive(
-    Debug, serde::Serialize, serde::Deserialize, Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash,
+    Debug,
+    rkyv::Archive,
+    rkyv::Deserialize,
+    rkyv::Serialize,
+    serde::Serialize,
+    serde::Deserialize,
+    Clone,
+    Copy,
+    PartialOrd,
+    Ord,
+    PartialEq,
+    Eq,
+    Hash,
 )]
+#[rkyv(compare(PartialEq), derive(Debug))]
 pub struct Bitmap<T: BitmapItem> {
     pub bitmap: u64,
     #[serde(skip)]
+    #[rkyv(omit_bounds)]
     _state: std::marker::PhantomData<T>,
 }
 
@@ -38,6 +52,11 @@ impl<T: BitmapItem> Bitmap<T> {
         self.bitmap |= items.bitmap;
     }
 
+    #[inline(always)]
+    pub fn union_raw(&mut self, items: impl Into<u64>) {
+        self.bitmap |= items.into();
+    }
+
     #[inline(always)]
     pub fn intersection(&mut self, items: &Bitmap<T>) {
         self.bitmap &= items.bitmap;
@@ -103,6 +122,24 @@ impl<T: BitmapItem> Bitmap<T> {
     }
 }
 
+impl<T: BitmapItem> From<ArchivedBitmap<T>> for Bitmap<T> {
+    fn from(value: ArchivedBitmap<T>) -> Self {
+        Self {
+            bitmap: value.bitmap.into(),
+            _state: std::marker::PhantomData,
+        }
+    }
+}
+
+impl<T: BitmapItem> From<&ArchivedBitmap<T>> for Bitmap<T> {
+    fn from(value: &ArchivedBitmap<T>) -> Self {
+        Self {
+            bitmap: value.bitmap.into(),
+            _state: std::marker::PhantomData,
+        }
+    }
+}
+
 impl<T: BitmapItem> From<u64> for Bitmap<T> {
     fn from(value: u64) -> Self {
         Self {

crates/utils/src/map/vec_map.rs

@@ -6,18 +6,19 @@
 
 use std::{borrow::Borrow, cmp::Ordering, fmt, hash::Hash};
 
+use rkyv::Archive;
 use serde::{Deserialize, Serialize, de::DeserializeOwned, ser::SerializeMap};
 
 // A map implemented using vectors
 // used for small datasets of less than 20 items
 // and when deserializing from JSON
 
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)]
 pub struct VecMap<K: Eq + PartialEq, V> {
     inner: Vec<KeyValue<K, V>>,
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq, Hash)]
 pub struct KeyValue<K: Eq + PartialEq, V> {
     key: K,
     value: V,
@@ -231,6 +232,28 @@ impl<K: Eq + PartialEq, V: Default> VecMap<K, V> {
     }
 }
 
+impl<K: Archive + Eq + PartialEq, V: Archive> ArchivedVecMap<K, V> {
+    pub fn len(&self) -> usize {
+        self.inner.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.inner.is_empty()
+    }
+
+    #[inline(always)]
+    pub fn iter(
+        &self,
+    ) -> impl Iterator<
+        Item = (
+            &<K as rkyv::Archive>::Archived,
+            &<V as rkyv::Archive>::Archived,
+        ),
+    > {
+        self.inner.iter().map(|kv| (&kv.key, &kv.value))
+    }
+}
+
 impl<K: Eq + PartialEq, V> IntoIterator for VecMap<K, V> {
     type Item = (K, V);