Storage layer refactoring: faster id generation, automatic batching and virtual thread ids

crates/utils/src/map/bitmap.rs

@@ -106,6 +106,20 @@ impl<T: BitmapItem> Bitmap<T> {
         false
     }
 
+    #[inline(always)]
+    pub fn contains_all(&self, items: impl Iterator<Item = T>) -> bool {
+        if !self.is_empty() {
+            for item in items {
+                if self.bitmap & (1 << item.into()) == 0 {
+                    return false;
+                }
+            }
+            true
+        } else {
+            false
+        }
+    }
+
     #[inline(always)]
     pub fn is_empty(&self) -> bool {
         self.bitmap == 0
@@ -245,3 +259,29 @@ impl<T: BitmapItem> Default for Bitmap<T> {
         }
     }
 }
+
+#[repr(transparent)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub struct ShortId(pub u8);
+
+impl BitmapItem for ShortId {
+    fn max() -> u64 {
+        u8::MAX as u64
+    }
+
+    fn is_valid(&self) -> bool {
+        true
+    }
+}
+
+impl From<u64> for ShortId {
+    fn from(value: u64) -> Self {
+        ShortId(value as u8)
+    }
+}
+
+impl From<ShortId> for u64 {
+    fn from(value: ShortId) -> Self {
+        value.0 as u64
+    }
+}

crates/utils/src/snowflake.rs

@@ -22,11 +22,14 @@ const NODE_ID_LEN: u64 = 9;
 const SEQUENCE_MASK: u64 = (1 << SEQUENCE_LEN) - 1;
 const NODE_ID_MASK: u64 = (1 << NODE_ID_LEN) - 1;
 
+const DEFAULT_EPOCH: u64 = 1632280000; // 52 years after UNIX_EPOCH
+const DEFAULT_EPOCH_MS: u128 = (DEFAULT_EPOCH as u128) * 1000; // 52 years after UNIX_EPOCH in milliseconds
+
 /*
 
 ID characteristics:
 
-- 43 bits for milliseconds since January 1st, 2022: 2^43 / (1000 * 60 * 60 * 24 * 365) = 278.92 years
+- 43 bits for milliseconds since January 1st, 2022: 2^43 / (1000 * 60 * 60 * 24 * 365) = 278.92 years (from year 2022 until 2300)
 - 9 bits for a node id: 2^9 = 512 nodes
 - 12 bits for a sequence number: 2^12 = 4096 ids per millisecond
 
@@ -38,7 +41,7 @@ impl SnowflakeIdGenerator {
     }
 
     pub fn from_duration(period: Duration) -> Option<u64> {
-        (SystemTime::UNIX_EPOCH + Duration::from_secs(1632280000))
+        (SystemTime::UNIX_EPOCH + Duration::from_secs(DEFAULT_EPOCH))
             .elapsed()
             .ok()
             .and_then(|elapsed| elapsed.checked_sub(period))
@@ -55,7 +58,7 @@ impl SnowflakeIdGenerator {
 
     pub fn with_node_id(node_id: u64) -> Self {
         Self {
-            epoch: SystemTime::UNIX_EPOCH + Duration::from_secs(1632280000), // 52 years after UNIX_EPOCH
+            epoch: SystemTime::UNIX_EPOCH + Duration::from_secs(DEFAULT_EPOCH), // 52 years after UNIX_EPOCH
             node_id,
             sequence: 0.into(),
         }
@@ -70,15 +73,35 @@ impl SnowflakeIdGenerator {
             .map(|elapsed| (elapsed.as_millis() as u64) << (SEQUENCE_LEN + NODE_ID_LEN))
     }
 
+    pub fn is_valid(&self) -> bool {
+        self.epoch.elapsed().is_ok()
+    }
+
     #[inline(always)]
-    pub fn generate(&self) -> Option<u64> {
-        let elapsed = self.epoch.elapsed().ok()?.as_millis() as u64;
+    pub fn generate(&self) -> u64 {
+        let elapsed = self
+            .epoch
+            .elapsed()
+            .map(|e| e.as_millis())
+            .unwrap_or_default() as u64;
         let sequence = self.sequence.fetch_add(1, Ordering::Relaxed);
 
-        ((elapsed << (SEQUENCE_LEN + NODE_ID_LEN))
+        (elapsed << (SEQUENCE_LEN + NODE_ID_LEN))
             | ((self.node_id & NODE_ID_MASK) << SEQUENCE_LEN)
-            | (sequence & SEQUENCE_MASK))
-            .into()
+            | (sequence & SEQUENCE_MASK)
+    }
+
+    #[inline(always)]
+    pub fn from_params(sequence: u64, node_id: u16) -> u64 {
+        let elapsed = SystemTime::UNIX_EPOCH
+            .elapsed()
+            .map(|e| e.as_millis())
+            .unwrap_or_default()
+            .saturating_sub(DEFAULT_EPOCH_MS) as u64;
+
+        (elapsed << (SEQUENCE_LEN + NODE_ID_LEN))
+            | (((node_id as u64) & NODE_ID_MASK) << SEQUENCE_LEN)
+            | (sequence & SEQUENCE_MASK)
     }
 }