wip

Author: michaelwoerister

.github/workflows/ci.yml

@@ -65,6 +65,10 @@ jobs:
       run: cargo test --verbose --target ${{ matrix.arch }}-${{ fromJSON(env.target_map)[matrix.os] }} --features=no_simd
     - name: Build docs
       run: cargo doc --verbose
+    - name: Run SIMD/no-SIMD tests
+      run: |
+        cd crosstest
+        cargo run
 
   benchmarks:
     strategy:

.gitignore

@@ -1,2 +1,3 @@
 /target
+/crosstest/target
 Cargo.lock

crosstest/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "crosstest"
+version = "0.1.0"
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+odht_with_simd = { package = "odht", version = "0.2.1" }
+odht_no_simd = { package = "odht", path = "..", features = ["no_simd"] }

crosstest/src/main.rs

@@ -0,0 +1,55 @@
+
+
+struct FxConfig;
+
+macro_rules! impl_config {
+    ($odht_version:ident) => {
+        impl $odht_version::Config for FxConfig {
+            type Key = u64;
+            type Value = u32;
+
+            type EncodedKey = [u8; 8];
+            type EncodedValue = [u8; 4];
+
+            type H = $odht_version::FxHashFn;
+
+            #[inline]
+            fn encode_key(k: &Self::Key) -> Self::EncodedKey {
+                k.to_le_bytes()
+            }
+
+            #[inline]
+            fn encode_value(v: &Self::Value) -> Self::EncodedValue {
+                v.to_le_bytes()
+            }
+
+            #[inline]
+            fn decode_key(k: &Self::EncodedKey) -> Self::Key {
+                u64::from_le_bytes(*k)
+            }
+
+            #[inline]
+            fn decode_value(v: &Self::EncodedValue) -> Self::Value {
+                u32::from_le_bytes(*v)
+            }
+        }
+    }
+}
+
+impl_config!(odht_no_simd);
+impl_config!(odht_with_simd);
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+
+    let entries: Vec<(u64, u32)> = (0 .. 1_000_000_u64).map(|x| (x * x, x as u32)).collect();
+
+    let with_simd_table = odht_with_simd::HashTableOwned::<FxConfig>::from_iterator(entries.clone(), 85);
+
+    let no_simd_table = odht_no_simd::HashTable::<FxConfig, _>::from_raw_bytes(with_simd_table.raw_bytes())?;
+
+    for (key, value) in entries {
+        assert_eq!(no_simd_table.get(&key), Some(value));
+    }
+
+    Ok(())
+}

src/lib.rs

@@ -95,12 +95,12 @@ mod unhash;
 
 use error::Error;
 use std::borrow::{Borrow, BorrowMut};
+use swisstable_group_query::REFERENCE_GROUP_SIZE;
 
 pub use crate::fxhash::FxHashFn;
 pub use crate::unhash::UnHashFn;
 
 use crate::raw_table::{ByteArray, RawIter, RawTable, RawTableMut};
-use crate::swisstable_group_query::GROUP_SIZE;
 
 /// This trait provides a complete "configuration" for a hash table, i.e. it
 /// defines the key and value types, how these are encoded and what hash
@@ -527,7 +527,7 @@ fn slots_needed(item_count: usize, max_load_factor: Factor) -> usize {
     let slots_needed = max_load_factor.apply_inverse(item_count);
     std::cmp::max(
         slots_needed.checked_next_power_of_two().unwrap(),
-        GROUP_SIZE,
+        REFERENCE_GROUP_SIZE,
     )
 }
 
@@ -708,7 +708,10 @@ mod tests {
 
     #[test]
     fn load_factor_and_item_count() {
-        assert_eq!(slots_needed(0, Factor::from_percent(100)), GROUP_SIZE);
+        assert_eq!(
+            slots_needed(0, Factor::from_percent(100)),
+            REFERENCE_GROUP_SIZE
+        );
         assert_eq!(slots_needed(6, Factor::from_percent(60)), 16);
         assert_eq!(slots_needed(5, Factor::from_percent(50)), 16);
         assert_eq!(slots_needed(5, Factor::from_percent(49)), 16);

src/memory_layout.rs

@@ -5,13 +5,12 @@ use std::{
     mem::{align_of, size_of},
 };
 
-use crate::Config;
 use crate::{
     error::Error,
     raw_table::{Entry, EntryMetadata, RawTable},
-    swisstable_group_query::GROUP_SIZE,
     Factor,
 };
+use crate::{swisstable_group_query::REFERENCE_GROUP_SIZE, Config};
 
 const CURRENT_FILE_FORMAT_VERSION: [u8; 4] = [0, 0, 0, 1];
 
@@ -223,7 +222,7 @@ where
         let entry_metadata = unsafe {
             std::slice::from_raw_parts(
                 raw_bytes.as_ptr().offset(metadata_offset) as *const EntryMetadata,
-                slot_count + GROUP_SIZE,
+                slot_count + REFERENCE_GROUP_SIZE,
             )
         };
 
@@ -291,7 +290,7 @@ where
         let entry_metadata = unsafe {
             std::slice::from_raw_parts_mut(
                 raw_bytes.as_mut_ptr().offset(metadata_offset) as *mut EntryMetadata,
-                slot_count + GROUP_SIZE,
+                slot_count + REFERENCE_GROUP_SIZE,
             )
         };
 
@@ -327,7 +326,9 @@ pub(crate) fn bytes_needed<C: Config>(slot_count: usize) -> usize {
     let size_of_entry = size_of::<Entry<C::EncodedKey, C::EncodedValue>>();
     let size_of_metadata = size_of::<EntryMetadata>();
 
-    HEADER_SIZE + slot_count * size_of_entry + (slot_count + GROUP_SIZE) * size_of_metadata
+    HEADER_SIZE
+        + slot_count * size_of_entry
+        + (slot_count + REFERENCE_GROUP_SIZE) * size_of_metadata
 }
 
 pub(crate) fn allocate<C: Config>(

src/raw_table.rs

@@ -186,7 +186,7 @@ where
         assert!(std::mem::align_of::<Entry<K, V>>() == 1);
 
         debug_assert!(data.len().is_power_of_two());
-        debug_assert!(metadata.len() == data.len() + GROUP_SIZE);
+        debug_assert!(metadata.len() == data.len() + REFERENCE_GROUP_SIZE);
 
         Self {
             metadata,
@@ -198,7 +198,7 @@ where
     #[inline]
     pub(crate) fn find(&self, key: &K) -> Option<&V> {
         debug_assert!(self.data.len().is_power_of_two());
-        debug_assert!(self.metadata.len() == self.data.len() + GROUP_SIZE);
+        debug_assert!(self.metadata.len() == self.data.len() + REFERENCE_GROUP_SIZE);
 
         let mask = self.data.len() - 1;
         let hash = H::hash(key.as_slice());
@@ -308,7 +308,7 @@ where
         assert!(std::mem::align_of::<Entry<K, V>>() == 1);
 
         debug_assert!(data.len().is_power_of_two());
-        debug_assert_eq!(metadata.len(), data.len() + GROUP_SIZE);
+        debug_assert_eq!(metadata.len(), data.len() + REFERENCE_GROUP_SIZE);
 
         Self {
             metadata,
@@ -324,7 +324,7 @@ where
     #[inline]
     pub(crate) fn insert(&mut self, key: K, value: V) -> Option<V> {
         debug_assert!(self.data.len().is_power_of_two());
-        debug_assert!(self.metadata.len() == self.data.len() + GROUP_SIZE);
+        debug_assert!(self.metadata.len() == self.data.len() + REFERENCE_GROUP_SIZE);
 
         let mask = self.data.len() - 1;
         let hash = H::hash(key.as_slice());
@@ -352,11 +352,11 @@ where
                 *entry_at_mut(self.data, index) = Entry::new(key, value);
                 *metadata_at_mut(self.metadata, index) = h2;
 
-                if index < GROUP_SIZE {
+                if index < REFERENCE_GROUP_SIZE {
                     let first_mirror = self.data.len();
                     *metadata_at_mut(self.metadata, first_mirror + index) = h2;
                     debug_assert_eq!(
-                        self.metadata[..GROUP_SIZE],
+                        self.metadata[..REFERENCE_GROUP_SIZE],
                         self.metadata[self.data.len()..]
                     );
                 }
@@ -408,7 +408,7 @@ where
 {
     pub(crate) fn new(metadata: &'a [EntryMetadata], data: &'a [Entry<K, V>]) -> RawIter<'a, K, V> {
         debug_assert!(data.len().is_power_of_two());
-        debug_assert!(metadata.len() == data.len() + GROUP_SIZE);
+        debug_assert!(metadata.len() == data.len() + REFERENCE_GROUP_SIZE);
 
         RawIter {
             metadata,
@@ -529,7 +529,7 @@ mod tests {
     ) -> (Vec<EntryMetadata>, Vec<Entry<K, V>>) {
         let size = xs.size_hint().0.next_power_of_two();
         let mut data = vec![Entry::default(); size];
-        let mut metadata = vec![255; size + GROUP_SIZE];
+        let mut metadata = vec![255; size + REFERENCE_GROUP_SIZE];
 
         assert!(metadata.iter().all(|b| is_empty_or_deleted(*b)));