Adding Middleware'24 configs, build options to install.sh

sreeharshau · sreeharshau · commit b582d36a8a77 · 2025-08-10T15:09:40.000-06:00
diff --git a/README.md b/README.md
@@ -14,16 +14,16 @@ DedupBench is a benchmarking tool for data chunking techniques used in data dedu
 
 It currently supports eleven different chunking algorithms and six different fingerprinting algorithms. It supports SIMD acceleration with [VectorCDC](https://www.usenix.org/conference/fast25/presentation/udayashankar) for these algorithms using five different vector instruction sets on Intel, AMD, ARM, and IBM CPUs. 
 
-The following chunking techniques and SIMD accelerations are currently supported by DedupBench.
+The following chunking techniques and vector instruction sets are currently supported by DedupBench.
 
 | CDC Algorithm | Link | Unaccelerated | SSE-128 | AVX-256 | AVX-512 | NEON-128 (ARM) | VSX-128 (IBM) |
 | :-------: | :-------: | :-------: | :-------: | :-------: | :-------: | :-------: | :-------: |
 | AE-Max | [Paper](https://ieeexplore.ieee.org/document/7218510) | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ |
 | AE-Min | [Paper](https://ieeexplore.ieee.org/document/7218510) | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ |
-| CRC-32 | [GitHub](https://github.com/google/crc32c) | ✔️ | ❌ | ❌ | ❌ | ❌ | ❌ |
+| CRC-32 | [Paper](https://dl.acm.org/doi/10.1145/3319647.3325834) | ✔️ | ❌ | ❌ | ❌ | ❌ | ❌ |
 | FastCDC | [Paper](https://www.usenix.org/conference/atc16/technical-sessions/presentation/xia) | ✔️ | ❌ | ❌ | ❌ | ❌ | ❌ |
 | Fixed-size | [Paper](https://www.usenix.org/conference/fast-02/venti-new-approach-archival-data-storage) |  ✔️ | ❌ | ❌ | ❌ | ❌ | ❌ |
-| Gear | [Paper](https://www.sciencedirect.com/science/article/pii/S0166531614000790) | ✔️ | ❌ | ❌ | ❌ | ❌ | ❌ |
+| Gear | [Paper](https://dl.acm.org/doi/10.1145/3319647.3325834) | ✔️ | ❌ | ❌ | ❌ | ❌ | ❌ |
 | MAXP| [Paper](https://www.sciencedirect.com/science/article/pii/S0022000009000580) | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ |
 | Rabin | [Paper](https://dl.acm.org/doi/abs/10.1145/502034.502052) | ✔️ | ❌ | ❌ | ❌ | ❌ | ❌ |
 | RAM | [Paper](https://www.sciencedirect.com/science/article/abs/pii/S0167739X16305829) | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ |
@@ -39,7 +39,7 @@ The following chunking techniques and SIMD accelerations are currently supported
 # 🚀 Quick start guide
 To quickly get started, run the following commands on Ubuntu:
 
-1. Clone repository and create a basic build without SIMD acceleration.
+1. Clone repository and choose a basic build without SIMD acceleration.
 ```
   git clone git@github.com:UWASL/dedup-bench.git
   cd dedup-bench/
@@ -63,7 +63,7 @@ To see a real dataset in action and generate the graph below, download and use t
 
 # ⚡ DedupBench SIMD Builds
 
-To use any of the vector-accelerated CDC algorithms, an alternative Dedupbench build is required. We have provided preconfigured files for all algorithms with 8KB chunk sizes for convenience.
+To use any of the vector-accelerated CDC algorithms, an alternative Dedupbench build is required. Choose the right SIMD build when running `install.sh` or use the manual commands commands below. We have provided preconfigured files for all algorithms with 8KB chunk sizes for convenience.
 
 **_Note that building with the wrong options (such as AVX-256 on a machine without AVX-256 support) may result in compile / runtime errors._**
 
diff --git a/build/Makefile b/build/Makefile
@@ -47,7 +47,7 @@ avx512:
 
 .PHONY: arm_neon128
 arm_neon128:
-	$(MAKE) all
+	$(MAKE) EXTRA_COMPILER_FLAGS="-mfpu=neon" all 
 
 .PHONY: ibm_altivec128
 ibm_altivec128:
diff --git a/build/config_16kb_middleware24/aemax_16kb.conf b/build/config_16kb_middleware24/aemax_16kb.conf
@@ -0,0 +1,11 @@
+# General Parameters
+chunking_algo=ae
+hashing_algo=sha1
+output_file=./hashes_16kb_middleware24/aemax_16kb.out
+simd_mode=none
+
+buffer_size=32768
+
+# AE Parameters
+ae_extreme_mode=max
+ae_avg_block_size=16384
diff --git a/build/config_16kb_middleware24/fastcdc_16kb.conf b/build/config_16kb_middleware24/fastcdc_16kb.conf
@@ -0,0 +1,14 @@
+# General Parameters
+chunking_algo=fastcdc
+hashing_algo=sha1
+output_file=./hashes_16kb_middleware24/fastcdc_16kb.out
+simd_mode=none
+
+buffer_size=32768
+
+# FastCDC Parameters
+fastcdc_min_block_size=8192
+fastcdc_avg_block_size=16384
+fastcdc_max_block_size=32768
+fastcdc_normalization_level=2
+fastcdc_disable_normalization=false
diff --git a/build/config_16kb_middleware24/rabins_16kb.conf b/build/config_16kb_middleware24/rabins_16kb.conf
@@ -0,0 +1,13 @@
+# General Parameters
+chunking_algo=rabins
+hashing_algo=sha1
+output_file=./hashes_16kb_middleware24/rabins_16kb.out
+simd_mode=none
+
+buffer_size=32768
+
+# Rabin Chunking Parameters
+rabinc_min_block_size=8192
+rabinc_avg_block_size=16384
+rabinc_max_block_size=32768
+rabinc_window_size=48
diff --git a/build/config_16kb_middleware24/ram_16kb.conf b/build/config_16kb_middleware24/ram_16kb.conf
@@ -0,0 +1,11 @@
+# General Parameters
+chunking_algo=ram
+hashing_algo=sha1
+output_file=./hashes_16kb_middleware24/ram_16kb.out
+simd_mode=none
+
+buffer_size=32768
+
+# RAM Parameters
+ram_max_block_size=32768
+ram_avg_block_size=16384
diff --git a/build/config_16kb_middleware24/seqcdc_16kb.conf b/build/config_16kb_middleware24/seqcdc_16kb.conf
@@ -0,0 +1,18 @@
+# General Parameters
+chunking_algo=seq
+hashing_algo=sha1
+output_file=./hashes_16kb_middleware24/seqcdc_16kb.out
+simd_mode=none
+
+buffer_size=32768
+
+# SeqCDC Parameters
+seq_op_mode=inc
+
+seq_jump_trigger=50
+seq_sequence_threshold=5
+seq_jump_size=512
+
+seq_min_block_size=8192
+seq_avg_block_size=16384
+seq_max_block_size=32768
diff --git a/build/config_16kb_middleware24/tttd_16kb.conf b/build/config_16kb_middleware24/tttd_16kb.conf
@@ -0,0 +1,18 @@
+# General Parameters
+chunking_algo=tttd
+hashing_algo=sha1
+output_file=./hashes_16kb_middleware24/tttd_16kb.out
+simd_mode=none
+
+buffer_size=32768
+
+# TTTD Parameters
+tttd_min_block_size=8192
+tttd_avg_block_size=16384
+tttd_max_block_size=32768
+
+# Rabin Chunking Parameters
+rabinc_min_block_size=8192
+rabinc_avg_block_size=16384
+rabinc_max_block_size=32768
+rabinc_window_size=48
diff --git a/build/config_simd512_8kb/maxp_8kb.conf b/build/config_simd512_8kb/maxp_8kb.conf
@@ -7,5 +7,5 @@ simd_mode=none
 buffer_size=32768
 
 # MAXP Parameters
-maxp_window_size = 960
+maxp_window_size = 1024
 maxp_max_block_size = 32768
diff --git a/build/config_simd512_8kb/vmaxp512_8kb.conf b/build/config_simd512_8kb/vmaxp512_8kb.conf
@@ -7,5 +7,5 @@ simd_mode=avx512
 buffer_size=32768
 
 # MAXP Parameters
-maxp_window_size = 960
+maxp_window_size = 1024
 maxp_max_block_size = 32768
diff --git a/build/config_simd_compare_8kb/vmaxp128_8kb.conf b/build/config_simd_compare_8kb/vmaxp128_8kb.conf
@@ -7,5 +7,5 @@ simd_mode=sse128
 buffer_size=32768
 
 # MAXP Parameters
-maxp_window_size = 960
+maxp_window_size = 1024
 maxp_max_block_size = 32768
diff --git a/build/config_simd_compare_8kb/vmaxp256_8kb.conf b/build/config_simd_compare_8kb/vmaxp256_8kb.conf
@@ -7,5 +7,5 @@ simd_mode=avx256
 buffer_size=32768
 
 # MAXP Parameters
-maxp_window_size = 960
+maxp_window_size = 1024
 maxp_max_block_size = 32768
diff --git a/build/config_simd_compare_8kb/vmaxp512_8kb.conf b/build/config_simd_compare_8kb/vmaxp512_8kb.conf
@@ -7,5 +7,5 @@ simd_mode=avx512
 buffer_size=32768
 
 # MAXP Parameters
-maxp_window_size = 960
+maxp_window_size = 1024
 maxp_max_block_size = 32768
diff --git a/build/plot_results.py b/build/plot_results.py
@@ -16,6 +16,7 @@
                 "maxp":"MAXP",
                 "rabins":"Rabin",
                 "ram":"RAM",
+                "seqcdc":"SeqCDC",
                 "tttd":"TTTD",
                 "vaemax128":"VAEMax-128",
                 "vaemax256":"VAEMax-256",
@@ -80,6 +81,13 @@ def build_custom_order(keys):
     if("VRAM-512" in keys):
         custom_order.append("VRAM-512")
     
+    if("SeqCDC" in keys):
+        custom_order.append("SeqCDC")
+    
+    for key in keys:
+        if(key not in rename_dict.keys()):
+            custom_order.append(key)
+
     return custom_order
 
 
@@ -169,5 +177,5 @@ def plot_results():
     read_src_spacesavings(results_path)
     plot_results()
 
-    print("Throughput graph plotted and saved as results_graph.png.")
+    print("Throughput and space savings graphs plotted and saved as results_graph.png.")
 
diff --git a/install.sh b/install.sh
@@ -1,17 +1,77 @@
-#!/bin/sh
+#!/bin/bash
 
 # Set up dependencies
+
+echo -e "\nInstalling dependencies....\n"
+
 sudo apt update
-sudo apt install libssl-dev libxxhash-dev python3 python3-pip 
+sudo apt -y install libssl-dev libxxhash-dev python3 python3-pip 
 python3 -m pip install matplotlib seaborn
 
-# Build without SIMD
-cd build/
-make clean
-make
+# Build choices
+
+declare -i CHOICE=0
+
+while [ $CHOICE -eq 0 ]
+do
+    echo -e "\nChoose a DedupBench build:"
+    echo -e "\t1. Unaccelerated (No SIMD)"
+    echo -e "\t2. SIMD-256 (SSE-128 and AVX-256)"
+    echo -e "\t3. SIMD-512 (Above and AVX-512)"
+    echo -e "\t4. ARM NEON-128"
+    echo -e "\t5. IBM Power VSX-128"
+    read -p "Choice? " BUILD
+
+    
+    if [ $BUILD -eq 1 ]; then
+        cd build/
+        make clean   
+        make
+
+        CHOICE=1
+    
+    elif [[ $BUILD -eq 2 ]]; then
+        cd build/
+        make clean   
+        make simd_all
+
+        CHOICE=1
+    
+    elif [[ $BUILD -eq 3 ]]; then
+        cd build/
+        make clean   
+        make simd512_all
+
+        CHOICE=1
+    
+    elif [[ $BUILD -eq 4 ]]; then
+        cd build/
+        make clean   
+        make arm_neon128
+
+        CHOICE=1
+    
+    elif [[ $BUILD -eq 5 ]]; then
+        cd build/
+        make clean   
+        make ibm_altivec128
+
+        CHOICE=1
+    
+    else
+        clear
+        echo -e "\tChoose the required option correctly."
+    fi
+
+done
+
+echo -e "\nBuild completed. Creating random dataset..."
 
 # Create random dataset
+rm -rf random_dataset
 mkdir random_dataset && cd random_dataset/
 base64 /dev/urandom | head -c 1000000000 > random_1.txt
 base64 /dev/urandom | head -c 1000000000 > random_2.txt
 base64 /dev/urandom | head -c 1000000000 > random_3.txt
+
+echo -e "\nInstallation and random dataset creation completed in build/.\n"
