Merge remote-tracking branch 'origin/unstable' into karsubba/1.0

Author: KarthikSubbarao

.github/workflows/ci.yml

@@ -7,14 +7,16 @@ on:
 env:
   CARGO_TERM_COLOR: always
   VALKEY_REPO_URL: https://github.com/valkey-io/valkey.git
+  TEST_FRAMEWORK_REPO: https://github.com/valkey-io/valkey-test-framework
+  TEST_FRAMEWORK_DIR: tests/build/valkeytestframework
 
 jobs:
   build-ubuntu-latest:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        server_version: ['unstable', '8.0.0']
+        server_version: ['unstable', '8.0', '8.1']
     steps:
     - uses: actions/checkout@v4
     - name: Set the server verison for python integeration tests
@@ -25,7 +27,7 @@ jobs:
         cargo clippy --profile release --all-targets -- -D clippy::all
     - name: Release Build
       run: |
-        if [ "${{ matrix.server_version }}" = "8.0.0" ]; then
+        if [ "${{ matrix.server_version }}" = "8.0" ]; then
           RUSTFLAGS="-D warnings" cargo build --all --all-targets --release --features valkey_8_0
         else
           RUSTFLAGS="-D warnings" cargo build --all --all-targets --release
@@ -34,13 +36,20 @@ jobs:
       run: cargo test --features enable-system-alloc
     - name: Make valkey-server binary
       run: |
-        mkdir -p "tests/.build/binaries/${{ matrix.server_version }}"
-        cd tests/.build
+        mkdir -p "tests/build/binaries/${{ matrix.server_version }}"
+        cd tests/build
         git clone "${{ env.VALKEY_REPO_URL }}"
         cd valkey
         git checkout ${{ matrix.server_version }}
         make -j
         cp src/valkey-server ../binaries/${{ matrix.server_version }}/
+    - name: Set up test framework
+      run: |
+        echo "Cloning test framework..."
+        git clone ${{ env.TEST_FRAMEWORK_REPO }}
+        mkdir -p ${{ env.TEST_FRAMEWORK_DIR }}
+        mv valkey-test-framework/src/* ${{ env.TEST_FRAMEWORK_DIR }}/
+        rm -rf valkey-test-framework
     - name: Set up Python
       uses: actions/setup-python@v3
       with:
@@ -72,7 +81,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        server_version: ['unstable', '8.0.0']
+        server_version: ['unstable', '8.0', '8.1']
     steps:
     - uses: actions/checkout@v4
     - name: Set the server verison for python integeration tests
@@ -83,7 +92,7 @@ jobs:
         cargo clippy --profile release --all-targets -- -D clippy::all
     - name: Release Build
       run: |
-        if [ "${{ matrix.server_version }}" = "8.0.0" ]; then
+        if [ "${{ matrix.server_version }}" = "8.0" ]; then
           RUSTFLAGS="-D warnings" cargo build --all --all-targets --release --features valkey_8_0
         else
           RUSTFLAGS="-D warnings" cargo build --all --all-targets --release
@@ -92,14 +101,21 @@ jobs:
       run: cargo test --features enable-system-alloc
     - name: Make Valkey-server binary with asan
       run: |
-        mkdir -p "tests/.build/binaries/${{ matrix.server_version }}"
-        cd tests/.build
+        mkdir -p "tests/build/binaries/${{ matrix.server_version }}"
+        cd tests/build
         git clone "${{ env.VALKEY_REPO_URL }}"
         cd valkey
         git checkout ${{ matrix.server_version }}
         make distclean
         make -j SANITIZER=address SERVER_CFLAGS='-Werror' BUILD_TLS=module
         cp src/valkey-server ../binaries/${{ matrix.server_version }}/
+    - name: Set up test framework
+      run: |
+        echo "Cloning test framework..."
+        git clone ${{ env.TEST_FRAMEWORK_REPO }}
+        mkdir -p ${{ env.TEST_FRAMEWORK_DIR }}
+        mv valkey-test-framework/src/* ${{ env.TEST_FRAMEWORK_DIR }}/
+        rm -rf valkey-test-framework
     - name: Set up Python
       uses: actions/setup-python@v3
       with:

.gitignore

@@ -1,6 +1,8 @@
 Cargo.lock
 target
-tests/.build
+tests/build
 __pycache__
+.pytest_cache
 test-data
 .attach_pid*
+src/commands/commands

QUICK_START.md

@@ -0,0 +1,58 @@
+# Quick Start
+
+Follow these steps to set up, build, and run the Valkey server with the valkey-bloom module. This guide will walk you through creating a bloom filter, inserting items, and checking for items in the filters.
+
+## Step 1: Install Valkey and valkey-bloom
+
+1. Build Valkey from source by following the instructions [here](https://github.com/valkey-io/valkey?tab=readme-ov-file#building-valkey-using-makefile). Make sure to use Valkey version 8.0 or above.
+
+2. Build the valkey-bloom module from source by following the instructions [here](https://github.com/valkey-io/valkey-bloom/blob/unstable/README.md#build-instructions).
+
+## Step 2: Run the Valkey Server with valkey-bloom
+
+Once valkey-bloom is built, run the Valkey server with the module loaded:
+
+In case of Linux:
+```bash
+./valkey-server --loadmodule ./target/release/libvalkey_bloom.so
+```
+
+You should see the Valkey server start, and it will be ready to accept commands.
+
+## Step 3: Create a Bloom Filter
+
+Start a Valkey CLI session:
+
+```bash
+valkey-cli
+```
+
+Create a bloom filter using the BF.ADD, BF.INSERT, BF.RESERVE or BF.MADD commands. For example:
+
+```bash
+BF.ADD filter-key item-val
+```
+
+- `filter-key` is the name of the bloom filter we will be operating on
+- `item-val` is the item we are inserting into the bloom filter
+
+## Step 4: Insert some more items
+
+To insert items on an already created filter, use the `BF.ADD`, `BF.MADD` or `BF.INSERT` commands:
+
+```bash
+BF.ADD filter-key example 
+BF.MADD filter-key example1 example2 
+```
+
+Replace the example with the actual items you want to add.
+
+## Step 5: Check if items are present
+
+Now that you've created a bloom filter and inserted items, you can check what items have been added. Use the `BF.EXISTS` or `BF.MEXISTS` commands to check for items:
+
+```bash
+BF.EXISTS filter-key example
+```
+
+This command checks if an item is present in a bloom filter. Bloom filters can have false positives, but no false negatives. This means that if the BF.EXISTS command returns 0, then the item is not present. But if the BF.EXISTS command returns 1, there is a possibility (determined by false positive rate) that the item is not actually present.

README.md

@@ -1,10 +1,10 @@
 # valkey-bloom
 
-Valkey-Bloom (BSD-3-Clause) is a Rust Valkey-Module which brings a native and space efficient probabilistic Module data type to Valkey. With this, users can create filters (space-efficient probabilistic Module data type) to add elements, perform “check” operation to test whether an element exists, auto scale their filters, perform RDB Save and load operations, etc.
+Valkey-Bloom (BSD-3-Clause) is a Rust based Valkey-Module which brings a Bloom Filter (Module) data type into Valkey and supports verions >= 8.0. With this, users can create bloom filters (space efficient probabilistic data structures) to add elements, check whether elements exists, auto scale their filters, customize bloom filter properties, perform RDB Save and load operations, etc.
 
 Valkey-Bloom is built using `bloomfilter::Bloom` (https://crates.io/crates/bloomfilter which has a BSD-2-Clause license).
 
-It is compatible with the BloomFilter (BF.*) command APIs in Redis offerings.
+It is API compatible with the bloom filter command syntax of the official Valkey client libraries including valkey-py, valkey-java, valkey-go (as well as the equivalent Redis libraries)
 
 ## Supported commands
 ```
@@ -25,7 +25,10 @@ curl https://sh.rustup.rs -sSf | sh
 sudo yum install clang
 git clone https://github.com/valkey-io/valkey-bloom.git
 cd valkey-bloom
-cargo build --all --all-targets  --release
+# Building for Valkey 8.1 and above:
+cargo build --release
+# Building for Valkey 8.0 specifically:
+cargo build --release --features valkey_8_0
 valkey-server --loadmodule ./target/release/libvalkey_bloom.so
 ```
 

build.sh

@@ -22,29 +22,29 @@ if [ -z "$SERVER_VERSION" ]; then
     export SERVER_VERSION="unstable"
 fi
 
-if [ "$SERVER_VERSION" != "unstable" ] && [ "$SERVER_VERSION" != "8.0.0" ] ; then
+if [ "$SERVER_VERSION" != "unstable" ] && [ "$SERVER_VERSION" != "8.0" ] && [ "$SERVER_VERSION" != "8.1" ]; then
   echo "ERROR: Unsupported version - $SERVER_VERSION"
   exit 1
 fi
 
 echo "Running cargo build release..."
-if [ "$SERVER_VERSION" == "8.0.0" ] ; then
+if [ "$SERVER_VERSION" == "8.0" ] ; then
     RUSTFLAGS="-D warnings" cargo build --all --all-targets  --release --features valkey_8_0
 else
     RUSTFLAGS="-D warnings" cargo build --all --all-targets  --release
 fi
 
 
 REPO_URL="https://github.com/valkey-io/valkey.git"
-BINARY_PATH="tests/.build/binaries/$SERVER_VERSION/valkey-server"
-
+BINARY_PATH="tests/build/binaries/$SERVER_VERSION/valkey-server"
+CACHED_VALKEY_PATH="tests/build/valkey"
 if [ -f "$BINARY_PATH" ] && [ -x "$BINARY_PATH" ]; then
     echo "valkey-server binary '$BINARY_PATH' found."
 else
     echo "valkey-server binary '$BINARY_PATH' not found."
-    mkdir -p "tests/.build/binaries/$SERVER_VERSION"
-    cd tests/.build
-    rm -rf valkey
+    mkdir -p "tests/build/binaries/$SERVER_VERSION"
+    rm -rf $CACHED_VALKEY_PATH
+    cd tests/build
     git clone "$REPO_URL"
     cd valkey
     git checkout "$SERVER_VERSION"
@@ -55,6 +55,22 @@ else
         make -j
     fi
     cp src/valkey-server ../binaries/$SERVER_VERSION/
+    cd $SCRIPT_DIR
+    rm -rf $CACHED_VALKEY_PATH
+fi
+
+
+TEST_FRAMEWORK_REPO="https://github.com/valkey-io/valkey-test-framework"
+TEST_FRAMEWORK_DIR="tests/build/valkeytestframework"
+
+if [ -d "$TEST_FRAMEWORK_DIR" ]; then
+    echo "valkeytestframework found."
+else
+    echo "Cloning valkey-test-framework..."
+    git clone "$TEST_FRAMEWORK_REPO"
+    mkdir -p "$TEST_FRAMEWORK_DIR"
+    mv "valkey-test-framework/src"/* "$TEST_FRAMEWORK_DIR/"
+    rm -rf valkey-test-framework
 fi
 
 REQUIREMENTS_FILE="requirements.txt"
@@ -78,8 +94,6 @@ if [[ "$os_type" == "Darwin" ]]; then
   MODULE_EXT=".dylib"
 elif [[ "$os_type" == "Linux" ]]; then
   MODULE_EXT=".so"
-elif [[ "$os_type" == "Windows" ]]; then
-  MODULE_EXT=".dll"
 else
   echo "Unsupported OS type: $os_type"
   exit 1
@@ -115,8 +129,6 @@ if [ ! -z "${ASAN_BUILD}" ]; then
         rm test_output.tmp
         exit 1
     fi
-
-
     rm test_output.tmp
 else 
     # TEST_PATTERN can be used to run specific tests or test patterns.

src/bloom/command_handler.rs

@@ -24,9 +24,11 @@ fn handle_bloom_add(
 ) -> Result<ValkeyValue, ValkeyError> {
     match multi {
         true => {
-            let mut result = Vec::new();
-            for item in args.iter().take(argc).skip(item_idx) {
-                match bf.add_item(item.as_slice(), validate_size_limit) {
+            let mut result = Vec::with_capacity(argc - item_idx);
+            let mut curr_cmd_idx = item_idx;
+            while curr_cmd_idx < argc {
+                let item = args[curr_cmd_idx].as_slice();
+                match bf.add_item(item, validate_size_limit) {
                     Ok(add_result) => {
                         if add_result == 1 {
                             *add_succeeded = true;
@@ -38,6 +40,7 @@ fn handle_bloom_add(
                         break;
                     }
                 };
+                curr_cmd_idx += 1;
             }
             Ok(ValkeyValue::Array(result))
         }
@@ -299,7 +302,7 @@ pub fn bloom_filter_exists(
         let item = input_args[curr_cmd_idx].as_slice();
         return Ok(handle_item_exists(value, item));
     }
-    let mut result = Vec::new();
+    let mut result = Vec::with_capacity(argc - curr_cmd_idx);
     while curr_cmd_idx < argc {
         let item = input_args[curr_cmd_idx].as_slice();
         result.push(handle_item_exists(value, item));

src/commands/bf.add.json

@@ -0,0 +1,25 @@
+{
+    "BF.ADD": {
+        "summary": "Add a single item to a bloom filter. The bloom filter is created if it doesn't exist",
+        "complexity": "O(N), where N is the number of hash functions used by the bloom filter.",
+        "group": "bloom",
+        "module_since": "1.0.0",
+        "arity": 3,
+        "acl_categories": [
+            "FAST",
+            "WRITE",
+            "BLOOM"
+        ],
+        "arguments": [
+            {
+                "name": "key",
+                "type": "key",
+                "key_spec_index": 0
+            },
+            {
+                "name": "value",
+                "type": "string"
+            }
+        ]
+    }
+}

src/commands/bf.card.json

@@ -0,0 +1,21 @@
+{
+    "BF.CARD": {
+        "summary": "Returns the cardinality of a bloom filter",
+        "complexity": "O(1)",
+        "group": "bloom",
+        "module_since": "1.0.0",
+        "arity": 2,
+        "acl_categories": [
+            "READ",
+            "FAST",
+            "BLOOM"
+        ],
+        "arguments": [
+            {
+                "name": "key",
+                "type": "key",
+                "key_spec_index": 0
+            }
+        ]
+    }
+}

src/commands/bf.exists.json

@@ -0,0 +1,25 @@
+{
+    "BF.EXISTS": {
+        "summary": "Determines if the bloom filter contains the specified item",
+        "complexity": "O(N), where N is the number of hash functions used by the bloom filter.",
+        "group": "bloom",
+        "module_since": "1.0.0",
+        "arity": 3,
+        "acl_categories": [
+            "READ",
+            "FAST",
+            "BLOOM"
+        ],
+        "arguments": [
+            {
+                "name": "key",
+                "type": "key",
+                "key_spec_index": 0
+            },
+            {
+                "name": "value",
+                "type": "string"
+            }
+        ]
+    }
+}