feat: JSON indexing for EQL V2

Cargo.toml

@@ -9,6 +9,13 @@ edition = "2021"
 [profile.dev]
 incremental = true
 debug = true
+opt-level = 0
+split-debuginfo = "unpacked" # or "unpacked" on macOS
+
+[profile.dev.package.sqltk]
+opt-level = 0
+debug = true
+split-debuginfo = "unpacked" # or "unpacked" on macOS
 
 # [profile.dev.package]# aws-lc-sys.opt-level = 3
 # proc-macro2.opt-level = 3
@@ -17,8 +24,8 @@ debug = true
 # sqlparser.opt-level = 3
 # syn.opt-level = 3
 
-[profile.dev.build-override]
-opt-level = 3
+# [profile.dev.build-override]
+# opt-level = 3
 
 [profile.test]
 incremental = true
@@ -36,6 +43,7 @@ debug = true
 
 [workspace.dependencies]
 sqltk = { version = "0.10.0" }
+cipherstash-client = "0.23.0"
 thiserror = "2.0.9"
 tokio = { version = "1.44.2", features = ["full"] }
 tracing = "0.1"

README.md

@@ -29,17 +29,19 @@
 
 [Read the announcement](https://cipherstash.com/blog/introducing-proxy)
 
+CipherStash Proxy provides transparent, *searchable* encryption for your existing Postgres database.
 
-CipherStash Proxy provides a transparent proxy to your existing Postgres database.
+CipherStash Proxy:
+* Automatically encrypts and decrypts data with zero changes to SQL
+* Supports queries over *encrypted* values:
+   - equality
+   - comparison
+   - ordering
+   - grouping
+* Is written in Rust for high performance and strongly-typed mapping of SQL statements.
+* Manages keys using CipherStash ZeroKMS, offering up to 14x the performance of AWS KMS
 
-Proxy:
-* Automatically encrypts and decrypts the columns you specify
-* Supports most query types over encrypted values
-* Runs in a Docker container
-* Is written in Rust and uses a formal type system for SQL mapping
-* Works with CipherStash ZeroKMS and offers up to 14x the performance of AWS KMS
-
-Behind the scenes, it uses the [Encrypt Query Language](https://github.com/cipherstash/encrypt-query-language/) to index and search encrypted data.
+Behind the scenes, CipherStash Proxy uses the [Encrypt Query Language](https://github.com/cipherstash/encrypt-query-language/) to index and search encrypted data.
 
 ## Table of contents
 
@@ -54,7 +56,7 @@ Behind the scenes, it uses the [Encrypt Query Language](https://github.com/ciphe
 > [!IMPORTANT]
 > **Prerequisites:** Before you start you need to have this software installed:
 >  - [Docker](https://www.docker.com/) — see Docker's [documentation for installing](https://docs.docker.com/get-started/get-docker/)
- 
+
 
 Get up and running in local dev in < 5 minutes:
 

mise.toml

@@ -31,7 +31,7 @@ CS_PROXY__HOST = "proxy"
 # Misc
 DOCKER_CLI_HINTS = "false" # Please don't show us What's Next.
 
-CS_EQL_VERSION = "eql-2.0.4"
+CS_EQL_VERSION = "eql-2.0.6"
 
 [tools]
 "cargo:cargo-binstall" = "latest"

packages/cipherstash-proxy-integration/Cargo.toml

@@ -4,13 +4,22 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
+bytes = "1.10.1"
+cipherstash-client = { workspace = true, features = ["tokio"] }
+cipherstash-config = "0.2.3"
 cipherstash-proxy = { path = "../cipherstash-proxy/" }
 chrono = { version = "0.4.39", features = ["clock"] }
+clap = "4.5.32"
+fake = { version = "4", features = ["chrono", "derive"] }
+
+hex = "0.4.3"
+postgres-types = { version = "0.2.9", features = ["derive"] }
 rand = "0.9"
 recipher = "0.1.3"
 rustls = { version = "0.23.20", default-features = false, features = ["std"] }
 serde = "1.0"
 serde_json = "1.0"
+tap = "1.0.1"
 temp-env = "0.3.6"
 tokio = { workspace = true }
 tokio-postgres = { version = "0.7", features = [
@@ -21,14 +30,5 @@ tokio-postgres-rustls = "0.13.0"
 tokio-rustls = "0.26.0"
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
-webpki-roots = "1.0"
-
-[dev-dependencies]
-cipherstash-client = { version = "0.22.0", features = ["tokio"] }
-cipherstash-config = "0.2.3"
-clap = "4.5.32"
-fake = { version = "4", features = ["chrono", "derive"] }
-hex = "0.4.3"
-postgres-types = { version = "0.2.9", features = ["derive"] }
-tap = "1.0.1"
 uuid = { version = "1.11.0", features = ["serde", "v4"] }
+webpki-roots = "1.0"

packages/cipherstash-proxy-integration/src/common.rs

@@ -5,6 +5,7 @@ use rustls::{
     client::danger::ServerCertVerifier, crypto::aws_lc_rs::default_provider,
     pki_types::CertificateDer, ClientConfig,
 };
+use serde_json::Value;
 use std::sync::{Arc, Once};
 use tokio_postgres::{types::ToSql, Client, NoTls};
 use tracing_subscriber::{filter::Directive, EnvFilter, FmtSubscriber};
@@ -105,7 +106,7 @@ pub async fn connect_with_tls(port: u16) -> Client {
 
     tokio::spawn(async move {
         if let Err(e) = connection.await {
-            eprintln!("connection error: {}", e);
+            eprintln!("connection error: {e}");
         }
     });
     client
@@ -117,7 +118,7 @@ pub async fn connect(port: u16) -> Client {
 
     tokio::spawn(async move {
         if let Err(e) = connection.await {
-            eprintln!("connection error: {}", e);
+            eprintln!("connection error: {e}");
         }
     });
 
@@ -175,7 +176,18 @@ where
     rows.iter()
         .filter_map(|row| {
             if let tokio_postgres::SimpleQueryMessage::Row(r) = row {
-                r.get(0).and_then(|val| val.parse::<T>().ok())
+                r.get(0).and_then(|val| {
+                    // Convert string value to FromSql compatible type
+                    // Try different type conversions based on the value format
+                    // PostgreSQL returns booleans as "t" or "f" in simple queries
+
+                    // Convert PostgreSQL boolean format to binary representation
+                    match val {
+                        "t" => "true".parse::<T>().ok(),
+                        "f" => "false".parse::<T>().ok(),
+                        _ => val.parse::<T>().ok(),
+                    }
+                })
             } else {
                 None
             }
@@ -199,6 +211,33 @@ pub async fn simple_query_with_null(sql: &str) -> Vec<Option<String>> {
         .collect()
 }
 
+pub async fn insert(sql: &str, params: &[&(dyn ToSql + Sync)]) {
+    let client = connect_with_tls(PROXY).await;
+    client.query(sql, params).await.unwrap();
+}
+
+pub async fn insert_jsonb() -> Value {
+    let id = random_id();
+
+    let encrypted_jsonb = serde_json::json!({
+        "id": id,
+        "string": "hello",
+        "number": 42,
+        "nested": {
+            "number": 1815,
+            "string": "world",
+        },
+        "array_string": ["hello", "world"],
+        "array_number": [42, 84],
+    });
+
+    let sql = "INSERT INTO encrypted (id, encrypted_jsonb) VALUES ($1, $2)".to_string();
+
+    insert(&sql, &[&id, &encrypted_jsonb]).await;
+
+    encrypted_jsonb
+}
+
 ///
 /// Configure the client TLS settings.
 /// These are the settings for connecting to the database with TLS.

packages/cipherstash-proxy-integration/src/insert/insert_with_literal.rs

@@ -9,6 +9,10 @@ mod tests {
 
     macro_rules! test_insert_with_literal {
         ($name: ident, $type: ident, $pg_type: ident) => {
+            test_insert_with_literal!($name, $type, $pg_type, false);
+        };
+
+        ($name: ident, $type: ident, $pg_type: ident, $cast: expr) => {
             #[tokio::test]
             pub async fn $name() {
                 trace();
@@ -22,8 +26,14 @@ mod tests {
 
                 let expected = vec![encrypted_val.clone()];
 
-                let insert_sql = format!("INSERT INTO encrypted (id, {encrypted_col}) VALUES ($1, '{encrypted_val}')");
-                let select_sql = format!("SELECT {encrypted_col} FROM encrypted WHERE id = $1");
+                let cast_to_type: &str = if $cast {
+                    &format!("::{}", stringify!($pg_type))
+                } else {
+                    ""
+                };
+
+                let insert_sql = format!("INSERT INTO encrypted (id, {encrypted_col}) VALUES ($1, '{encrypted_val}'{cast_to_type})");
+                let select_sql = format!("SELECT {encrypted_col}{cast_to_type} FROM encrypted WHERE id = $1");
 
                 execute_query(&insert_sql, &[&id]).await;
                 let actual = query_by::<$type>(&select_sql, &id).await;
@@ -36,6 +46,10 @@ mod tests {
 
     macro_rules! test_insert_simple_query_with_literal {
         ($name: ident, $type: ident, $pg_type: ident) => {
+            test_insert_simple_query_with_literal!($name, $type, $pg_type, false);
+        };
+
+        ($name: ident, $type: ident, $pg_type: ident, $cast: expr) => {
             #[tokio::test]
             pub async fn $name() {
                 trace();
@@ -48,8 +62,14 @@ mod tests {
                 let encrypted_col = format!("encrypted_{}", stringify!($pg_type));
                 let encrypted_val = crate::value_for_type!($type, random_limited());
 
-                let insert_sql = format!("INSERT INTO encrypted (id, {encrypted_col}) VALUES ({id}, '{encrypted_val}')");
-                let select_sql = format!("SELECT {encrypted_col} FROM encrypted WHERE id = {id}");
+                let cast_to_type: &str = if $cast {
+                    &format!("::{}", stringify!($pg_type))
+                } else {
+                    ""
+                };
+
+                let insert_sql = format!("INSERT INTO encrypted (id, {encrypted_col}) VALUES ({id}, '{encrypted_val}'{cast_to_type})");
+                let select_sql = format!("SELECT {encrypted_col}{cast_to_type} FROM encrypted WHERE id = {id}");
 
 
                 let expected = vec![encrypted_val];
@@ -69,7 +89,7 @@ mod tests {
     test_insert_with_literal!(insert_with_literal_bool, bool, bool);
     test_insert_with_literal!(insert_with_literal_text, String, text);
     test_insert_with_literal!(insert_with_literal_date, NaiveDate, date);
-    test_insert_with_literal!(insert_with_literal_jsonb, Value, jsonb);
+    test_insert_with_literal!(insert_with_literal_jsonb, Value, jsonb, true);
 
     test_insert_simple_query_with_literal!(insert_simple_query_with_literal_int2, i16, int2);
     test_insert_simple_query_with_literal!(insert_simple_query_with_literal_int4, i32, int4);
@@ -78,7 +98,12 @@ mod tests {
     test_insert_simple_query_with_literal!(insert_simple_query_with_literal_bool, bool, bool);
     test_insert_simple_query_with_literal!(insert_simple_query_with_literal_text, String, text);
     test_insert_simple_query_with_literal!(insert_simple_query_with_literal_date, NaiveDate, date);
-    test_insert_simple_query_with_literal!(insert_simple_query_with_literal_jsonb, Value, jsonb);
+    test_insert_simple_query_with_literal!(
+        insert_simple_query_with_literal_jsonb,
+        Value,
+        jsonb,
+        true
+    );
 
     // -----------------------------------------------------------------
 

packages/cipherstash-proxy-integration/src/lib.rs

@@ -18,6 +18,7 @@ mod pipeline;
 mod schema_change;
 mod select;
 mod simple_protocol;
+mod support;
 
 #[macro_export]
 macro_rules! value_for_type {

packages/cipherstash-proxy-integration/src/map_literals.rs

@@ -55,12 +55,12 @@ mod tests {
         let encrypted_jsonb = serde_json::json!({"key": "value"});
 
         let sql = format!(
-            "INSERT INTO encrypted (id, encrypted_jsonb) VALUES ($1, '{encrypted_jsonb}')",
+            "INSERT INTO encrypted (id, encrypted_jsonb) VALUES ($1, '{encrypted_jsonb}'::jsonb)",
         );
 
         client.query(&sql, &[&id]).await.unwrap();
 
-        let sql = "SELECT id, encrypted_jsonb FROM encrypted WHERE id = $1";
+        let sql = "SELECT id, encrypted_jsonb::jsonb FROM encrypted WHERE id = $1";
         let rows = client.query(sql, &[&id]).await.unwrap();
 
         assert_eq!(rows.len(), 1);

packages/cipherstash-proxy-integration/src/migrate/mod.rs

@@ -47,7 +47,7 @@ mod tests {
         let config = match TandemConfig::load(&args) {
             Ok(config) => config,
             Err(err) => {
-                eprintln!("Configuration Error: {}", err);
+                eprintln!("Configuration Error: {err}");
                 panic!();
             }
         };

packages/cipherstash-proxy-integration/src/select/indexing.rs

@@ -0,0 +1,54 @@
+#[cfg(test)]
+mod tests {
+    use crate::common::{
+        connect_with_tls, insert, query_by, random_id, simple_query, trace, PROXY,
+    };
+    use tokio_postgres::types::{FromSql, ToSql};
+    use tracing::info;
+
+    #[derive(Debug, ToSql, FromSql, PartialEq)]
+    #[postgres(name = "domain_type_with_check")]
+    pub struct Domain(String);
+
+    ///
+    /// Tests insertion of custom domain type
+    ///
+    #[tokio::test]
+    async fn select_with_index() {
+        trace();
+
+        // let id = random_id();
+        // let encrypted_val = Domain("ZZ".to_string());
+
+        // CREATE INDEX ON encrypted (e eql_v2.encrypted_operator_class);
+        // SELECT ore.e FROM ore WHERE id = 42 INTO ore_term;
+
+        for n in 1..=10 {
+            let id = random_id();
+
+            let encrypted_text = format!("hello_{}", n);
+
+            let sql = format!("INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2)");
+            insert(&sql, &[&id, &encrypted_text]).await;
+        }
+
+        let client = connect_with_tls(PROXY).await;
+
+        let sql = "CREATE INDEX ON encrypted (encrypted_text eql_v2.encrypted_operator_class)";
+        let _ = client.simple_query(sql).await;
+
+        // let sql =
+        //     "EXPLAIN ANALYZE SELECT encrypted_text FROM encrypted WHERE encrypted_text <= '{\"hm\": \"abc\"}'::jsonb::eql_v2_encrypted";
+        // let result = simple_query::<String>(sql).await;
+
+        let sql = "EXPLAIN ANALYZE SELECT encrypted_text FROM encrypted WHERE encrypted_text <= $1";
+
+        let encrypted_text = "hello_10".to_string();
+        let result = query_by::<String>(sql, &encrypted_text).await;
+
+        info!("Result: {:?}", result);
+
+        // let expected = vec![encrypted_val];
+        // assert_eq!(expected, result);
+    }
+}