diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c9539c8002..b3c175a4d4 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -30,3 +30,4 @@ updates: schedule: interval: "weekly" day: "sunday" + open-pull-requests-limit: 50 diff --git a/Cargo.lock b/Cargo.lock index 4fb605b28c..8e35090e83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,7 +40,7 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ - "getrandom", + "getrandom 0.2.15", "once_cell", "version_check", ] @@ -53,7 +53,7 @@ checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", "const-random", - "getrandom", + "getrandom 0.2.15", "once_cell", "serde", "version_check", @@ -356,9 +356,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "53.3.0" +version = "53.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ece7b5bc1180e6d82d1a60e1688c199829e8842e38497563c3ab6ea813e527fd" +checksum = "c6f202a879d287099139ff0d121e7f55ae5e0efe634b8cf2106ebc27a8715dee" dependencies = [ "arrow-array", "arrow-buffer", @@ -391,9 +391,9 @@ checksum = "9579b9d8bce47aa41389fe344f2c6758279983b7c0ebb4013e283e3e91bb450e" [[package]] name = "arrow-select" -version = "53.3.0" +version = "53.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e415279094ea70323c032c6e739c48ad8d80e78a09bef7117b8718ad5bf3722" +checksum = "7471ba126d0b0aaa24b50a36bc6c25e4e74869a1fd1a5553357027a0b1c8d1f1" dependencies = [ "ahash 0.8.11", "arrow-array", @@ -717,9 +717,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3tables" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bac1682a1faaa29aeb8fb14be54727fd52fda934c4838b30b42727b810886823" +checksum = "6c4b058ceaba1bdaee241e79f8d520dfcf3009a3fe6ac2aa453a58d7a22e8548" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1375,7 +1375,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" dependencies = [ "lazy_static", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -1419,7 +1419,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom", + "getrandom 0.2.15", "once_cell", "tiny-keccak", ] @@ -2509,10 +2509,22 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.13.3+wasi-0.2.2", + "windows-targets 0.52.6", +] + [[package]] name = "gimli" version = "0.31.1" @@ -3673,7 +3685,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", ] @@ -3970,7 +3982,7 @@ dependencies = [ "chrono", "crc32c", "futures", - "getrandom", + "getrandom 0.2.15", "http 1.2.0", "log", "md-5", @@ -4521,7 +4533,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" dependencies = [ "bytes", - "getrandom", + "getrandom 0.2.15", "rand", "ring", "rustc-hash", @@ -4600,7 +4612,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.15", "serde", ] @@ -4729,7 +4741,7 @@ dependencies = [ "base64 0.22.1", "chrono", "form_urlencoded", - "getrandom", + "getrandom 0.2.15", "hex", "hmac", "home", @@ -4801,7 +4813,7 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.15", "libc", "spin", "untrusted", @@ -5187,9 +5199,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.135" +version = "1.0.138" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" +checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" dependencies = [ "itoa", "memchr", @@ -5796,13 +5808,13 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.15.0" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" +checksum = "38c246215d7d24f48ae091a2902398798e05d978b24315d6efbc00ede9a8bb91" dependencies = [ "cfg-if", "fastrand", - "getrandom", + "getrandom 0.3.1", "once_cell", "rustix", "windows-sys 0.59.0", @@ -6345,7 +6357,7 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3758f5e68192bb96cc8f9b7e2c2cfdabb435499a28499a42f8f984092adad4b" dependencies = [ - "getrandom", + "getrandom 0.2.15", "serde", ] @@ -6460,6 +6472,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.13.3+wasi-0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasite" version = "0.1.0" @@ -6611,7 +6632,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -6861,6 +6882,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen-rt" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +dependencies = [ + "bitflags 2.7.0", +] + [[package]] name = "write16" version = "1.0.0" diff --git a/Cargo.toml b/Cargo.toml index f6923b3287..68bbb3a031 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,9 +44,9 @@ array-init = "2" arrow-arith = { version = "53.3.0" } arrow-array = { version = "53.4.0" } arrow-cast = { version = "53.3.0" } -arrow-ord = { version = "53.3.0" } +arrow-ord = { version = "53.4.0" } arrow-schema = { version = "53.4.0" } -arrow-select = { version = "53.3.0" } +arrow-select = { version = "53.4.0" } arrow-string = { version = "53.3.0" } async-stream = "0.3.5" async-trait = "0.1.85" @@ -89,10 +89,10 @@ rust_decimal = "1.31" serde = { version = "1.0.204", features = ["rc"] } serde_bytes = "0.11.15" serde_derive = "1.0.204" -serde_json = "1.0.120" +serde_json = "1.0.138" serde_repr = "0.1.16" serde_with = "3.4" -tempfile = "3.15" +tempfile = "3.16" tokio = { version = "1.36", default-features = false } typed-builder = "0.20" url = "2.2.2" diff --git a/crates/catalog/s3tables/Cargo.toml b/crates/catalog/s3tables/Cargo.toml index ea46f5cde3..4bb4e47138 100644 --- a/crates/catalog/s3tables/Cargo.toml +++ b/crates/catalog/s3tables/Cargo.toml @@ -32,7 +32,7 @@ keywords = ["iceberg", "sql", "catalog"] anyhow = { workspace = true } async-trait = { workspace = true } aws-config = { workspace = true } -aws-sdk-s3tables = "1.6.0" +aws-sdk-s3tables = "1.7.0" iceberg = { workspace = true } serde_json = { workspace = true } uuid = { workspace = true, features = ["v4"] } diff --git a/crates/iceberg/src/transform/truncate.rs b/crates/iceberg/src/transform/truncate.rs index ed45987aad..0982907a66 100644 --- a/crates/iceberg/src/transform/truncate.rs +++ b/crates/iceberg/src/transform/truncate.rs @@ -42,6 +42,11 @@ impl Truncate { } } + #[inline] + fn truncate_binary(s: &[u8], width: usize) -> &[u8] { + &s[0..width] + } + #[inline] fn truncate_i32(v: i32, width: i32) -> i32 { v - v.rem_euclid(width) @@ -119,6 +124,18 @@ impl TransformFunction for Truncate { ); Ok(Arc::new(res)) } + DataType::Binary => { + let len = self.width as usize; + let res: arrow_array::BinaryArray = arrow_array::BinaryArray::from_iter( + input + .as_any() + .downcast_ref::() + .unwrap() + .iter() + .map(|v| v.map(|v| Self::truncate_binary(v, len))), + ); + Ok(Arc::new(res)) + } _ => Err(crate::Error::new( crate::ErrorKind::FeatureUnsupported, format!( @@ -747,6 +764,17 @@ mod test { .value(0), "ice" ); + + // test binary + let input = Arc::new(arrow_array::BinaryArray::from_vec(vec![b"iceberg"])); + let res = super::Truncate::new(3).transform(input).unwrap(); + assert_eq!( + res.as_any() + .downcast_ref::() + .unwrap() + .value(0), + b"ice" + ); } #[test]