diff --git a/Cargo.lock b/Cargo.lock index 16fccdb2865b..6dfd091aed41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -246,9 +246,8 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "54.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc208515aa0151028e464cc94a692156e945ce5126abd3537bb7fd6ba2143ed1" +version = "54.3.1" +source = "git+https://github.com/XiangpengHao/arrow-rs.git?rev=7c10b4a93d99328928434462a044098993442a0f#7c10b4a93d99328928434462a044098993442a0f" dependencies = [ "arrow-arith", "arrow-array", @@ -265,14 +264,13 @@ dependencies = [ "arrow-string", "half", "pyo3", - "rand 0.8.5", + "rand 0.9.0", ] [[package]] name = "arrow-arith" -version = "54.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e07e726e2b3f7816a85c6a45b6ec118eeeabf0b2a8c208122ad949437181f49a" +version = "54.3.1" +source = "git+https://github.com/XiangpengHao/arrow-rs.git?rev=7c10b4a93d99328928434462a044098993442a0f#7c10b4a93d99328928434462a044098993442a0f" dependencies = [ "arrow-array", "arrow-buffer", @@ -284,9 +282,8 @@ dependencies = [ [[package]] name = "arrow-array" -version = "54.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2262eba4f16c78496adfd559a29fe4b24df6088efc9985a873d58e92be022d5" +version = "54.3.1" +source = "git+https://github.com/XiangpengHao/arrow-rs.git?rev=7c10b4a93d99328928434462a044098993442a0f#7c10b4a93d99328928434462a044098993442a0f" dependencies = [ "ahash 0.8.11", "arrow-buffer", @@ -301,9 +298,8 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "54.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e899dade2c3b7f5642eb8366cfd898958bcca099cde6dfea543c7e8d3ad88d4" +version = "54.3.1" +source = "git+https://github.com/XiangpengHao/arrow-rs.git?rev=7c10b4a93d99328928434462a044098993442a0f#7c10b4a93d99328928434462a044098993442a0f" dependencies = [ "bytes", "half", @@ -312,9 +308,8 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "54.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4103d88c5b441525ed4ac23153be7458494c2b0c9a11115848fdb9b81f6f886a" +version = "54.3.1" +source = "git+https://github.com/XiangpengHao/arrow-rs.git?rev=7c10b4a93d99328928434462a044098993442a0f#7c10b4a93d99328928434462a044098993442a0f" dependencies = [ "arrow-array", "arrow-buffer", @@ -333,9 +328,8 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "54.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d3cb0914486a3cae19a5cad2598e44e225d53157926d0ada03c20521191a65" +version = "54.3.1" +source = "git+https://github.com/XiangpengHao/arrow-rs.git?rev=7c10b4a93d99328928434462a044098993442a0f#7c10b4a93d99328928434462a044098993442a0f" dependencies = [ "arrow-array", "arrow-cast", @@ -349,9 +343,8 @@ dependencies = [ [[package]] name = "arrow-data" -version = "54.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a329fb064477c9ec5f0870d2f5130966f91055c7c5bce2b3a084f116bc28c3b" +version = "54.3.1" +source = "git+https://github.com/XiangpengHao/arrow-rs.git?rev=7c10b4a93d99328928434462a044098993442a0f#7c10b4a93d99328928434462a044098993442a0f" dependencies = [ "arrow-buffer", "arrow-schema", @@ -361,9 +354,8 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "54.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7408f2bf3b978eddda272c7699f439760ebc4ac70feca25fefa82c5b8ce808d" +version = "54.3.1" +source = "git+https://github.com/XiangpengHao/arrow-rs.git?rev=7c10b4a93d99328928434462a044098993442a0f#7c10b4a93d99328928434462a044098993442a0f" dependencies = [ "arrow-arith", "arrow-array", @@ -388,9 +380,8 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "54.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddecdeab02491b1ce88885986e25002a3da34dd349f682c7cfe67bab7cc17b86" +version = "54.3.1" +source = "git+https://github.com/XiangpengHao/arrow-rs.git?rev=7c10b4a93d99328928434462a044098993442a0f#7c10b4a93d99328928434462a044098993442a0f" dependencies = [ "arrow-array", "arrow-buffer", @@ -402,9 +393,8 @@ dependencies = [ [[package]] name = "arrow-json" -version = "54.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d03b9340013413eb84868682ace00a1098c81a5ebc96d279f7ebf9a4cac3c0fd" +version = "54.3.1" +source = "git+https://github.com/XiangpengHao/arrow-rs.git?rev=7c10b4a93d99328928434462a044098993442a0f#7c10b4a93d99328928434462a044098993442a0f" dependencies = [ "arrow-array", "arrow-buffer", @@ -415,16 +405,17 @@ dependencies = [ "half", "indexmap 2.8.0", "lexical-core", + "memchr", "num", "serde", "serde_json", + "simdutf8", ] [[package]] name = "arrow-ord" -version = "54.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f841bfcc1997ef6ac48ee0305c4dfceb1f7c786fe31e67c1186edf775e1f1160" +version = "54.3.1" +source = "git+https://github.com/XiangpengHao/arrow-rs.git?rev=7c10b4a93d99328928434462a044098993442a0f#7c10b4a93d99328928434462a044098993442a0f" dependencies = [ "arrow-array", "arrow-buffer", @@ -435,9 +426,8 @@ dependencies = [ [[package]] name = "arrow-row" -version = "54.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1eeb55b0a0a83851aa01f2ca5ee5648f607e8506ba6802577afdda9d75cdedcd" +version = "54.3.1" +source = "git+https://github.com/XiangpengHao/arrow-rs.git?rev=7c10b4a93d99328928434462a044098993442a0f#7c10b4a93d99328928434462a044098993442a0f" dependencies = [ "arrow-array", "arrow-buffer", @@ -448,9 +438,8 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "54.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85934a9d0261e0fa5d4e2a5295107d743b543a6e0484a835d4b8db2da15306f9" +version = "54.3.1" +source = "git+https://github.com/XiangpengHao/arrow-rs.git?rev=7c10b4a93d99328928434462a044098993442a0f#7c10b4a93d99328928434462a044098993442a0f" dependencies = [ "bitflags 2.8.0", "serde", @@ -458,9 +447,8 @@ dependencies = [ [[package]] name = "arrow-select" -version = "54.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e2932aece2d0c869dd2125feb9bd1709ef5c445daa3838ac4112dcfa0fda52c" +version = "54.3.1" +source = "git+https://github.com/XiangpengHao/arrow-rs.git?rev=7c10b4a93d99328928434462a044098993442a0f#7c10b4a93d99328928434462a044098993442a0f" dependencies = [ "ahash 0.8.11", "arrow-array", @@ -472,9 +460,8 @@ dependencies = [ [[package]] name = "arrow-string" -version = "54.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "912e38bd6a7a7714c1d9b61df80315685553b7455e8a6045c27531d8ecd5b458" +version = "54.3.1" +source = "git+https://github.com/XiangpengHao/arrow-rs.git?rev=7c10b4a93d99328928434462a044098993442a0f#7c10b4a93d99328928434462a044098993442a0f" dependencies = [ "arrow-array", "arrow-buffer", @@ -1347,9 +1334,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.39" +version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" +checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1357,7 +1344,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.6", + "windows-link", ] [[package]] @@ -2903,11 +2890,11 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flatbuffers" -version = "24.12.23" +version = "25.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096" +checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.8.0", "rustc_version", ] @@ -2918,6 +2905,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" dependencies = [ "crc32fast", + "libz-rs-sys", "miniz_oxide", ] @@ -3897,7 +3885,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.48.5", ] [[package]] @@ -3939,6 +3927,15 @@ dependencies = [ "escape8259", ] +[[package]] +name = "libz-rs-sys" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "902bc563b5d65ad9bba616b490842ef0651066a1a1dc3ce1087113ffcb873c8d" +dependencies = [ + "zlib-rs", +] + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -3985,7 +3982,7 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" dependencies = [ - "twox-hash", + "twox-hash 1.6.3", ] [[package]] @@ -4346,9 +4343,8 @@ dependencies = [ [[package]] name = "parquet" -version = "54.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f88838dca3b84d41444a0341b19f347e8098a3898b0f21536654b8b799e11abd" +version = "54.3.1" +source = "git+https://github.com/XiangpengHao/arrow-rs.git?rev=7c10b4a93d99328928434462a044098993442a0f#7c10b4a93d99328928434462a044098993442a0f" dependencies = [ "ahash 0.8.11", "arrow-array", @@ -4376,9 +4372,8 @@ dependencies = [ "snap", "thrift", "tokio", - "twox-hash", + "twox-hash 2.1.0", "zstd", - "zstd-sys", ] [[package]] @@ -4825,9 +4820,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.23.5" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872" +checksum = "7f1c6c3591120564d64db2261bec5f910ae454f01def849b9c22835a84695e86" dependencies = [ "cfg-if", "indoc", @@ -4843,9 +4838,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.23.5" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb" +checksum = "e9b6c2b34cf71427ea37c7001aefbaeb85886a074795e35f161f5aecc7620a7a" dependencies = [ "once_cell", "target-lexicon", @@ -4853,9 +4848,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.23.5" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d" +checksum = "5507651906a46432cdda02cd02dd0319f6064f1374c9147c45b978621d2c3a9c" dependencies = [ "libc", "pyo3-build-config", @@ -4863,9 +4858,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.23.5" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da" +checksum = "b0d394b5b4fd8d97d48336bb0dd2aebabad39f1d294edd6bcd2cccf2eefe6f42" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -4875,9 +4870,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.23.5" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028" +checksum = "fd72da09cfa943b1080f621f024d2ef7e2773df7badd51aa30a2be1f8caa7c8e" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -6113,9 +6108,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "target-lexicon" -version = "0.12.16" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" +checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" [[package]] name = "tempfile" @@ -6616,6 +6611,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "twox-hash" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7b17f197b3050ba473acf9181f7b1d3b66d1cf7356c6cc57886662276e65908" + [[package]] name = "typed-arena" version = "2.0.2" @@ -7046,7 +7047,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.48.0", ] [[package]] @@ -7108,6 +7109,12 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "windows-link" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + [[package]] name = "windows-registry" version = "0.2.0" @@ -7474,6 +7481,12 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "zlib-rs" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b20717f0917c908dc63de2e44e97f1e6b126ca58d0e391cee86d504eb8fbd05" + [[package]] name = "zstd" version = "0.13.3" diff --git a/Cargo.toml b/Cargo.toml index b6164f89d31e..22435afa6659 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,19 +87,19 @@ ahash = { version = "0.8", default-features = false, features = [ "runtime-rng", ] } apache-avro = { version = "0.17", default-features = false } -arrow = { version = "54.2.1", features = [ +arrow = { version = "54.3.1", features = [ "prettyprint", "chrono-tz", ] } -arrow-buffer = { version = "54.1.0", default-features = false } -arrow-flight = { version = "54.2.1", features = [ +arrow-buffer = { version = "54.3.1", default-features = false } +arrow-flight = { version = "54.3.1", features = [ "flight-sql-experimental", ] } -arrow-ipc = { version = "54.2.0", default-features = false, features = [ +arrow-ipc = { version = "54.3.1", default-features = false, features = [ "lz4", ] } -arrow-ord = { version = "54.1.0", default-features = false } -arrow-schema = { version = "54.1.0", default-features = false } +arrow-ord = { version = "54.3.1", default-features = false } +arrow-schema = { version = "54.3.1", default-features = false } async-trait = "0.1.88" bigdecimal = "0.4.7" bytes = "1.10" @@ -149,7 +149,7 @@ itertools = "0.14" log = "^0.4" object_store = { version = "0.11.0", default-features = false } parking_lot = "0.12" -parquet = { version = "54.2.1", default-features = false, features = [ +parquet = { version = "54.3.1", default-features = false, features = [ "arrow", "async", "object_store", @@ -206,3 +206,21 @@ used_underscore_binding = "warn" [workspace.lints.rust] unexpected_cfgs = { level = "warn", check-cfg = ["cfg(tarpaulin)"] } unused_qualifications = "deny" + + +## Temporary arrow-rs patch +## https://github.com/apache/arrow-rs/pull/6921cli + +[patch.crates-io] +arrow = { git = "https://github.com/XiangpengHao/arrow-rs.git", rev = "7c10b4a93d99328928434462a044098993442a0f" } +arrow-array = { git = "https://github.com/XiangpengHao/arrow-rs.git", rev = "7c10b4a93d99328928434462a044098993442a0f" } +arrow-buffer = { git = "https://github.com/XiangpengHao/arrow-rs.git", rev = "7c10b4a93d99328928434462a044098993442a0f" } +arrow-cast = { git = "https://github.com/XiangpengHao/arrow-rs.git", rev = "7c10b4a93d99328928434462a044098993442a0f" } +arrow-data = { git = "https://github.com/XiangpengHao/arrow-rs.git", rev = "7c10b4a93d99328928434462a044098993442a0f" } +arrow-ipc = { git = "https://github.com/XiangpengHao/arrow-rs.git", rev = "7c10b4a93d99328928434462a044098993442a0f" } +arrow-schema = { git = "https://github.com/XiangpengHao/arrow-rs.git", rev = "7c10b4a93d99328928434462a044098993442a0f" } +arrow-select = { git = "https://github.com/XiangpengHao/arrow-rs.git", rev = "7c10b4a93d99328928434462a044098993442a0f" } +arrow-string = { git = "https://github.com/XiangpengHao/arrow-rs.git", rev = "7c10b4a93d99328928434462a044098993442a0f" } +arrow-ord = { git = "https://github.com/XiangpengHao/arrow-rs.git", rev = "7c10b4a93d99328928434462a044098993442a0f" } +arrow-flight = { git = "https://github.com/XiangpengHao/arrow-rs.git", rev = "7c10b4a93d99328928434462a044098993442a0f" } +parquet = { git = "https://github.com/XiangpengHao/arrow-rs.git", rev = "7c10b4a93d99328928434462a044098993442a0f" } diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml index 39b47a96bccf..74e99163955e 100644 --- a/datafusion/common/Cargo.toml +++ b/datafusion/common/Cargo.toml @@ -63,7 +63,7 @@ log = { workspace = true } object_store = { workspace = true, optional = true } parquet = { workspace = true, optional = true, default-features = true } paste = "1.0.15" -pyo3 = { version = "0.23.5", optional = true } +pyo3 = { version = "0.24.0", optional = true } recursive = { workspace = true, optional = true } sqlparser = { workspace = true } tokio = { workspace = true } diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index b0f17630c910..13537a7ace30 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -432,12 +432,12 @@ config_namespace! { /// (reading) If true, filter expressions are be applied during the parquet decoding operation to /// reduce the number of rows decoded. This optimization is sometimes called "late materialization". - pub pushdown_filters: bool, default = false + pub pushdown_filters: bool, default = true /// (reading) If true, filter expressions evaluated during the parquet decoding operation /// will be reordered heuristically to minimize the cost of evaluation. If false, /// the filters are applied in the same order as written in the query - pub reorder_filters: bool, default = false + pub reorder_filters: bool, default = true /// (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, /// and `Binary/BinaryLarge` with `BinaryView`. diff --git a/datafusion/functions-aggregate/benches/array_agg.rs b/datafusion/functions-aggregate/benches/array_agg.rs index fb605e87ed0c..e22be611d8d7 100644 --- a/datafusion/functions-aggregate/benches/array_agg.rs +++ b/datafusion/functions-aggregate/benches/array_agg.rs @@ -19,17 +19,23 @@ use std::sync::Arc; use arrow::array::{ Array, ArrayRef, ArrowPrimitiveType, AsArray, ListArray, NullBufferBuilder, + PrimitiveArray, }; use arrow::datatypes::{Field, Int64Type}; -use arrow::util::bench_util::create_primitive_array; use criterion::{black_box, criterion_group, criterion_main, Criterion}; use datafusion_expr::Accumulator; use datafusion_functions_aggregate::array_agg::ArrayAggAccumulator; use arrow::buffer::OffsetBuffer; -use arrow::util::test_util::seedable_rng; use rand::distributions::{Distribution, Standard}; +use rand::prelude::StdRng; use rand::Rng; +use rand::SeedableRng; + +/// Returns fixed seedable RNG +pub fn seedable_rng() -> StdRng { + StdRng::seed_from_u64(42) +} fn merge_batch_bench(c: &mut Criterion, name: &str, values: ArrayRef) { let list_item_data_type = values.as_list::().values().data_type().clone(); @@ -46,6 +52,24 @@ fn merge_batch_bench(c: &mut Criterion, name: &str, values: ArrayRef) { }); } +pub fn create_primitive_array(size: usize, null_density: f32) -> PrimitiveArray +where + T: ArrowPrimitiveType, + Standard: Distribution, +{ + let mut rng = seedable_rng(); + + (0..size) + .map(|_| { + if rng.gen::() < null_density { + None + } else { + Some(rng.gen()) + } + }) + .collect() +} + /// Create List array with the given item data type, null density, null locations and zero length lists density /// Creates an random (but fixed-seeded) array of a given size and null density pub fn create_list_array( diff --git a/datafusion/functions/benches/chr.rs b/datafusion/functions/benches/chr.rs index 4750fb466653..8575809c21c8 100644 --- a/datafusion/functions/benches/chr.rs +++ b/datafusion/functions/benches/chr.rs @@ -17,15 +17,21 @@ extern crate criterion; -use arrow::{array::PrimitiveArray, datatypes::Int64Type, util::test_util::seedable_rng}; +use arrow::{array::PrimitiveArray, datatypes::Int64Type}; use criterion::{black_box, criterion_group, criterion_main, Criterion}; use datafusion_expr::{ColumnarValue, ScalarFunctionArgs}; use datafusion_functions::string::chr; -use rand::Rng; +use rand::{Rng, SeedableRng}; use arrow::datatypes::DataType; +use rand::rngs::StdRng; use std::sync::Arc; +/// Returns fixed seedable RNG +pub fn seedable_rng() -> StdRng { + StdRng::seed_from_u64(42) +} + fn criterion_benchmark(c: &mut Criterion) { let cot_fn = chr(); let size = 1024; diff --git a/datafusion/sqllogictest/test_files/dates.slt b/datafusion/sqllogictest/test_files/dates.slt index 4425eee33373..148f0dfe64bb 100644 --- a/datafusion/sqllogictest/test_files/dates.slt +++ b/datafusion/sqllogictest/test_files/dates.slt @@ -183,7 +183,7 @@ query error input contains invalid characters SELECT to_date('2020-09-08 12/00/00+00:00', '%c', '%+') # to_date with broken formatting -query error bad or unsupported format string +query error DataFusion error: Execution error: Error parsing timestamp from '2020\-09\-08 12/00/00\+00:00' using format '%q': trailing input SELECT to_date('2020-09-08 12/00/00+00:00', '%q') statement ok diff --git a/datafusion/sqllogictest/test_files/expr/date_part.slt b/datafusion/sqllogictest/test_files/expr/date_part.slt index dec796aa59cb..39c42cbe1e97 100644 --- a/datafusion/sqllogictest/test_files/expr/date_part.slt +++ b/datafusion/sqllogictest/test_files/expr/date_part.slt @@ -884,7 +884,7 @@ SELECT extract(day from arrow_cast('14400 minutes', 'Interval(DayTime)')) query I SELECT extract(minute from arrow_cast('14400 minutes', 'Interval(DayTime)')) ---- -14400 +0 query I SELECT extract(second from arrow_cast('5.1 seconds', 'Interval(DayTime)')) @@ -894,7 +894,7 @@ SELECT extract(second from arrow_cast('5.1 seconds', 'Interval(DayTime)')) query I SELECT extract(second from arrow_cast('14400 minutes', 'Interval(DayTime)')) ---- -864000 +0 query I SELECT extract(second from arrow_cast('2 months', 'Interval(MonthDayNano)')) @@ -954,7 +954,7 @@ from t order by id; ---- 0 0 5 -1 0 15 +1 0 3 2 0 0 3 2 0 4 0 8 diff --git a/datafusion/sqllogictest/test_files/timestamps.slt b/datafusion/sqllogictest/test_files/timestamps.slt index dcbcfbfa439d..e3f8d2e4c8bb 100644 --- a/datafusion/sqllogictest/test_files/timestamps.slt +++ b/datafusion/sqllogictest/test_files/timestamps.slt @@ -2241,23 +2241,23 @@ query error input contains invalid characters SELECT to_timestamp_seconds('2020-09-08 12/00/00+00:00', '%c', '%+') # to_timestamp with broken formatting -query error bad or unsupported format string +query error DataFusion error: Execution error: Error parsing timestamp from '2020\-09\-08 12/00/00\+00:00' using format '%q': trailing input SELECT to_timestamp('2020-09-08 12/00/00+00:00', '%q') # to_timestamp_nanos with broken formatting -query error bad or unsupported format string +query error DataFusion error: Execution error: Error parsing timestamp from '2020\-09\-08 12/00/00\+00:00' using format '%q': trailing input SELECT to_timestamp_nanos('2020-09-08 12/00/00+00:00', '%q') # to_timestamp_millis with broken formatting -query error bad or unsupported format string +query error DataFusion error: Execution error: Error parsing timestamp from '2020\-09\-08 12/00/00\+00:00' using format '%q': trailing input SELECT to_timestamp_millis('2020-09-08 12/00/00+00:00', '%q') # to_timestamp_micros with broken formatting -query error bad or unsupported format string +query error DataFusion error: Execution error: Error parsing timestamp from '2020\-09\-08 12/00/00\+00:00' using format '%q': trailing input SELECT to_timestamp_micros('2020-09-08 12/00/00+00:00', '%q') # to_timestamp_seconds with broken formatting -query error bad or unsupported format string +query error DataFusion error: Execution error: Error parsing timestamp from '2020\-09\-08 12/00/00\+00:00' using format '%q': trailing input SELECT to_timestamp_seconds('2020-09-08 12/00/00+00:00', '%q') # Create string timestamp table with different formats