diff --git a/Cargo.lock b/Cargo.lock index 8eb2229c9a1dc..d40d2c4713dc6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -361,8 +361,8 @@ dependencies = [ "futures", "once_cell", "paste", - "prost", - "prost-types", + "prost 0.14.1", + "prost-types 0.14.1", "tonic", "tonic-prost", ] @@ -1131,8 +1131,8 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85a885520bf6249ab931a764ffdb87b0ceef48e6e7d807cfdb21b751e086e1ad" dependencies = [ - "prost", - "prost-types", + "prost 0.14.1", + "prost-types 0.14.1", "tonic", "tonic-prost", "ureq", @@ -1148,7 +1148,7 @@ dependencies = [ "bollard-buildkit-proto", "bytes", "chrono", - "prost", + "prost 0.14.1", "serde", "serde_json", "serde_repr", @@ -2127,7 +2127,7 @@ dependencies = [ "mimalloc", "nix", "object_store", - "prost", + "prost 0.14.1", "rand 0.9.2", "serde_json", "strum 0.27.2", @@ -2227,7 +2227,7 @@ dependencies = [ "doc-comment", "futures", "log", - "prost", + "prost 0.13.5", "semver", "tokio", ] @@ -2536,9 +2536,9 @@ dependencies = [ "datafusion-proto-common", "doc-comment", "object_store", - "pbjson", + "pbjson 0.8.0", "pretty_assertions", - "prost", + "prost 0.13.5", "serde", "serde_json", "tokio", @@ -2551,8 +2551,8 @@ dependencies = [ "arrow", "datafusion-common", "doc-comment", - "pbjson", - "prost", + "pbjson 0.8.0", + "prost 0.13.5", "serde", ] @@ -2679,7 +2679,7 @@ dependencies = [ "itertools 0.14.0", "object_store", "pbjson-types", - "prost", + "prost 0.13.5", "serde_json", "substrait", "tokio", @@ -3172,16 +3172,16 @@ dependencies = [ name = "gen" version = "0.1.0" dependencies = [ - "pbjson-build", - "prost-build", + "pbjson-build 0.8.0", + "prost-build 0.14.1", ] [[package]] name = "gen-common" version = "0.1.0" dependencies = [ - "pbjson-build", - "prost-build", + "pbjson-build 0.8.0", + "prost-build 0.14.1", ] [[package]] @@ -4447,6 +4447,16 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pbjson" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7e6349fa080353f4a597daffd05cb81572a9c031a6d4fff7e504947496fcc68" +dependencies = [ + "base64 0.21.7", + "serde", +] + [[package]] name = "pbjson" version = "0.8.0" @@ -4457,6 +4467,18 @@ dependencies = [ "serde", ] +[[package]] +name = "pbjson-build" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eea3058763d6e656105d1403cb04e0a41b7bbac6362d413e7c33be0c32279c9" +dependencies = [ + "heck", + "itertools 0.13.0", + "prost 0.13.5", + "prost-types 0.13.5", +] + [[package]] name = "pbjson-build" version = "0.8.0" @@ -4465,22 +4487,22 @@ checksum = "af22d08a625a2213a78dbb0ffa253318c5c79ce3133d32d296655a7bdfb02095" dependencies = [ "heck", "itertools 0.14.0", - "prost", - "prost-types", + "prost 0.14.1", + "prost-types 0.14.1", ] [[package]] name = "pbjson-types" -version = "0.8.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e748e28374f10a330ee3bb9f29b828c0ac79831a32bab65015ad9b661ead526" +checksum = "e54e5e7bfb1652f95bc361d76f3c780d8e526b134b85417e774166ee941f0887" dependencies = [ "bytes", "chrono", - "pbjson", - "pbjson-build", - "prost", - "prost-build", + "pbjson 0.7.0", + "pbjson-build 0.7.0", + "prost 0.13.5", + "prost-build 0.13.5", "serde", ] @@ -4735,6 +4757,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive 0.13.5", +] + [[package]] name = "prost" version = "0.14.1" @@ -4742,7 +4774,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.14.1", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools 0.14.0", + "log", + "multimap", + "once_cell", + "petgraph 0.7.1", + "prettyplease", + "prost 0.13.5", + "prost-types 0.13.5", + "regex", + "syn 2.0.113", + "tempfile", ] [[package]] @@ -4758,13 +4810,26 @@ dependencies = [ "once_cell", "petgraph 0.7.1", "prettyplease", - "prost", - "prost-types", + "prost 0.14.1", + "prost-types 0.14.1", "regex", "syn 2.0.113", "tempfile", ] +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.113", +] + [[package]] name = "prost-derive" version = "0.14.1" @@ -4778,13 +4843,22 @@ dependencies = [ "syn 2.0.113", ] +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost 0.13.5", +] + [[package]] name = "prost-types" version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9b4db3d6da204ed77bb26ba83b6122a73aeb2e87e25fbf7ad2e84c4ccbf8f72" dependencies = [ - "prost", + "prost 0.14.1", ] [[package]] @@ -5853,18 +5927,18 @@ dependencies = [ [[package]] name = "substrait" -version = "0.62.2" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0" +checksum = "de6d24c270c6c672a86c183c3a8439ba46c1936f93cf7296aa692de3b0ff0228" dependencies = [ "heck", - "pbjson", - "pbjson-build", + "pbjson 0.7.0", + "pbjson-build 0.7.0", "pbjson-types", "prettyplease", - "prost", - "prost-build", - "prost-types", + "prost 0.13.5", + "prost-build 0.13.5", + "prost-types 0.13.5", "protobuf-src", "regress", "schemars 0.8.22", @@ -6271,7 +6345,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" dependencies = [ "bytes", - "prost", + "prost 0.14.1", "tonic", ] @@ -6428,9 +6502,9 @@ checksum = "f8c1ae7cc0fdb8b842d65d127cb981574b0d2b249b74d1c7a2986863dc134f71" [[package]] name = "typify" -version = "0.5.0" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5bcc6f62eb1fa8aa4098f39b29f93dcb914e17158b76c50360911257aa629" +checksum = "7144144e97e987c94758a3017c920a027feac0799df325d6df4fc8f08d02068e" dependencies = [ "typify-impl", "typify-macro", @@ -6438,9 +6512,9 @@ dependencies = [ [[package]] name = "typify-impl" -version = "0.5.0" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1eb359f7ffa4f9ebe947fa11a1b2da054564502968db5f317b7e37693cb2240" +checksum = "062879d46aa4c9dfe0d33b035bbaf512da192131645d05deacb7033ec8581a09" dependencies = [ "heck", "log", @@ -6458,9 +6532,9 @@ dependencies = [ [[package]] name = "typify-macro" -version = "0.5.0" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "911c32f3c8514b048c1b228361bebb5e6d73aeec01696e8cc0e82e2ffef8ab7a" +checksum = "9708a3ceb6660ba3f8d2b8f0567e7d4b8b198e2b94d093b8a6077a751425de9e" dependencies = [ "proc-macro2", "quote", diff --git a/datafusion/ffi/Cargo.toml b/datafusion/ffi/Cargo.toml index 28e1b2ee5681f..d5b0375565f0c 100644 --- a/datafusion/ffi/Cargo.toml +++ b/datafusion/ffi/Cargo.toml @@ -67,7 +67,7 @@ datafusion-proto-common = { workspace = true } datafusion-session = { workspace = true } futures = { workspace = true } log = { workspace = true } -prost = { workspace = true } +prost = { version = "0.13" } semver = "1.0.27" tokio = { workspace = true } diff --git a/datafusion/functions/Cargo.toml b/datafusion/functions/Cargo.toml index 2bdc05abe3806..7aadfefcc31e0 100644 --- a/datafusion/functions/Cargo.toml +++ b/datafusion/functions/Cargo.toml @@ -85,7 +85,7 @@ md-5 = { version = "^0.10.0", optional = true } num-traits = { workspace = true } rand = { workspace = true } regex = { workspace = true, optional = true } -sha2 = { version = "^0.10.9", optional = true } +sha2 = { version = "^0.10.8", optional = true } unicode-segmentation = { version = "^1.7.1", optional = true } uuid = { version = "1.19", features = ["v4"], optional = true } diff --git a/datafusion/physical-expr/src/expressions/dynamic_filters.rs b/datafusion/physical-expr/src/expressions/dynamic_filters.rs index 7703d201aaea9..643745ac0f07e 100644 --- a/datafusion/physical-expr/src/expressions/dynamic_filters.rs +++ b/datafusion/physical-expr/src/expressions/dynamic_filters.rs @@ -310,14 +310,14 @@ impl DynamicFilterPhysicalExpr { /// that created the filter). This is useful to avoid computing expensive filter /// expressions when no consumer will actually use them. /// - /// Note: We check the inner Arc's strong_count, not the outer Arc's count, because - /// when filters are transformed (e.g., via reassign_expr_columns during filter pushdown), - /// new outer Arc instances are created via with_new_children(), but they all share the - /// same inner `Arc>`. This is what allows filter updates to propagate to - /// consumers even after transformation. + /// # Implementation Details + /// + /// We check both Arc counts to handle two cases: + /// - Transformed filters (via `with_new_children`) share the inner Arc (inner count > 1) + /// - Direct clones (via `Arc::clone`) increment the outer count (outer count > 1) pub fn is_used(self: &Arc) -> bool { // Strong count > 1 means at least one consumer is holding a reference beyond the producer. - Arc::strong_count(&self.inner) > 1 + Arc::strong_count(self) > 1 || Arc::strong_count(&self.inner) > 1 } fn render( diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs index 91fc1ee4436ee..f1c365c98434d 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs @@ -513,10 +513,8 @@ impl HashJoinExec { /// /// This method is intended for testing only and should not be used in production code. #[doc(hidden)] - pub fn dynamic_filter_for_test(&self) -> Option> { - self.dynamic_filter - .as_ref() - .map(|df| Arc::clone(&df.filter)) + pub fn dynamic_filter_for_test(&self) -> Option<&Arc> { + self.dynamic_filter.as_ref().map(|df| &df.filter) } /// Calculate order preservation flags for this hash join. diff --git a/datafusion/proto-common/Cargo.toml b/datafusion/proto-common/Cargo.toml index c67c8892a3ded..a5618e1627384 100644 --- a/datafusion/proto-common/Cargo.toml +++ b/datafusion/proto-common/Cargo.toml @@ -45,7 +45,7 @@ json = ["serde", "pbjson"] arrow = { workspace = true } datafusion-common = { workspace = true } pbjson = { workspace = true, optional = true } -prost = { workspace = true } +prost = { version = "0.13" } serde = { version = "1.0", optional = true } [dev-dependencies] diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.toml index b00bd0dcc6bfd..d86c1d70b35ac 100644 --- a/datafusion/proto/Cargo.toml +++ b/datafusion/proto/Cargo.toml @@ -68,7 +68,7 @@ datafusion-physical-plan = { workspace = true } datafusion-proto-common = { workspace = true } object_store = { workspace = true } pbjson = { workspace = true, optional = true } -prost = { workspace = true } +prost = { version = "0.13" } serde = { version = "1.0", optional = true } serde_json = { workspace = true, optional = true } diff --git a/datafusion/substrait/Cargo.toml b/datafusion/substrait/Cargo.toml index 8bfec86497ef0..0cb7e39a91974 100644 --- a/datafusion/substrait/Cargo.toml +++ b/datafusion/substrait/Cargo.toml @@ -41,9 +41,9 @@ datafusion = { workspace = true, features = ["sql"] } half = { workspace = true } itertools = { workspace = true } object_store = { workspace = true } -pbjson-types = { workspace = true } -prost = { workspace = true } -substrait = { version = "0.62", features = ["serde"] } +pbjson-types = { version = "0.7" } +prost = { version = "0.13" } +substrait = { version = "0.58", features = ["serde"] } url = { workspace = true } tokio = { workspace = true, features = ["fs"] } uuid = { version = "1.19.0", features = ["v4"] } diff --git a/datafusion/substrait/src/extensions.rs b/datafusion/substrait/src/extensions.rs index 0f848270babb9..b2b159dbf7799 100644 --- a/datafusion/substrait/src/extensions.rs +++ b/datafusion/substrait/src/extensions.rs @@ -115,13 +115,11 @@ impl TryFrom<&Vec> for Extensions { impl From for Vec { // Silence deprecation warnings for `extension_uri_reference` during the uri -> urn migration // See: https://github.com/substrait-io/substrait/issues/856 - #[expect(deprecated)] fn from(val: Extensions) -> Vec { let mut extensions = vec![]; for (f_anchor, f_name) in val.functions { let function_extension = ExtensionFunction { extension_uri_reference: u32::MAX, - extension_urn_reference: u32::MAX, function_anchor: f_anchor, name: f_name, }; @@ -134,7 +132,6 @@ impl From for Vec { for (t_anchor, t_name) in val.types { let type_extension = ExtensionType { extension_uri_reference: u32::MAX, // https://github.com/apache/datafusion/issues/11545 - extension_urn_reference: u32::MAX, // https://github.com/apache/datafusion/issues/11545 type_anchor: t_anchor, name: t_name, }; @@ -147,7 +144,6 @@ impl From for Vec { for (tv_anchor, tv_name) in val.type_variations { let type_variation_extension = ExtensionTypeVariation { extension_uri_reference: u32::MAX, // We don't register proper extension URIs yet - extension_urn_reference: u32::MAX, // We don't register proper extension URIs yet type_variation_anchor: tv_anchor, name: tv_name, }; diff --git a/datafusion/substrait/src/logical_plan/producer/expr/mod.rs b/datafusion/substrait/src/logical_plan/producer/expr/mod.rs index 5057564d370cf..8a601bb4b84c9 100644 --- a/datafusion/substrait/src/logical_plan/producer/expr/mod.rs +++ b/datafusion/substrait/src/logical_plan/producer/expr/mod.rs @@ -62,7 +62,6 @@ use substrait::version; /// message. The field names of the input schema will be serialized. // Silence deprecation warnings for `extension_uris` during the uri -> urn migration // See: https://github.com/substrait-io/substrait/issues/856 -#[expect(deprecated)] pub fn to_substrait_extended_expr( exprs: &[(&Expr, &Field)], schema: &DFSchemaRef, @@ -88,7 +87,6 @@ pub fn to_substrait_extended_expr( advanced_extensions: None, expected_type_urls: vec![], extension_uris: vec![], - extension_urns: vec![], extensions: extensions.into(), version: Some(version::version_with_producer("datafusion")), referred_expr: substrait_exprs, diff --git a/datafusion/substrait/src/logical_plan/producer/plan.rs b/datafusion/substrait/src/logical_plan/producer/plan.rs index 9396329f8d3e7..e015652a8f76c 100644 --- a/datafusion/substrait/src/logical_plan/producer/plan.rs +++ b/datafusion/substrait/src/logical_plan/producer/plan.rs @@ -26,7 +26,6 @@ use substrait::version; /// Convert DataFusion LogicalPlan to Substrait Plan // Silence deprecation warnings for `extension_uris` during the uri -> urn migration // See: https://github.com/substrait-io/substrait/issues/856 -#[expect(deprecated)] pub fn to_substrait_plan( plan: &LogicalPlan, state: &SessionState, @@ -48,13 +47,11 @@ pub fn to_substrait_plan( Ok(Box::new(Plan { version: Some(version::version_with_producer("datafusion")), extension_uris: vec![], - extension_urns: vec![], extensions: extensions.into(), relations: plan_rels, advanced_extensions: None, expected_type_urls: vec![], parameter_bindings: vec![], - type_aliases: vec![], })) }