diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml index edaad87c5f4e..d90ec3333cb9 100644 --- a/datafusion-examples/Cargo.toml +++ b/datafusion-examples/Cargo.toml @@ -54,21 +54,15 @@ path = "examples/external_dependency/query-aws-s3.rs" [dev-dependencies] arrow = { workspace = true } +# arrow_schema is required for record_batch! macro :sad: arrow-flight = { workspace = true } arrow-schema = { workspace = true } async-trait = { workspace = true } bytes = { workspace = true } dashmap = { workspace = true } +# note only use main datafusion crate for examples datafusion = { workspace = true, default-features = true, features = ["avro"] } -datafusion-catalog = { workspace = true } -datafusion-common = { workspace = true, default-features = true } -datafusion-expr = { workspace = true } -datafusion-functions-window-common = { workspace = true } -datafusion-optimizer = { workspace = true, default-features = true } -datafusion-physical-expr = { workspace = true, default-features = true } -datafusion-physical-optimizer = { workspace = true, default-features = true } datafusion-proto = { workspace = true } -datafusion-sql = { workspace = true } env_logger = { workspace = true } futures = { workspace = true } log = { workspace = true } diff --git a/datafusion-examples/examples/advanced_parquet_index.rs b/datafusion-examples/examples/advanced_parquet_index.rs index 3f7879d2e5d5..7e2139370e20 100644 --- a/datafusion-examples/examples/advanced_parquet_index.rs +++ b/datafusion-examples/examples/advanced_parquet_index.rs @@ -24,8 +24,13 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray}; -use arrow_schema::SchemaRef; +use arrow::datatypes::SchemaRef; +use async_trait::async_trait; +use bytes::Bytes; use datafusion::catalog::Session; +use datafusion::common::{ + internal_datafusion_err, DFSchema, DataFusionError, Result, ScalarValue, +}; use datafusion::datasource::listing::PartitionedFile; use datafusion::datasource::physical_plan::parquet::{ ParquetAccessPlan, ParquetExecBuilder, @@ -35,6 +40,8 @@ use datafusion::datasource::physical_plan::{ }; use datafusion::datasource::TableProvider; use datafusion::execution::object_store::ObjectStoreUrl; +use datafusion::logical_expr::utils::conjunction; +use datafusion::logical_expr::{TableProviderFilterPushDown, TableType}; use datafusion::parquet::arrow::arrow_reader::{ ArrowReaderOptions, ParquetRecordBatchReaderBuilder, RowSelection, RowSelector, }; @@ -43,20 +50,12 @@ use datafusion::parquet::arrow::ArrowWriter; use datafusion::parquet::file::metadata::ParquetMetaData; use datafusion::parquet::file::properties::{EnabledStatistics, WriterProperties}; use datafusion::parquet::schema::types::ColumnPath; +use datafusion::physical_expr::utils::{Guarantee, LiteralGuarantee}; use datafusion::physical_expr::PhysicalExpr; +use datafusion::physical_optimizer::pruning::PruningPredicate; use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; use datafusion::physical_plan::ExecutionPlan; use datafusion::prelude::*; -use datafusion_common::{ - internal_datafusion_err, DFSchema, DataFusionError, Result, ScalarValue, -}; -use datafusion_expr::utils::conjunction; -use datafusion_expr::{TableProviderFilterPushDown, TableType}; -use datafusion_physical_expr::utils::{Guarantee, LiteralGuarantee}; -use datafusion_physical_optimizer::pruning::PruningPredicate; - -use async_trait::async_trait; -use bytes::Bytes; use futures::future::BoxFuture; use futures::FutureExt; use object_store::ObjectStore; @@ -284,7 +283,7 @@ impl IndexTableProvider { .transpose()? // if there are no filters, use a literal true to have a predicate // that always evaluates to true we can pass to the index - .unwrap_or_else(|| datafusion_physical_expr::expressions::lit(true)); + .unwrap_or_else(|| datafusion::physical_expr::expressions::lit(true)); Ok(predicate) } diff --git a/datafusion-examples/examples/advanced_udaf.rs b/datafusion-examples/examples/advanced_udaf.rs index a914cea4a928..fd65c3352bbc 100644 --- a/datafusion-examples/examples/advanced_udaf.rs +++ b/datafusion-examples/examples/advanced_udaf.rs @@ -15,27 +15,25 @@ // specific language governing permissions and limitations // under the License. -use arrow_schema::{Field, Schema}; +use arrow::datatypes::{Field, Schema}; +use datafusion::physical_expr::NullState; use datafusion::{arrow::datatypes::DataType, logical_expr::Volatility}; -use datafusion_physical_expr::NullState; use std::{any::Any, sync::Arc}; -use arrow::{ - array::{ - ArrayRef, AsArray, Float32Array, PrimitiveArray, PrimitiveBuilder, UInt32Array, - }, - datatypes::{ArrowNativeTypeOp, ArrowPrimitiveType, Float64Type, UInt32Type}, - record_batch::RecordBatch, +use arrow::array::{ + ArrayRef, AsArray, Float32Array, PrimitiveArray, PrimitiveBuilder, UInt32Array, }; +use arrow::datatypes::{ArrowNativeTypeOp, ArrowPrimitiveType, Float64Type, UInt32Type}; +use arrow::record_batch::RecordBatch; +use datafusion::common::{cast::as_float64_array, ScalarValue}; use datafusion::error::Result; -use datafusion::prelude::*; -use datafusion_common::{cast::as_float64_array, ScalarValue}; -use datafusion_expr::{ +use datafusion::logical_expr::{ expr::AggregateFunction, function::{AccumulatorArgs, AggregateFunctionSimplification, StateFieldsArgs}, simplify::SimplifyInfo, - Accumulator, AggregateUDF, AggregateUDFImpl, GroupsAccumulator, Signature, + Accumulator, AggregateUDF, AggregateUDFImpl, EmitTo, GroupsAccumulator, Signature, }; +use datafusion::prelude::*; /// This example shows how to use the full AggregateUDFImpl API to implement a user /// defined aggregate function. As in the `simple_udaf.rs` example, this struct implements @@ -308,7 +306,7 @@ impl GroupsAccumulator for GeometricMeanGroupsAccumulator { } /// Generate output, as specified by `emit_to` and update the intermediate state - fn evaluate(&mut self, emit_to: datafusion_expr::EmitTo) -> Result { + fn evaluate(&mut self, emit_to: EmitTo) -> Result { let counts = emit_to.take_needed(&mut self.counts); let prods = emit_to.take_needed(&mut self.prods); let nulls = self.null_state.build(emit_to); @@ -344,7 +342,7 @@ impl GroupsAccumulator for GeometricMeanGroupsAccumulator { } // return arrays for counts and prods - fn state(&mut self, emit_to: datafusion_expr::EmitTo) -> Result> { + fn state(&mut self, emit_to: EmitTo) -> Result> { let nulls = self.null_state.build(emit_to); let nulls = Some(nulls); diff --git a/datafusion-examples/examples/advanced_udf.rs b/datafusion-examples/examples/advanced_udf.rs index ae35cff6facf..290d1c53334b 100644 --- a/datafusion-examples/examples/advanced_udf.rs +++ b/datafusion-examples/examples/advanced_udf.rs @@ -24,14 +24,14 @@ use arrow::array::{ use arrow::compute; use arrow::datatypes::{DataType, Float64Type}; use arrow::record_batch::RecordBatch; +use datafusion::common::{exec_err, internal_err, ScalarValue}; use datafusion::error::Result; +use datafusion::logical_expr::sort_properties::{ExprProperties, SortProperties}; use datafusion::logical_expr::Volatility; -use datafusion::prelude::*; -use datafusion_common::{exec_err, internal_err, ScalarValue}; -use datafusion_expr::sort_properties::{ExprProperties, SortProperties}; -use datafusion_expr::{ +use datafusion::logical_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, }; +use datafusion::prelude::*; /// This example shows how to use the full ScalarUDFImpl API to implement a user /// defined function. As in the `simple_udf.rs` example, this struct implements diff --git a/datafusion-examples/examples/advanced_udwf.rs b/datafusion-examples/examples/advanced_udwf.rs index 49e890467d21..ac326be9cb04 100644 --- a/datafusion-examples/examples/advanced_udwf.rs +++ b/datafusion-examples/examples/advanced_udwf.rs @@ -18,22 +18,24 @@ use datafusion::{arrow::datatypes::DataType, logical_expr::Volatility}; use std::any::Any; +use arrow::datatypes::Field; use arrow::{ array::{ArrayRef, AsArray, Float64Array}, datatypes::Float64Type, }; -use arrow_schema::Field; +use datafusion::common::ScalarValue; use datafusion::error::Result; use datafusion::functions_aggregate::average::avg_udaf; -use datafusion::prelude::*; -use datafusion_common::ScalarValue; -use datafusion_expr::expr::WindowFunction; -use datafusion_expr::function::{WindowFunctionSimplification, WindowUDFFieldArgs}; -use datafusion_expr::simplify::SimplifyInfo; -use datafusion_expr::{ - Expr, PartitionEvaluator, Signature, WindowFrame, WindowUDF, WindowUDFImpl, +use datafusion::logical_expr::expr::WindowFunction; +use datafusion::logical_expr::function::{ + PartitionEvaluatorArgs, WindowFunctionSimplification, WindowUDFFieldArgs, +}; +use datafusion::logical_expr::simplify::SimplifyInfo; +use datafusion::logical_expr::{ + Expr, PartitionEvaluator, Signature, WindowFrame, WindowFunctionDefinition, + WindowUDF, WindowUDFImpl, }; -use datafusion_functions_window_common::partition::PartitionEvaluatorArgs; +use datafusion::prelude::*; /// This example shows how to use the full WindowUDFImpl API to implement a user /// defined window function. As in the `simple_udwf.rs` example, this struct implements @@ -189,7 +191,7 @@ impl WindowUDFImpl for SimplifySmoothItUdf { fn simplify(&self) -> Option { let simplify = |window_function: WindowFunction, _: &dyn SimplifyInfo| { Ok(Expr::WindowFunction(WindowFunction { - fun: datafusion_expr::WindowFunctionDefinition::AggregateUDF(avg_udaf()), + fun: WindowFunctionDefinition::AggregateUDF(avg_udaf()), args: window_function.args, partition_by: window_function.partition_by, order_by: window_function.order_by, diff --git a/datafusion-examples/examples/analyzer_rule.rs b/datafusion-examples/examples/analyzer_rule.rs index aded64ed4105..cb81cd167a88 100644 --- a/datafusion-examples/examples/analyzer_rule.rs +++ b/datafusion-examples/examples/analyzer_rule.rs @@ -16,12 +16,12 @@ // under the License. use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray}; +use datafusion::common::config::ConfigOptions; +use datafusion::common::tree_node::{Transformed, TreeNode}; +use datafusion::common::Result; +use datafusion::logical_expr::{col, lit, Expr, LogicalPlan, LogicalPlanBuilder}; +use datafusion::optimizer::analyzer::AnalyzerRule; use datafusion::prelude::SessionContext; -use datafusion_common::config::ConfigOptions; -use datafusion_common::tree_node::{Transformed, TreeNode}; -use datafusion_common::Result; -use datafusion_expr::{col, lit, Expr, LogicalPlan, LogicalPlanBuilder}; -use datafusion_optimizer::analyzer::AnalyzerRule; use std::sync::{Arc, Mutex}; /// This example demonstrates how to add your own [`AnalyzerRule`] to diff --git a/datafusion-examples/examples/composed_extension_codec.rs b/datafusion-examples/examples/composed_extension_codec.rs index 5c34eccf26e1..4baefcae507f 100644 --- a/datafusion-examples/examples/composed_extension_codec.rs +++ b/datafusion-examples/examples/composed_extension_codec.rs @@ -36,11 +36,11 @@ use std::ops::Deref; use std::sync::Arc; use datafusion::common::Result; +use datafusion::common::{internal_err, DataFusionError}; +use datafusion::logical_expr::registry::FunctionRegistry; +use datafusion::logical_expr::{AggregateUDF, ScalarUDF}; use datafusion::physical_plan::{DisplayAs, ExecutionPlan}; use datafusion::prelude::SessionContext; -use datafusion_common::{internal_err, DataFusionError}; -use datafusion_expr::registry::FunctionRegistry; -use datafusion_expr::{AggregateUDF, ScalarUDF}; use datafusion_proto::physical_plan::{AsExecutionPlan, PhysicalExtensionCodec}; use datafusion_proto::protobuf; diff --git a/datafusion-examples/examples/csv_json_opener.rs b/datafusion-examples/examples/csv_json_opener.rs index 334e4c83404f..0b71432f621c 100644 --- a/datafusion-examples/examples/csv_json_opener.rs +++ b/datafusion-examples/examples/csv_json_opener.rs @@ -17,7 +17,7 @@ use std::sync::Arc; -use arrow_schema::{DataType, Field, Schema}; +use arrow::datatypes::{DataType, Field, Schema}; use datafusion::{ assert_batches_eq, datasource::{ diff --git a/datafusion-examples/examples/custom_file_format.rs b/datafusion-examples/examples/custom_file_format.rs index 95168597ebaa..5493fa21968e 100644 --- a/datafusion-examples/examples/custom_file_format.rs +++ b/datafusion-examples/examples/custom_file_format.rs @@ -19,11 +19,12 @@ use std::{any::Any, sync::Arc}; use arrow::{ array::{AsArray, RecordBatch, StringArray, UInt8Array}, - datatypes::UInt64Type, + datatypes::{DataType, Field, Schema, SchemaRef, UInt64Type}, }; -use arrow_schema::{DataType, Field, Schema, SchemaRef}; +use datafusion::common::{GetExt, Statistics}; use datafusion::execution::session_state::SessionStateBuilder; use datafusion::physical_expr::LexRequirement; +use datafusion::physical_expr::PhysicalExpr; use datafusion::{ datasource::{ file_format::{ @@ -38,8 +39,6 @@ use datafusion::{ physical_plan::ExecutionPlan, prelude::SessionContext, }; -use datafusion_common::{GetExt, Statistics}; -use datafusion_physical_expr::PhysicalExpr; use object_store::{ObjectMeta, ObjectStore}; use tempfile::tempdir; diff --git a/datafusion-examples/examples/dataframe.rs b/datafusion-examples/examples/dataframe.rs index c93d74dd85b0..6f61c164f41d 100644 --- a/datafusion-examples/examples/dataframe.rs +++ b/datafusion-examples/examples/dataframe.rs @@ -17,15 +17,15 @@ use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray}; use datafusion::arrow::datatypes::{DataType, Field, Schema}; +use datafusion::common::config::CsvOptions; +use datafusion::common::parsers::CompressionTypeVariant; +use datafusion::common::DataFusionError; +use datafusion::common::ScalarValue; use datafusion::dataframe::DataFrameWriteOptions; use datafusion::error::Result; use datafusion::functions_aggregate::average::avg; use datafusion::functions_aggregate::min_max::max; use datafusion::prelude::*; -use datafusion_common::config::CsvOptions; -use datafusion_common::parsers::CompressionTypeVariant; -use datafusion_common::DataFusionError; -use datafusion_common::ScalarValue; use std::fs::File; use std::io::Write; use std::sync::Arc; diff --git a/datafusion-examples/examples/date_time_functions.rs b/datafusion-examples/examples/date_time_functions.rs index d9e53e61d1ff..dbe9970439df 100644 --- a/datafusion-examples/examples/date_time_functions.rs +++ b/datafusion-examples/examples/date_time_functions.rs @@ -22,9 +22,9 @@ use datafusion::arrow::array::StringArray; use datafusion::arrow::datatypes::{DataType, Field, Schema}; use datafusion::arrow::record_batch::RecordBatch; use datafusion::assert_batches_eq; +use datafusion::common::assert_contains; use datafusion::error::Result; use datafusion::prelude::*; -use datafusion_common::assert_contains; #[tokio::main] async fn main() -> Result<()> { diff --git a/datafusion-examples/examples/deserialize_to_struct.rs b/datafusion-examples/examples/deserialize_to_struct.rs index 5ac3ee6187d1..d6655b3b654f 100644 --- a/datafusion-examples/examples/deserialize_to_struct.rs +++ b/datafusion-examples/examples/deserialize_to_struct.rs @@ -17,9 +17,9 @@ use arrow::array::{AsArray, PrimitiveArray}; use arrow::datatypes::{Float64Type, Int32Type}; +use datafusion::common::assert_batches_eq; use datafusion::error::Result; use datafusion::prelude::*; -use datafusion_common::assert_batches_eq; use futures::StreamExt; /// This example shows how to convert query results into Rust structs by using diff --git a/datafusion-examples/examples/expr_api.rs b/datafusion-examples/examples/expr_api.rs index 943e5d5e027c..6bfde2ebbf52 100644 --- a/datafusion-examples/examples/expr_api.rs +++ b/datafusion-examples/examples/expr_api.rs @@ -22,20 +22,20 @@ use arrow::array::{BooleanArray, Int32Array, Int8Array}; use arrow::record_batch::RecordBatch; use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; +use datafusion::common::tree_node::{Transformed, TreeNode}; use datafusion::common::DFSchema; +use datafusion::common::{ScalarValue, ToDFSchema}; use datafusion::error::Result; use datafusion::functions_aggregate::first_last::first_value_udaf; +use datafusion::logical_expr::execution_props::ExecutionProps; +use datafusion::logical_expr::expr::BinaryExpr; +use datafusion::logical_expr::interval_arithmetic::Interval; +use datafusion::logical_expr::simplify::SimplifyContext; +use datafusion::logical_expr::{ColumnarValue, ExprFunctionExt, ExprSchemable, Operator}; +use datafusion::optimizer::analyzer::type_coercion::TypeCoercionRewriter; use datafusion::optimizer::simplify_expressions::ExprSimplifier; use datafusion::physical_expr::{analyze, AnalysisContext, ExprBoundaries}; use datafusion::prelude::*; -use datafusion_common::tree_node::{Transformed, TreeNode}; -use datafusion_common::{ScalarValue, ToDFSchema}; -use datafusion_expr::execution_props::ExecutionProps; -use datafusion_expr::expr::BinaryExpr; -use datafusion_expr::interval_arithmetic::Interval; -use datafusion_expr::simplify::SimplifyContext; -use datafusion_expr::{ColumnarValue, ExprFunctionExt, ExprSchemable, Operator}; -use datafusion_optimizer::analyzer::type_coercion::TypeCoercionRewriter; /// This example demonstrates the DataFusion [`Expr`] API. /// @@ -357,7 +357,7 @@ fn type_coercion_demo() -> Result<()> { // Evaluation with an expression that has not been type coerced cannot succeed. let props = ExecutionProps::default(); let physical_expr = - datafusion_physical_expr::create_physical_expr(&expr, &df_schema, &props)?; + datafusion::physical_expr::create_physical_expr(&expr, &df_schema, &props)?; let e = physical_expr.evaluate(&batch).unwrap_err(); assert!(e .find_root() @@ -373,7 +373,7 @@ fn type_coercion_demo() -> Result<()> { let context = SimplifyContext::new(&props).with_schema(Arc::new(df_schema.clone())); let simplifier = ExprSimplifier::new(context); let coerced_expr = simplifier.coerce(expr.clone(), &df_schema)?; - let physical_expr = datafusion_physical_expr::create_physical_expr( + let physical_expr = datafusion::physical_expr::create_physical_expr( &coerced_expr, &df_schema, &props, @@ -385,7 +385,7 @@ fn type_coercion_demo() -> Result<()> { .clone() .rewrite(&mut TypeCoercionRewriter::new(&df_schema))? .data; - let physical_expr = datafusion_physical_expr::create_physical_expr( + let physical_expr = datafusion::physical_expr::create_physical_expr( &coerced_expr, &df_schema, &props, @@ -413,7 +413,7 @@ fn type_coercion_demo() -> Result<()> { } })? .data; - let physical_expr = datafusion_physical_expr::create_physical_expr( + let physical_expr = datafusion::physical_expr::create_physical_expr( &coerced_expr, &df_schema, &props, diff --git a/datafusion-examples/examples/file_stream_provider.rs b/datafusion-examples/examples/file_stream_provider.rs index e4fd937fd373..e6c59d57e98d 100644 --- a/datafusion-examples/examples/file_stream_provider.rs +++ b/datafusion-examples/examples/file_stream_provider.rs @@ -18,7 +18,7 @@ #[cfg(not(target_os = "windows"))] mod non_windows { use datafusion::assert_batches_eq; - use datafusion_common::instant::Instant; + use datafusion::common::instant::Instant; use std::fs::{File, OpenOptions}; use std::io::Write; use std::path::PathBuf; @@ -27,19 +27,18 @@ mod non_windows { use std::thread; use std::time::Duration; - use arrow::datatypes::{DataType, Field, Schema}; - use arrow_schema::SchemaRef; + use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use futures::StreamExt; use nix::sys::stat; use nix::unistd; use tempfile::TempDir; use tokio::task::JoinSet; + use datafusion::common::{exec_err, Result}; use datafusion::datasource::stream::{FileStreamProvider, StreamConfig, StreamTable}; use datafusion::datasource::TableProvider; + use datafusion::logical_expr::SortExpr; use datafusion::prelude::{SessionConfig, SessionContext}; - use datafusion_common::{exec_err, Result}; - use datafusion_expr::SortExpr; // Number of lines written to FIFO const TEST_BATCH_SIZE: usize = 5; @@ -157,7 +156,7 @@ mod non_windows { ])); // Specify the ordering: - let order = vec![vec![datafusion_expr::col("a1").sort(true, false)]]; + let order = vec![vec![datafusion::logical_expr::col("a1").sort(true, false)]]; let provider = fifo_table(schema.clone(), fifo_path, order.clone()); ctx.register_table("fifo", provider)?; @@ -189,7 +188,7 @@ mod non_windows { } #[tokio::main] -async fn main() -> datafusion_common::Result<()> { +async fn main() -> datafusion::error::Result<()> { #[cfg(target_os = "windows")] { println!("file_stream_provider example does not work on windows"); diff --git a/datafusion-examples/examples/flight/flight_sql_server.rs b/datafusion-examples/examples/flight/flight_sql_server.rs index 2e46daf7cb4e..54e8de7177cb 100644 --- a/datafusion-examples/examples/flight/flight_sql_server.rs +++ b/datafusion-examples/examples/flight/flight_sql_server.rs @@ -16,6 +16,7 @@ // under the License. use arrow::array::{ArrayRef, StringArray}; +use arrow::datatypes::{DataType, Field, Schema}; use arrow::ipc::writer::IpcWriteOptions; use arrow::record_batch::RecordBatch; use arrow_flight::encode::FlightDataEncoderBuilder; @@ -32,7 +33,6 @@ use arrow_flight::{ Action, FlightDescriptor, FlightEndpoint, FlightInfo, HandshakeRequest, HandshakeResponse, IpcMessage, SchemaAsIpc, Ticket, }; -use arrow_schema::{DataType, Field, Schema}; use dashmap::DashMap; use datafusion::logical_expr::LogicalPlan; use datafusion::prelude::{DataFrame, ParquetReadOptions, SessionConfig, SessionContext}; diff --git a/datafusion-examples/examples/function_factory.rs b/datafusion-examples/examples/function_factory.rs index 58ffa060ebaa..06367f5c09e3 100644 --- a/datafusion-examples/examples/function_factory.rs +++ b/datafusion-examples/examples/function_factory.rs @@ -15,20 +15,21 @@ // specific language governing permissions and limitations // under the License. -use std::result::Result as RResult; -use std::sync::Arc; - +use arrow::datatypes::DataType; +use datafusion::common::tree_node::{Transformed, TreeNode}; +use datafusion::common::{exec_err, internal_err, DataFusionError}; use datafusion::error::Result; use datafusion::execution::context::{ FunctionFactory, RegisterFunction, SessionContext, SessionState, }; -use datafusion_common::tree_node::{Transformed, TreeNode}; -use datafusion_common::{exec_err, internal_err, DataFusionError}; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyInfo}; -use datafusion_expr::sort_properties::{ExprProperties, SortProperties}; -use datafusion_expr::{ - CreateFunction, Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, +use datafusion::logical_expr::simplify::{ExprSimplifyResult, SimplifyInfo}; +use datafusion::logical_expr::sort_properties::{ExprProperties, SortProperties}; +use datafusion::logical_expr::{ + ColumnarValue, CreateFunction, Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, + Signature, Volatility, }; +use std::result::Result as RResult; +use std::sync::Arc; /// This example shows how to utilize [FunctionFactory] to implement simple /// SQL-macro like functions using a `CREATE FUNCTION` statement. The same @@ -111,7 +112,7 @@ struct ScalarFunctionWrapper { name: String, expr: Expr, signature: Signature, - return_type: arrow_schema::DataType, + return_type: DataType, } impl ScalarUDFImpl for ScalarFunctionWrapper { @@ -127,17 +128,11 @@ impl ScalarUDFImpl for ScalarFunctionWrapper { &self.signature } - fn return_type( - &self, - _arg_types: &[arrow_schema::DataType], - ) -> Result { + fn return_type(&self, _arg_types: &[DataType]) -> Result { Ok(self.return_type.clone()) } - fn invoke_with_args( - &self, - _args: ScalarFunctionArgs, - ) -> Result { + fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result { // Since this function is always simplified to another expression, it // should never actually be invoked internal_err!("This function should not get invoked!") @@ -226,10 +221,7 @@ impl TryFrom for ScalarFunctionWrapper { .into_iter() .map(|a| a.data_type) .collect(), - definition - .params - .behavior - .unwrap_or(datafusion_expr::Volatility::Volatile), + definition.params.behavior.unwrap_or(Volatility::Volatile), ), }) } diff --git a/datafusion-examples/examples/optimizer_rule.rs b/datafusion-examples/examples/optimizer_rule.rs index e8a272f28318..0206c7cd157e 100644 --- a/datafusion-examples/examples/optimizer_rule.rs +++ b/datafusion-examples/examples/optimizer_rule.rs @@ -16,16 +16,16 @@ // under the License. use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray}; -use arrow_schema::DataType; -use datafusion::prelude::SessionContext; -use datafusion_common::tree_node::{Transformed, TreeNode}; -use datafusion_common::{assert_batches_eq, Result, ScalarValue}; -use datafusion_expr::{ +use arrow::datatypes::DataType; +use datafusion::common::tree_node::{Transformed, TreeNode}; +use datafusion::common::{assert_batches_eq, Result, ScalarValue}; +use datafusion::logical_expr::{ BinaryExpr, ColumnarValue, Expr, LogicalPlan, Operator, ScalarUDF, ScalarUDFImpl, Signature, Volatility, }; -use datafusion_optimizer::optimizer::ApplyOrder; -use datafusion_optimizer::{OptimizerConfig, OptimizerRule}; +use datafusion::optimizer::ApplyOrder; +use datafusion::optimizer::{OptimizerConfig, OptimizerRule}; +use datafusion::prelude::SessionContext; use std::any::Any; use std::sync::Arc; diff --git a/datafusion-examples/examples/parquet_exec_visitor.rs b/datafusion-examples/examples/parquet_exec_visitor.rs index eeb288beb0df..2f1b6333373e 100644 --- a/datafusion-examples/examples/parquet_exec_visitor.rs +++ b/datafusion-examples/examples/parquet_exec_visitor.rs @@ -20,6 +20,7 @@ use std::sync::Arc; use datafusion::datasource::file_format::parquet::ParquetFormat; use datafusion::datasource::listing::{ListingOptions, PartitionedFile}; use datafusion::datasource::physical_plan::ParquetExec; +use datafusion::error::DataFusionError; use datafusion::execution::context::SessionContext; use datafusion::physical_plan::metrics::MetricValue; use datafusion::physical_plan::{ @@ -88,7 +89,7 @@ struct ParquetExecVisitor { } impl ExecutionPlanVisitor for ParquetExecVisitor { - type Error = datafusion_common::DataFusionError; + type Error = DataFusionError; /// This function is called once for every node in the tree. /// Based on your needs implement either `pre_visit` (visit each node before its children/inputs) diff --git a/datafusion-examples/examples/parquet_index.rs b/datafusion-examples/examples/parquet_index.rs index aab5fa0f9d33..67fa038e2d31 100644 --- a/datafusion-examples/examples/parquet_index.rs +++ b/datafusion-examples/examples/parquet_index.rs @@ -15,43 +15,41 @@ // specific language governing permissions and limitations // under the License. -use std::any::Any; -use std::collections::HashSet; -use std::fmt::Display; -use std::fs::{self, DirEntry, File}; -use std::ops::Range; -use std::path::{Path, PathBuf}; -use std::sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, -}; - use arrow::array::{ Array, ArrayRef, AsArray, BooleanArray, Int32Array, RecordBatch, StringArray, UInt64Array, }; -use arrow::datatypes::Int32Type; +use arrow::datatypes::{Int32Type, SchemaRef}; use arrow::util::pretty::pretty_format_batches; -use arrow_schema::SchemaRef; +use async_trait::async_trait; use datafusion::catalog::Session; +use datafusion::common::{ + internal_datafusion_err, DFSchema, DataFusionError, Result, ScalarValue, +}; use datafusion::datasource::listing::PartitionedFile; use datafusion::datasource::physical_plan::{FileScanConfig, ParquetExec}; use datafusion::datasource::TableProvider; use datafusion::execution::object_store::ObjectStoreUrl; +use datafusion::logical_expr::{ + utils::conjunction, TableProviderFilterPushDown, TableType, +}; use datafusion::parquet::arrow::arrow_reader::statistics::StatisticsConverter; use datafusion::parquet::arrow::{ arrow_reader::ParquetRecordBatchReaderBuilder, ArrowWriter, }; +use datafusion::physical_expr::PhysicalExpr; +use datafusion::physical_optimizer::pruning::{PruningPredicate, PruningStatistics}; use datafusion::physical_plan::ExecutionPlan; use datafusion::prelude::*; -use datafusion_common::{ - internal_datafusion_err, DFSchema, DataFusionError, Result, ScalarValue, -}; -use datafusion_expr::{utils::conjunction, TableProviderFilterPushDown, TableType}; -use datafusion_physical_expr::PhysicalExpr; -use datafusion_physical_optimizer::pruning::{PruningPredicate, PruningStatistics}; - -use async_trait::async_trait; +use std::any::Any; +use std::collections::HashSet; +use std::fmt::Display; +use std::fs; +use std::fs::{DirEntry, File}; +use std::ops::Range; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; use tempfile::TempDir; use url::Url; @@ -235,7 +233,7 @@ impl TableProvider for IndexTableProvider { .transpose()? // if there are no filters, use a literal true to have a predicate // that always evaluates to true we can pass to the index - .unwrap_or_else(|| datafusion_physical_expr::expressions::lit(true)); + .unwrap_or_else(|| datafusion::physical_expr::expressions::lit(true)); // Use the index to find the files that might have data that matches the // predicate. Any file that can not have data that matches the predicate diff --git a/datafusion-examples/examples/parse_sql_expr.rs b/datafusion-examples/examples/parse_sql_expr.rs index d8f0778e19e3..5387e7c4a05d 100644 --- a/datafusion-examples/examples/parse_sql_expr.rs +++ b/datafusion-examples/examples/parse_sql_expr.rs @@ -16,14 +16,14 @@ // under the License. use arrow::datatypes::{DataType, Field, Schema}; +use datafusion::common::DFSchema; +use datafusion::logical_expr::{col, lit}; +use datafusion::sql::unparser::Unparser; use datafusion::{ assert_batches_eq, error::Result, prelude::{ParquetReadOptions, SessionContext}, }; -use datafusion_common::DFSchema; -use datafusion_expr::{col, lit}; -use datafusion_sql::unparser::Unparser; /// This example demonstrates the programmatic parsing of SQL expressions using /// the DataFusion [`SessionContext::parse_sql_expr`] API or the [`DataFrame::parse_sql_expr`] API. diff --git a/datafusion-examples/examples/plan_to_sql.rs b/datafusion-examples/examples/plan_to_sql.rs index cf1202498416..54483b143a16 100644 --- a/datafusion-examples/examples/plan_to_sql.rs +++ b/datafusion-examples/examples/plan_to_sql.rs @@ -15,24 +15,24 @@ // specific language governing permissions and limitations // under the License. +use datafusion::common::DFSchemaRef; use datafusion::error::Result; use datafusion::logical_expr::sqlparser::ast::Statement; -use datafusion::prelude::*; -use datafusion::sql::unparser::expr_to_sql; -use datafusion_common::DFSchemaRef; -use datafusion_expr::{ +use datafusion::logical_expr::{ Extension, LogicalPlan, LogicalPlanBuilder, UserDefinedLogicalNode, UserDefinedLogicalNodeCore, }; -use datafusion_sql::unparser::ast::{ +use datafusion::prelude::*; +use datafusion::sql::unparser::ast::{ DerivedRelationBuilder, QueryBuilder, RelationBuilder, SelectBuilder, }; -use datafusion_sql::unparser::dialect::CustomDialectBuilder; -use datafusion_sql::unparser::extension_unparser::UserDefinedLogicalNodeUnparser; -use datafusion_sql::unparser::extension_unparser::{ +use datafusion::sql::unparser::dialect::CustomDialectBuilder; +use datafusion::sql::unparser::expr_to_sql; +use datafusion::sql::unparser::extension_unparser::UserDefinedLogicalNodeUnparser; +use datafusion::sql::unparser::extension_unparser::{ UnparseToStatementResult, UnparseWithinStatementResult, }; -use datafusion_sql::unparser::{plan_to_sql, Unparser}; +use datafusion::sql::unparser::{plan_to_sql, Unparser}; use std::fmt; use std::sync::Arc; diff --git a/datafusion-examples/examples/planner_api.rs b/datafusion-examples/examples/planner_api.rs index 35cf766ba1af..e52f0d78682f 100644 --- a/datafusion-examples/examples/planner_api.rs +++ b/datafusion-examples/examples/planner_api.rs @@ -16,10 +16,10 @@ // under the License. use datafusion::error::Result; +use datafusion::logical_expr::{LogicalPlan, PlanType}; use datafusion::physical_plan::displayable; use datafusion::physical_planner::DefaultPhysicalPlanner; use datafusion::prelude::*; -use datafusion_expr::{LogicalPlan, PlanType}; /// This example demonstrates the process of converting logical plan /// into physical execution plans using DataFusion. diff --git a/datafusion-examples/examples/pruning.rs b/datafusion-examples/examples/pruning.rs index 88f573056442..4c802bcdbda0 100644 --- a/datafusion-examples/examples/pruning.rs +++ b/datafusion-examples/examples/pruning.rs @@ -23,8 +23,8 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion::common::{DFSchema, ScalarValue}; use datafusion::execution::context::ExecutionProps; use datafusion::physical_expr::create_physical_expr; +use datafusion::physical_optimizer::pruning::{PruningPredicate, PruningStatistics}; use datafusion::prelude::*; -use datafusion_physical_optimizer::pruning::{PruningPredicate, PruningStatistics}; /// This example shows how to use DataFusion's `PruningPredicate` to prove /// filter expressions can never be true based on statistics such as min/max diff --git a/datafusion-examples/examples/regexp.rs b/datafusion-examples/examples/regexp.rs index 5419efd2faea..12d115b9b502 100644 --- a/datafusion-examples/examples/regexp.rs +++ b/datafusion-examples/examples/regexp.rs @@ -16,9 +16,9 @@ // specific language governing permissions and limitations // under the License. +use datafusion::common::{assert_batches_eq, assert_contains}; use datafusion::error::Result; use datafusion::prelude::*; -use datafusion_common::{assert_batches_eq, assert_contains}; /// This example demonstrates how to use the regexp_* functions /// diff --git a/datafusion-examples/examples/remote_catalog.rs b/datafusion-examples/examples/remote_catalog.rs index 38629328d71c..e44b0c9569f9 100644 --- a/datafusion-examples/examples/remote_catalog.rs +++ b/datafusion-examples/examples/remote_catalog.rs @@ -30,18 +30,18 @@ /// [Unity]: https://github.com/unitycatalog/unitycatalog /// [Hive]: https://hive.apache.org/ use arrow::array::record_batch; -use arrow_schema::{Field, Fields, Schema, SchemaRef}; +use arrow::datatypes::{Field, Fields, Schema, SchemaRef}; use async_trait::async_trait; use datafusion::catalog::TableProvider; +use datafusion::catalog::{AsyncSchemaProvider, Session}; use datafusion::common::Result; +use datafusion::common::{assert_batches_eq, internal_datafusion_err, plan_err}; use datafusion::execution::SendableRecordBatchStream; +use datafusion::logical_expr::{Expr, TableType}; use datafusion::physical_plan::memory::MemoryExec; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::ExecutionPlan; use datafusion::prelude::{DataFrame, SessionContext}; -use datafusion_catalog::{AsyncSchemaProvider, Session}; -use datafusion_common::{assert_batches_eq, internal_datafusion_err, plan_err}; -use datafusion_expr::{Expr, TableType}; use futures::TryStreamExt; use std::any::Any; use std::sync::Arc; diff --git a/datafusion-examples/examples/simple_udaf.rs b/datafusion-examples/examples/simple_udaf.rs index ef97bf9763b0..82bde7c034a5 100644 --- a/datafusion-examples/examples/simple_udaf.rs +++ b/datafusion-examples/examples/simple_udaf.rs @@ -20,9 +20,9 @@ use datafusion::arrow::{ array::ArrayRef, array::Float32Array, datatypes::DataType, record_batch::RecordBatch, }; +use datafusion::common::cast::as_float64_array; use datafusion::{error::Result, physical_plan::Accumulator}; use datafusion::{logical_expr::Volatility, prelude::*, scalar::ScalarValue}; -use datafusion_common::cast::as_float64_array; use std::sync::Arc; // create local session context with an in-memory table diff --git a/datafusion-examples/examples/simple_udf.rs b/datafusion-examples/examples/simple_udf.rs index 6879a17f34be..5612e0939f70 100644 --- a/datafusion-examples/examples/simple_udf.rs +++ b/datafusion-examples/examples/simple_udf.rs @@ -24,10 +24,10 @@ use datafusion::{ logical_expr::Volatility, }; +use datafusion::common::cast::as_float64_array; use datafusion::error::Result; +use datafusion::logical_expr::ColumnarValue; use datafusion::prelude::*; -use datafusion_common::cast::as_float64_array; -use datafusion_expr::ColumnarValue; use std::sync::Arc; /// create local execution context with an in-memory table: diff --git a/datafusion-examples/examples/simple_udtf.rs b/datafusion-examples/examples/simple_udtf.rs index 7cf1ce87690e..75c7645c18a5 100644 --- a/datafusion-examples/examples/simple_udtf.rs +++ b/datafusion-examples/examples/simple_udtf.rs @@ -21,17 +21,17 @@ use async_trait::async_trait; use datafusion::arrow::datatypes::SchemaRef; use datafusion::arrow::record_batch::RecordBatch; use datafusion::catalog::Session; +use datafusion::catalog::TableFunctionImpl; +use datafusion::common::{plan_err, ScalarValue}; use datafusion::datasource::TableProvider; use datafusion::error::Result; use datafusion::execution::context::ExecutionProps; +use datafusion::logical_expr::simplify::SimplifyContext; +use datafusion::logical_expr::{Expr, TableType}; +use datafusion::optimizer::simplify_expressions::ExprSimplifier; use datafusion::physical_plan::memory::MemoryExec; use datafusion::physical_plan::ExecutionPlan; -use datafusion::prelude::SessionContext; -use datafusion_catalog::TableFunctionImpl; -use datafusion_common::{plan_err, ScalarValue}; -use datafusion_expr::simplify::SimplifyContext; -use datafusion_expr::{Expr, TableType}; -use datafusion_optimizer::simplify_expressions::ExprSimplifier; +use datafusion::prelude::*; use std::fs::File; use std::io::Seek; use std::path::Path; diff --git a/datafusion-examples/examples/simple_udwf.rs b/datafusion-examples/examples/simple_udwf.rs index 22dfbbbf0c3a..1736ff00bd70 100644 --- a/datafusion-examples/examples/simple_udwf.rs +++ b/datafusion-examples/examples/simple_udwf.rs @@ -19,14 +19,13 @@ use std::sync::Arc; use arrow::{ array::{ArrayRef, AsArray, Float64Array}, - datatypes::Float64Type, + datatypes::{DataType, Float64Type}, }; -use arrow_schema::DataType; +use datafusion::common::ScalarValue; use datafusion::error::Result; +use datafusion::logical_expr::{PartitionEvaluator, Volatility, WindowFrame}; use datafusion::prelude::*; -use datafusion_common::ScalarValue; -use datafusion_expr::{PartitionEvaluator, Volatility, WindowFrame}; // create local execution context with `cars.csv` registered as a table named `cars` async fn create_context() -> Result { diff --git a/datafusion-examples/examples/sql_analysis.rs b/datafusion-examples/examples/sql_analysis.rs index 2158b8e4b016..d3826026a972 100644 --- a/datafusion-examples/examples/sql_analysis.rs +++ b/datafusion-examples/examples/sql_analysis.rs @@ -23,13 +23,13 @@ use std::sync::Arc; +use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion}; use datafusion::common::Result; +use datafusion::logical_expr::LogicalPlan; use datafusion::{ datasource::MemTable, execution::context::{SessionConfig, SessionContext}, }; -use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion}; -use datafusion_expr::LogicalPlan; use test_utils::tpcds::tpcds_schemas; /// Counts the total number of joins in a plan diff --git a/datafusion-examples/examples/sql_dialect.rs b/datafusion-examples/examples/sql_dialect.rs index 259f38216b80..16aa5be02635 100644 --- a/datafusion-examples/examples/sql_dialect.rs +++ b/datafusion-examples/examples/sql_dialect.rs @@ -18,7 +18,7 @@ use std::fmt::Display; use datafusion::error::Result; -use datafusion_sql::{ +use datafusion::sql::{ parser::{CopyToSource, CopyToStatement, DFParser, Statement}, sqlparser::{keywords::Keyword, parser::ParserError, tokenizer::Token}, }; diff --git a/datafusion-examples/examples/sql_frontend.rs b/datafusion-examples/examples/sql_frontend.rs index 839ee95eb181..c7afb79084bf 100644 --- a/datafusion-examples/examples/sql_frontend.rs +++ b/datafusion-examples/examples/sql_frontend.rs @@ -16,19 +16,20 @@ // under the License. use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; -use datafusion_common::config::ConfigOptions; -use datafusion_common::{plan_err, Result}; -use datafusion_expr::{ +use datafusion::common::plan_err; +use datafusion::config::ConfigOptions; +use datafusion::error::Result; +use datafusion::logical_expr::{ AggregateUDF, Expr, LogicalPlan, ScalarUDF, TableProviderFilterPushDown, TableSource, WindowUDF, }; -use datafusion_optimizer::{ +use datafusion::optimizer::{ Analyzer, AnalyzerRule, Optimizer, OptimizerConfig, OptimizerContext, OptimizerRule, }; -use datafusion_sql::planner::{ContextProvider, SqlToRel}; -use datafusion_sql::sqlparser::dialect::PostgreSqlDialect; -use datafusion_sql::sqlparser::parser::Parser; -use datafusion_sql::TableReference; +use datafusion::sql::planner::{ContextProvider, SqlToRel}; +use datafusion::sql::sqlparser::dialect::PostgreSqlDialect; +use datafusion::sql::sqlparser::parser::Parser; +use datafusion::sql::TableReference; use std::any::Any; use std::sync::Arc; diff --git a/datafusion-examples/examples/sql_query.rs b/datafusion-examples/examples/sql_query.rs index a6e7fe91dda5..0ac203cfb7e7 100644 --- a/datafusion-examples/examples/sql_query.rs +++ b/datafusion-examples/examples/sql_query.rs @@ -18,12 +18,12 @@ use datafusion::arrow::array::{UInt64Array, UInt8Array}; use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion::arrow::record_batch::RecordBatch; +use datafusion::common::{assert_batches_eq, exec_datafusion_err}; use datafusion::datasource::file_format::parquet::ParquetFormat; use datafusion::datasource::listing::ListingOptions; use datafusion::datasource::MemTable; use datafusion::error::{DataFusionError, Result}; -use datafusion::prelude::SessionContext; -use datafusion_common::{assert_batches_eq, exec_datafusion_err}; +use datafusion::prelude::*; use object_store::local::LocalFileSystem; use std::path::Path; use std::sync::Arc; diff --git a/datafusion/core/src/lib.rs b/datafusion/core/src/lib.rs index 416b69c3afdd..e9ec2d2788e1 100644 --- a/datafusion/core/src/lib.rs +++ b/datafusion/core/src/lib.rs @@ -720,6 +720,11 @@ pub mod optimizer { pub use datafusion_optimizer::*; } +/// re-export of [`datafusion_physical_optimizer`] crate +pub mod physical_optimizer { + pub use datafusion_physical_optimizer::*; +} + /// re-export of [`datafusion_physical_expr`] crate pub mod physical_expr_common { pub use datafusion_physical_expr_common::*; diff --git a/datafusion/optimizer/src/lib.rs b/datafusion/optimizer/src/lib.rs index 263770b81fcd..614284e1b477 100644 --- a/datafusion/optimizer/src/lib.rs +++ b/datafusion/optimizer/src/lib.rs @@ -62,7 +62,9 @@ pub mod utils; pub mod test; pub use analyzer::{Analyzer, AnalyzerRule}; -pub use optimizer::{Optimizer, OptimizerConfig, OptimizerContext, OptimizerRule}; +pub use optimizer::{ + ApplyOrder, Optimizer, OptimizerConfig, OptimizerContext, OptimizerRule, +}; #[allow(deprecated)] pub use utils::optimize_children;