Update to arrow 38 (#6115)

* Update to arrow 38

* Further fixes

* Update sqllogictest

Author: tustvold

Cargo.toml

@@ -46,12 +46,12 @@ repository = "https://github.com/apache/arrow-datafusion"
 rust-version = "1.64"
 
 [workspace.dependencies]
-arrow = { version = "37.0.0", features = ["prettyprint"] }
-arrow-flight = { version = "37.0.0", features = ["flight-sql-experimental"] }
-arrow-buffer = { version = "37.0.0", default-features = false }
-arrow-schema = { version = "37.0.0", default-features = false }
-arrow-array = { version = "37.0.0", default-features = false, features = ["chrono-tz"] }
-parquet = { version = "37.0.0", features = ["arrow", "async"] }
+arrow = { version = "38.0.0", features = ["prettyprint"] }
+arrow-flight = { version = "38.0.0", features = ["flight-sql-experimental"] }
+arrow-buffer = { version = "38.0.0", default-features = false }
+arrow-schema = { version = "38.0.0", default-features = false }
+arrow-array = { version = "38.0.0", default-features = false, features = ["chrono-tz"] }
+parquet = { version = "38.0.0", features = ["arrow", "async"] }
 
 [profile.release]
 codegen-units = 1

datafusion-cli/Cargo.lock

@@ -29,7 +29,7 @@ rust-version = "1.62"
 readme = "README.md"
 
 [dependencies]
-arrow = "37.0.0"
+arrow = "38.0.0"
 async-trait = "0.1.41"
 clap = { version = "3", features = ["derive", "cargo"] }
 datafusion = { path = "../datafusion/core", version = "23.0.0" }

datafusion-cli/Cargo.toml

@@ -26,9 +26,9 @@ use arrow_flight::sql::{
     ActionCreatePreparedStatementResult, Any, CommandGetCatalogs,
     CommandGetCrossReference, CommandGetDbSchemas, CommandGetExportedKeys,
     CommandGetImportedKeys, CommandGetPrimaryKeys, CommandGetSqlInfo,
-    CommandGetTableTypes, CommandGetTables, CommandPreparedStatementQuery,
-    CommandPreparedStatementUpdate, CommandStatementQuery, CommandStatementUpdate,
-    ProstMessageExt, SqlInfo, TicketStatementQuery,
+    CommandGetTableTypes, CommandGetTables, CommandGetXdbcTypeInfo,
+    CommandPreparedStatementQuery, CommandPreparedStatementUpdate, CommandStatementQuery,
+    CommandStatementUpdate, ProstMessageExt, SqlInfo, TicketStatementQuery,
 };
 use arrow_flight::{
     Action, FlightData, FlightDescriptor, FlightEndpoint, FlightInfo, HandshakeRequest,
@@ -408,6 +408,17 @@ impl FlightSqlService for FlightSqlServiceImpl {
         ))
     }
 
+    async fn get_flight_info_xdbc_type_info(
+        &self,
+        _query: CommandGetXdbcTypeInfo,
+        _request: Request<FlightDescriptor>,
+    ) -> Result<Response<FlightInfo>, Status> {
+        info!("get_flight_info_xdbc_type_info");
+        Err(Status::unimplemented(
+            "Implement get_flight_info_xdbc_type_info",
+        ))
+    }
+
     async fn do_get_statement(
         &self,
         _ticket: TicketStatementQuery,
@@ -507,6 +518,15 @@ impl FlightSqlService for FlightSqlServiceImpl {
         Err(Status::unimplemented("Implement do_get_cross_reference"))
     }
 
+    async fn do_get_xdbc_type_info(
+        &self,
+        _query: CommandGetXdbcTypeInfo,
+        _request: Request<Ticket>,
+    ) -> Result<Response<<Self as FlightService>::DoGetStream>, Status> {
+        info!("do_get_xdbc_type_info");
+        Err(Status::unimplemented("Implement do_get_xdbc_type_info"))
+    }
+
     async fn do_put_statement_update(
         &self,
         _ticket: CommandStatementUpdate,

datafusion-examples/examples/flight_sql_server.rs

@@ -1472,15 +1472,15 @@ fn dict_from_scalar<K: ArrowDictionaryKeyType>(
     // APIs and skipping validation, if it every comes up in
     // performance traces.
     Arc::new(
-        DictionaryArray::<K>::try_new(&key_array, &values_array)
+        DictionaryArray::<K>::try_new(key_array, values_array)
             // should always be valid by construction above
             .expect("Can not construct dictionary array"),
     )
 }
 
 /// Create a dictionary array representing all the values in values
 fn dict_from_values<K: ArrowDictionaryKeyType>(
-    values_array: &dyn Array,
+    values_array: ArrayRef,
 ) -> Result<ArrayRef> {
     // Create a key array with `size` elements of 0..array_len for all
     // non-null value elements
@@ -1508,7 +1508,7 @@ fn dict_from_values<K: ArrowDictionaryKeyType>(
     // Note: this path could be made faster by using the ArrayData
     // APIs and skipping validation, if it every comes up in
     // performance traces.
-    let dict_array = DictionaryArray::<K>::try_new(&key_array, values_array)?;
+    let dict_array = DictionaryArray::<K>::try_new(key_array, values_array)?;
     Ok(Arc::new(dict_array))
 }
 
@@ -2406,14 +2406,14 @@ impl ScalarValue {
                 assert_eq!(values.data_type(), value_type.as_ref());
 
                 match key_type.as_ref() {
-                    DataType::Int8 => dict_from_values::<Int8Type>(&values)?,
-                    DataType::Int16 => dict_from_values::<Int16Type>(&values)?,
-                    DataType::Int32 => dict_from_values::<Int32Type>(&values)?,
-                    DataType::Int64 => dict_from_values::<Int64Type>(&values)?,
-                    DataType::UInt8 => dict_from_values::<UInt8Type>(&values)?,
-                    DataType::UInt16 => dict_from_values::<UInt16Type>(&values)?,
-                    DataType::UInt32 => dict_from_values::<UInt32Type>(&values)?,
-                    DataType::UInt64 => dict_from_values::<UInt64Type>(&values)?,
+                    DataType::Int8 => dict_from_values::<Int8Type>(values)?,
+                    DataType::Int16 => dict_from_values::<Int16Type>(values)?,
+                    DataType::Int32 => dict_from_values::<Int32Type>(values)?,
+                    DataType::Int64 => dict_from_values::<Int64Type>(values)?,
+                    DataType::UInt8 => dict_from_values::<UInt8Type>(values)?,
+                    DataType::UInt16 => dict_from_values::<UInt16Type>(values)?,
+                    DataType::UInt32 => dict_from_values::<UInt32Type>(values)?,
+                    DataType::UInt64 => dict_from_values::<UInt64Type>(values)?,
                     _ => unreachable!("Invalid dictionary keys type: {:?}", key_type),
                 }
             }

datafusion/common/src/scalar.rs

@@ -33,7 +33,7 @@ use arrow::{datatypes::SchemaRef, json};
 use bytes::{Buf, Bytes};
 
 use crate::physical_plan::common::AbortOnDropSingle;
-use arrow::json::RawReaderBuilder;
+use arrow::json::ReaderBuilder;
 use futures::{ready, stream, StreamExt, TryStreamExt};
 use object_store::{GetResult, ObjectStore};
 use std::any::Any;
@@ -198,13 +198,13 @@ impl FileOpener for JsonOpener {
             match store.get(file_meta.location()).await? {
                 GetResult::File(file, _) => {
                     let bytes = file_compression_type.convert_read(file)?;
-                    let reader = RawReaderBuilder::new(schema)
+                    let reader = ReaderBuilder::new(schema)
                         .with_batch_size(batch_size)
                         .build(BufReader::new(bytes))?;
                     Ok(futures::stream::iter(reader).boxed())
                 }
                 GetResult::Stream(s) => {
-                    let mut decoder = RawReaderBuilder::new(schema)
+                    let mut decoder = ReaderBuilder::new(schema)
                         .with_batch_size(batch_size)
                         .build_decoder()?;
 

datafusion/core/src/physical_plan/file_format/json.rs

@@ -1089,7 +1089,7 @@ mod tests {
     #[tokio::test]
     async fn test_sort_fetch_memory_calculation() -> Result<()> {
         // This test mirrors down the size from the example above.
-        let avg_batch_size = 6000;
+        let avg_batch_size = 5000;
         let partitions = 4;
 
         // A tuple of (fetch, expect_spillage)

datafusion/core/src/physical_plan/sorts/sort.rs

@@ -303,7 +303,7 @@ select arrow_cast('30 minutes', 'Interval(MonthDayNano)');
 
 ## Duration
 
-query error DataFusion error: Error during planning: Cannot automatically convert Interval\(MonthDayNano\) to Duration\(Second\)
+query error DataFusion error: This feature is not implemented: Can't create a scalar from array of type "Duration\(Second\)"
 ---
 select arrow_cast(interval '30 minutes', 'Duration(Second)');