Skip to content

Commit aebe71e

Browse files
authored
Minor: Add debug logging for schema mismatch errors (#6626)
1 parent e6265c1 commit aebe71e

File tree

3 files changed

+44
-24
lines changed

3 files changed

+44
-24
lines changed

datafusion/core/src/datasource/memory.rs

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
//! [`MemTable`] for querying `Vec<RecordBatch>` by DataFusion.
1919
2020
use futures::StreamExt;
21+
use log::debug;
2122
use std::any::Any;
2223
use std::fmt::{self, Debug, Display};
2324
use std::sync::Arc;
@@ -55,23 +56,26 @@ pub struct MemTable {
5556
impl MemTable {
5657
/// Create a new in-memory table from the provided schema and record batches
5758
pub fn try_new(schema: SchemaRef, partitions: Vec<Vec<RecordBatch>>) -> Result<Self> {
58-
if partitions
59-
.iter()
60-
.flatten()
61-
.all(|batches| schema.contains(&batches.schema()))
62-
{
63-
Ok(Self {
64-
schema,
65-
batches: partitions
66-
.into_iter()
67-
.map(|e| Arc::new(RwLock::new(e)))
68-
.collect::<Vec<_>>(),
69-
})
70-
} else {
71-
Err(DataFusionError::Plan(
72-
"Mismatch between schema and batches".to_string(),
73-
))
59+
for batches in partitions.iter().flatten() {
60+
let batches_schema = batches.schema();
61+
if !schema.contains(&batches_schema) {
62+
debug!(
63+
"mem table schema does not contain batches schema. \
64+
Target_schema: {schema:?}. Batches Schema: {batches_schema:?}"
65+
);
66+
return Err(DataFusionError::Plan(
67+
"Mismatch between schema and batches".to_string(),
68+
));
69+
}
7470
}
71+
72+
Ok(Self {
73+
schema,
74+
batches: partitions
75+
.into_iter()
76+
.map(|e| Arc::new(RwLock::new(e)))
77+
.collect::<Vec<_>>(),
78+
})
7579
}
7680

7781
/// Create a mem table by reading from another data source

datafusion/core/src/datasource/streaming.rs

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ use async_trait::async_trait;
2525

2626
use datafusion_common::{DataFusionError, Result};
2727
use datafusion_expr::{Expr, TableType};
28+
use log::debug;
2829

2930
use crate::datasource::TableProvider;
3031
use crate::execution::context::{SessionState, TaskContext};
@@ -53,10 +54,17 @@ impl StreamingTable {
5354
schema: SchemaRef,
5455
partitions: Vec<Arc<dyn PartitionStream>>,
5556
) -> Result<Self> {
56-
if !partitions.iter().all(|x| schema.contains(x.schema())) {
57-
return Err(DataFusionError::Plan(
58-
"Mismatch between schema and batches".to_string(),
59-
));
57+
for x in partitions.iter() {
58+
let partition_schema = x.schema();
59+
if !schema.contains(partition_schema) {
60+
debug!(
61+
"target schema does not contain partition schema. \
62+
Target_schema: {schema:?}. Partiton Schema: {partition_schema:?}"
63+
);
64+
return Err(DataFusionError::Plan(
65+
"Mismatch between schema and batches".to_string(),
66+
));
67+
}
6068
}
6169

6270
Ok(Self {

datafusion/core/src/physical_plan/streaming.rs

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ use futures::stream::StreamExt;
2626

2727
use datafusion_common::{DataFusionError, Result, Statistics};
2828
use datafusion_physical_expr::PhysicalSortExpr;
29+
use log::debug;
2930

3031
use crate::datasource::streaming::PartitionStream;
3132
use crate::physical_plan::stream::RecordBatchStreamAdapter;
@@ -48,10 +49,17 @@ impl StreamingTableExec {
4849
projection: Option<&Vec<usize>>,
4950
infinite: bool,
5051
) -> Result<Self> {
51-
if !partitions.iter().all(|x| schema.contains(x.schema())) {
52-
return Err(DataFusionError::Plan(
53-
"Mismatch between schema and batches".to_string(),
54-
));
52+
for x in partitions.iter() {
53+
let partition_schema = x.schema();
54+
if !schema.contains(partition_schema) {
55+
debug!(
56+
"target schema does not contain partition schema. \
57+
Target_schema: {schema:?}. Partiton Schema: {partition_schema:?}"
58+
);
59+
return Err(DataFusionError::Plan(
60+
"Mismatch between schema and batches".to_string(),
61+
));
62+
}
5563
}
5664

5765
let projected_schema = match projection {

0 commit comments

Comments
 (0)