Skip to content

Commit 66c3cd5

Browse files
blagininblaginin
committed
Deprecate and ignore enable_options_value_normalization 
1 parent 947d1c2 commit 66c3cd5

File tree

8 files changed

+18
-106
lines changed

8 files changed

+18
-106
lines changed

‎datafusion-cli/Cargo.lock

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/common/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ half = { workspace = true }
5757
hashbrown = { workspace = true }
5858
indexmap = { workspace = true }
5959
libc = "0.2.140"
60+
log = { workspace = true }
6061
object_store = { workspace = true, optional = true }
6162
parquet = { workspace = true, optional = true, default-features = true }
6263
paste = "1.0.15"

‎datafusion/common/src/config.rs

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,9 @@ use crate::{DataFusionError, Result};
3030

3131
/// A macro that wraps a configuration struct and automatically derives
3232
/// [`Default`] and [`ConfigField`] for it, allowing it to be used
33-
/// in the [`ConfigOptions`] configuration tree
33+
/// in the [`ConfigOptions`] configuration tree.
34+
///
35+
/// transform is be used to normalize the value before parsing.
3436
///
3537
/// For example,
3638
///
@@ -113,7 +115,7 @@ macro_rules! config_namespace {
113115
$vis:vis struct $struct_name:ident {
114116
$(
115117
$(#[doc = $d:tt])*
116-
$field_vis:vis $field_name:ident : $field_type:ty, $(transform = $transform:expr,)? default = $default:expr
118+
$field_vis:vis $field_name:ident : $field_type:ty, $(warn = $warn: expr,)? $(transform = $transform:expr,)? default = $default:expr
117119
)*$(,)*
118120
}
119121
) => {
@@ -135,6 +137,7 @@ macro_rules! config_namespace {
135137
$(
136138
stringify!($field_name) => {
137139
$(let value = $transform(value);)?
140+
$(log::warn!($warn);)?
138141
self.$field_name.set(rem, value.as_ref())
139142
},
140143
)*
@@ -218,8 +221,10 @@ config_namespace! {
218221
/// When set to true, SQL parser will normalize ident (convert ident to lowercase when not quoted)
219222
pub enable_ident_normalization: bool, default = true
220223

221-
/// When set to true, SQL parser will normalize options value (convert value to lowercase)
222-
pub enable_options_value_normalization: bool, default = false
224+
/// When set to true, SQL parser will normalize options value (convert value to lowercase).
225+
/// Note that this option is ignored and will be removed in the future. All case-insensitive values
226+
/// are normalized automatically.
227+
pub enable_options_value_normalization: bool, warn = "`enable_options_value_normalization` is deprecated and ignored", default = false
223228

224229
/// Configure the SQL dialect used by DataFusion's parser; supported values include: Generic,
225230
/// MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, and Ansi.

‎datafusion/sql/src/planner.rs

Lines changed: 2 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,10 @@ use arrow_schema::*;
2424
use datafusion_common::{
2525
field_not_found, internal_err, plan_datafusion_err, DFSchemaRef, SchemaError,
2626
};
27+
use sqlparser::ast::TimezoneInfo;
2728
use sqlparser::ast::{ArrayElemTypeDef, ExactNumberInfo};
2829
use sqlparser::ast::{ColumnDef as SQLColumnDef, ColumnOption};
2930
use sqlparser::ast::{DataType as SQLDataType, Ident, ObjectName, TableAlias};
30-
use sqlparser::ast::{TimezoneInfo, Value};
3131

3232
use datafusion_common::TableReference;
3333
use datafusion_common::{
@@ -38,7 +38,7 @@ use datafusion_expr::logical_plan::{LogicalPlan, LogicalPlanBuilder};
3838
use datafusion_expr::utils::find_column_exprs;
3939
use datafusion_expr::{col, Expr};
4040

41-
use crate::utils::{make_decimal_type, value_to_string};
41+
use crate::utils::make_decimal_type;
4242
pub use datafusion_expr::planner::ContextProvider;
4343

4444
/// SQL parser options
@@ -87,32 +87,6 @@ impl IdentNormalizer {
8787
}
8888
}
8989

90-
/// Value Normalizer
91-
#[derive(Debug)]
92-
pub struct ValueNormalizer {
93-
normalize: bool,
94-
}
95-
96-
impl Default for ValueNormalizer {
97-
fn default() -> Self {
98-
Self { normalize: true }
99-
}
100-
}
101-
102-
impl ValueNormalizer {
103-
pub fn new(normalize: bool) -> Self {
104-
Self { normalize }
105-
}
106-
107-
pub fn normalize(&self, value: Value) -> Option<String> {
108-
match (value_to_string(&value), self.normalize) {
109-
(Some(s), true) => Some(s.to_ascii_lowercase()),
110-
(Some(s), false) => Some(s),
111-
(None, _) => None,
112-
}
113-
}
114-
}
115-
11690
/// Struct to store the states used by the Planner. The Planner will leverage the states to resolve
11791
/// CTEs, Views, subqueries and PREPARE statements. The states include
11892
/// Common Table Expression (CTE) provided with WITH clause and
@@ -254,7 +228,6 @@ pub struct SqlToRel<'a, S: ContextProvider> {
254228
pub(crate) context_provider: &'a S,
255229
pub(crate) options: ParserOptions,
256230
pub(crate) ident_normalizer: IdentNormalizer,
257-
pub(crate) value_normalizer: ValueNormalizer,
258231
}
259232

260233
impl<'a, S: ContextProvider> SqlToRel<'a, S> {
@@ -266,13 +239,11 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
266239
/// Create a new query planner
267240
pub fn new_with_options(context_provider: &'a S, options: ParserOptions) -> Self {
268241
let ident_normalize = options.enable_ident_normalization;
269-
let options_value_normalize = options.enable_options_value_normalization;
270242

271243
SqlToRel {
272244
context_provider,
273245
options,
274246
ident_normalizer: IdentNormalizer::new(ident_normalize),
275-
value_normalizer: ValueNormalizer::new(options_value_normalize),
276247
}
277248
}
278249

‎datafusion/sql/src/statement.rs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1386,8 +1386,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
13861386
return plan_err!("Option {key} is specified multiple times");
13871387
}
13881388

1389-
let Some(value_string) = self.value_normalizer.normalize(value.clone())
1390-
else {
1389+
let Some(value_string) = crate::utils::value_to_string(&value) else {
13911390
return plan_err!("Unsupported Value {}", value);
13921391
};
13931392

‎datafusion/sql/tests/sql_integration.rs

Lines changed: 2 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,10 @@ use datafusion_common::{
2929
};
3030
use datafusion_expr::{
3131
col,
32-
dml::CopyTo,
3332
logical_plan::{LogicalPlan, Prepare},
3433
test::function_stub::sum_udaf,
35-
ColumnarValue, CreateExternalTable, CreateIndex, DdlStatement, ScalarUDF,
36-
ScalarUDFImpl, Signature, Statement, Volatility,
34+
ColumnarValue, CreateIndex, DdlStatement, ScalarUDF, ScalarUDFImpl, Signature,
35+
Statement, Volatility,
3736
};
3837
use datafusion_functions::{string, unicode};
3938
use datafusion_sql::{
@@ -161,70 +160,6 @@ fn parse_ident_normalization() {
161160
}
162161
}
163162

164-
#[test]
165-
fn test_parse_options_value_normalization() {
166-
let test_data = [
167-
(
168-
"CREATE EXTERNAL TABLE test OPTIONS ('location' 'LoCaTiOn') STORED AS PARQUET LOCATION 'fake_location'",
169-
"CreateExternalTable: Bare { table: \"test\" }",
170-
HashMap::from([("format.location", "LoCaTiOn")]),
171-
false,
172-
),
173-
(
174-
"CREATE EXTERNAL TABLE test OPTIONS ('location' 'LoCaTiOn') STORED AS PARQUET LOCATION 'fake_location'",
175-
"CreateExternalTable: Bare { table: \"test\" }",
176-
HashMap::from([("format.location", "location")]),
177-
true,
178-
),
179-
(
180-
"COPY test TO 'fake_location' STORED AS PARQUET OPTIONS ('location' 'LoCaTiOn')",
181-
"CopyTo: format=csv output_url=fake_location options: (format.location LoCaTiOn)\n TableScan: test",
182-
HashMap::from([("format.location", "LoCaTiOn")]),
183-
false,
184-
),
185-
(
186-
"COPY test TO 'fake_location' STORED AS PARQUET OPTIONS ('location' 'LoCaTiOn')",
187-
"CopyTo: format=csv output_url=fake_location options: (format.location location)\n TableScan: test",
188-
HashMap::from([("format.location", "location")]),
189-
true,
190-
),
191-
];
192-
193-
for (sql, expected_plan, expected_options, enable_options_value_normalization) in
194-
test_data
195-
{
196-
let plan = logical_plan_with_options(
197-
sql,
198-
ParserOptions {
199-
parse_float_as_decimal: false,
200-
enable_ident_normalization: false,
201-
support_varchar_with_length: false,
202-
enable_options_value_normalization,
203-
},
204-
);
205-
if let Ok(plan) = plan {
206-
assert_eq!(expected_plan, format!("{plan}"));
207-
208-
match plan {
209-
LogicalPlan::Ddl(DdlStatement::CreateExternalTable(
210-
CreateExternalTable { options, .. },
211-
))
212-
| LogicalPlan::Copy(CopyTo { options, .. }) => {
213-
expected_options.iter().for_each(|(k, v)| {
214-
assert_eq!(Some(&v.to_string()), options.get(*k));
215-
});
216-
}
217-
_ => panic!(
218-
"Expected Ddl(CreateExternalTable) or Copy(CopyTo) but got {:?}",
219-
plan
220-
),
221-
}
222-
} else {
223-
assert_eq!(expected_plan, plan.unwrap_err().strip_backtrace());
224-
}
225-
}
226-
}
227-
228163
#[test]
229164
fn select_no_relation() {
230165
quick_test(

‎datafusion/sqllogictest/test_files/information_schema.slt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -351,7 +351,7 @@ datafusion.optimizer.skip_failed_rules false When set to true, the logical plan
351351
datafusion.optimizer.top_down_join_key_reordering true When set to true, the physical plan optimizer will run a top down process to reorder the join keys
352352
datafusion.sql_parser.dialect generic Configure the SQL dialect used by DataFusion's parser; supported values include: Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, and Ansi.
353353
datafusion.sql_parser.enable_ident_normalization true When set to true, SQL parser will normalize ident (convert ident to lowercase when not quoted)
354-
datafusion.sql_parser.enable_options_value_normalization false When set to true, SQL parser will normalize options value (convert value to lowercase)
354+
datafusion.sql_parser.enable_options_value_normalization false When set to true, SQL parser will normalize options value (convert value to lowercase). Note that this option is ignored and will be removed in the future. All case-insensitive values are normalized automatically.
355355
datafusion.sql_parser.parse_float_as_decimal false When set to true, SQL parser will parse float as decimal type
356356
datafusion.sql_parser.support_varchar_with_length true If true, permit lengths for `VARCHAR` such as `VARCHAR(20)`, but ignore the length. If false, error if a `VARCHAR` with a length is specified. The Arrow type system does not have a notion of maximum string length and thus DataFusion can not enforce such limits.
357357

‎docs/source/user-guide/configs.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,6 @@ Environment variables are read during `SessionConfig` initialisation so they mus
122122
| datafusion.explain.show_schema | false | When set to true, the explain statement will print schema information |
123123
| datafusion.sql_parser.parse_float_as_decimal | false | When set to true, SQL parser will parse float as decimal type |
124124
| datafusion.sql_parser.enable_ident_normalization | true | When set to true, SQL parser will normalize ident (convert ident to lowercase when not quoted) |
125-
| datafusion.sql_parser.enable_options_value_normalization | false | When set to true, SQL parser will normalize options value (convert value to lowercase) |
125+
| datafusion.sql_parser.enable_options_value_normalization | false | When set to true, SQL parser will normalize options value (convert value to lowercase). Note that this option is ignored and will be removed in the future. All case-insensitive values are normalized automatically. |
126126
| datafusion.sql_parser.dialect | generic | Configure the SQL dialect used by DataFusion's parser; supported values include: Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, and Ansi. |
127127
| datafusion.sql_parser.support_varchar_with_length | true | If true, permit lengths for `VARCHAR` such as `VARCHAR(20)`, but ignore the length. If false, error if a `VARCHAR` with a length is specified. The Arrow type system does not have a notion of maximum string length and thus DataFusion can not enforce such limits. |

0 commit comments

Comments
 (0)