Merge branch 'main' into scope-expect-panic

Author: LucaCappelletti94

Cargo.toml

@@ -18,7 +18,7 @@
 [package]
 name = "sqlparser"
 description = "Extensible SQL Lexer and Parser with support for ANSI SQL:2011"
-version = "0.60.0"
+version = "0.61.0"
 authors = ["Apache DataFusion <dev@datafusion.apache.org>"]
 homepage = "https://github.com/apache/datafusion-sqlparser-rs"
 documentation = "https://docs.rs/sqlparser/"
@@ -55,7 +55,7 @@ serde = { version = "1.0", default-features = false, features = ["derive", "allo
 # of dev-dependencies because of
 # https://github.com/rust-lang/cargo/issues/1596
 serde_json = { version = "1.0", optional = true }
-sqlparser_derive = { version = "0.4.0", path = "derive", optional = true }
+sqlparser_derive = { version = "0.5.0", path = "derive", optional = true }
 
 [dev-dependencies]
 simple_logger = "5.0"

changelog/0.61.0.md

@@ -0,0 +1,127 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# sqlparser-rs 0.61.0 Changelog
+
+This release consists of 66 commits from 22 contributors. See credits at the end of this changelog for more information.
+
+**Performance related:**
+
+- perf: remove unnecessary string clone in maybe_concat_string_literal [#2173](https://github.com/apache/datafusion-sqlparser-rs/pull/2173) (andygrove)
+- perf: optimize `make_word()` to avoid unnecessary allocations [#2176](https://github.com/apache/datafusion-sqlparser-rs/pull/2176) (andygrove)
+
+**Fixed bugs:**
+
+- fix: Set the current folder as a "primary" for the `find` command [#2120](https://github.com/apache/datafusion-sqlparser-rs/pull/2120) (martin-g)
+- fix: qualified column names with SQL keywords parse as identifiers [#2157](https://github.com/apache/datafusion-sqlparser-rs/pull/2157) (bombsimon)
+
+**Other:**
+
+- Fixing location for extrenal tables [#2108](https://github.com/apache/datafusion-sqlparser-rs/pull/2108) (romanoff)
+- Added support for `ALTER OPERATOR` syntax [#2114](https://github.com/apache/datafusion-sqlparser-rs/pull/2114) (LucaCappelletti94)
+- Oracle: Support for MERGE predicates [#2101](https://github.com/apache/datafusion-sqlparser-rs/pull/2101) (xitep)
+- [Oracle] Lower StringConcat precedence [#2115](https://github.com/apache/datafusion-sqlparser-rs/pull/2115) (xitep)
+- Added alter external table support for snowflake [#2122](https://github.com/apache/datafusion-sqlparser-rs/pull/2122) (romanoff)
+- MSSQL: Add support for parenthesized stored procedure name in EXEC [#2126](https://github.com/apache/datafusion-sqlparser-rs/pull/2126) (yoavcloud)
+- MSSQL: Parse IF/ELSE without semicolon delimiters [#2128](https://github.com/apache/datafusion-sqlparser-rs/pull/2128) (yoavcloud)
+- Extract source comments [#2107](https://github.com/apache/datafusion-sqlparser-rs/pull/2107) (xitep)
+- PostgreSQL: Support schema-qualified operator classes in CREATE INDEX [#2131](https://github.com/apache/datafusion-sqlparser-rs/pull/2131) (dddenis)
+- Oracle: Support for quote delimited strings [#2130](https://github.com/apache/datafusion-sqlparser-rs/pull/2130) (xitep)
+- Added support for `ALTER OPERATOR FAMILY` syntax [#2125](https://github.com/apache/datafusion-sqlparser-rs/pull/2125) (LucaCappelletti94)
+- PostgreSQL Tokenization: Fix unexpected characters after question mark being silently ignored [#2129](https://github.com/apache/datafusion-sqlparser-rs/pull/2129) (jnlt3)
+- Support parsing parenthesized wildcard `(*)` [#2123](https://github.com/apache/datafusion-sqlparser-rs/pull/2123) (romanoff)
+- Make benchmark statement valid [#2139](https://github.com/apache/datafusion-sqlparser-rs/pull/2139) (xitep)
+- Fix parse_identifiers not taking semicolons into account [#2137](https://github.com/apache/datafusion-sqlparser-rs/pull/2137) (jnlt3)
+- Add PostgreSQL PARTITION OF syntax support [#2127](https://github.com/apache/datafusion-sqlparser-rs/pull/2127) (fmguerreiro)
+- Databricks: Support Timetravel With "TIMESTAMP AS OF" [#2134](https://github.com/apache/datafusion-sqlparser-rs/pull/2134) (JamesVorder)
+- MySQL: Parse bitwise shift left/right operators [#2152](https://github.com/apache/datafusion-sqlparser-rs/pull/2152) (mvzink)
+- Redshift: Add support for optional JSON format in copy option [#2141](https://github.com/apache/datafusion-sqlparser-rs/pull/2141) (yoavcloud)
+- MySQL: Add missing support for TREE explain format [#2145](https://github.com/apache/datafusion-sqlparser-rs/pull/2145) (yoavcloud)
+- MySQL: Add support for && as boolean AND [#2144](https://github.com/apache/datafusion-sqlparser-rs/pull/2144) (yoavcloud)
+- PostgreSQL: ALTER USER password option [#2142](https://github.com/apache/datafusion-sqlparser-rs/pull/2142) (yoavcloud)
+- Key Value Options: add support for trailing semicolon [#2140](https://github.com/apache/datafusion-sqlparser-rs/pull/2140) (yoavcloud)
+- Added support for `ALTER OPERATOR CLASS` syntax [#2135](https://github.com/apache/datafusion-sqlparser-rs/pull/2135) (LucaCappelletti94)
+- Added missing `Copy` derives [#2158](https://github.com/apache/datafusion-sqlparser-rs/pull/2158) (LucaCappelletti94)
+- Tokenize empty line comments correctly [#2161](https://github.com/apache/datafusion-sqlparser-rs/pull/2161) (zyuiop)
+- Add support for DuckDB `LAMBDA` keyword syntax [#2149](https://github.com/apache/datafusion-sqlparser-rs/pull/2149) (lovasoa)
+- MySQL: Add support for casting using the BINARY keyword [#2146](https://github.com/apache/datafusion-sqlparser-rs/pull/2146) (yoavcloud)
+- Added missing `From` impls for `Statement` variants [#2160](https://github.com/apache/datafusion-sqlparser-rs/pull/2160) (LucaCappelletti94)
+- GenericDialect: support colon operator for JsonAccess [#2124](https://github.com/apache/datafusion-sqlparser-rs/pull/2124) (Samyak2)
+- Databricks: Support Timetravel With "VERSION AS OF" [#2155](https://github.com/apache/datafusion-sqlparser-rs/pull/2155) (JamesVorder)
+- Fixed truncate table if exists for snowflake [#2166](https://github.com/apache/datafusion-sqlparser-rs/pull/2166) (romanoff)
+- Refactor: replace some `dialect_of!` checks with `Dialect` trait methods [#2171](https://github.com/apache/datafusion-sqlparser-rs/pull/2171) (andygrove)
+- MySQL: Support `CAST(... AS ... ARRAY)` syntax [#2151](https://github.com/apache/datafusion-sqlparser-rs/pull/2151) (mvzink)
+- Snowflake: Support SAMPLE clause on subqueries [#2164](https://github.com/apache/datafusion-sqlparser-rs/pull/2164) (finchxxia)
+- refactor: use `to_ident()` instead of `clone().into_ident()` for borrowed Words [#2177](https://github.com/apache/datafusion-sqlparser-rs/pull/2177) (andygrove)
+- Refactor: replace more `dialect_of!` checks with `Dialect` trait methods [#2175](https://github.com/apache/datafusion-sqlparser-rs/pull/2175) (andygrove)
+- minor: reduce unnecessary string allocations [#2178](https://github.com/apache/datafusion-sqlparser-rs/pull/2178) (andygrove)
+- PostgreSQL: Support force row level security [#2169](https://github.com/apache/datafusion-sqlparser-rs/pull/2169) (isaacparker0)
+- PostgreSQL: Add support for `*` (descendant) option in TRUNCATE [#2181](https://github.com/apache/datafusion-sqlparser-rs/pull/2181) (mvzink)
+- Fix identifier parsing not breaking on the `|>` pipe operator [#2156](https://github.com/apache/datafusion-sqlparser-rs/pull/2156) (alexander-beedie)
+- [MySQL, Oracle] Parse optimizer hints [#2162](https://github.com/apache/datafusion-sqlparser-rs/pull/2162) (xitep)
+- Redshift: Support implicit string concatenation using newline [#2167](https://github.com/apache/datafusion-sqlparser-rs/pull/2167) (yoavcloud)
+- PostgreSQL: Fix REPLICA IDENTITY to use NOTHING [#2179](https://github.com/apache/datafusion-sqlparser-rs/pull/2179) (mvzink)
+- Add ENFORCED/NOT ENFORCED support for column-level CHECK constraints [#2180](https://github.com/apache/datafusion-sqlparser-rs/pull/2180) (mvzink)
+- Implement `core::error::Error` for `ParserError` and `TokenizerError` [#2189](https://github.com/apache/datafusion-sqlparser-rs/pull/2189) (LucaCappelletti94)
+- Moved more structs outside of Statement to facilitate reuse [#2188](https://github.com/apache/datafusion-sqlparser-rs/pull/2188) (LucaCappelletti94)
+- Fix parsing cast operator after parenthesized `DEFAULT` expression [#2168](https://github.com/apache/datafusion-sqlparser-rs/pull/2168) (isaacparker0)
+- Streamlined derivation of new `Dialect` objects [#2174](https://github.com/apache/datafusion-sqlparser-rs/pull/2174) (alexander-beedie)
+- MSSQL: Support standalone BEGIN...END blocks [#2186](https://github.com/apache/datafusion-sqlparser-rs/pull/2186) (guan404ming)
+- MySQL: Add support for `SELECT` modifiers [#2172](https://github.com/apache/datafusion-sqlparser-rs/pull/2172) (mvzink)
+- MySQL: Add support for DEFAULT CHARACTER SET in CREATE DATABASE [#2182](https://github.com/apache/datafusion-sqlparser-rs/pull/2182) (mvzink)
+- [Oracle] Support hierarchical queries [#2185](https://github.com/apache/datafusion-sqlparser-rs/pull/2185) (xitep)
+- MySQL: Allow optional constraint name after CONSTRAINT keyword [#2183](https://github.com/apache/datafusion-sqlparser-rs/pull/2183) (mvzink)
+- Added missing derives to dialect marker structs [#2191](https://github.com/apache/datafusion-sqlparser-rs/pull/2191) (LucaCappelletti94)
+- Fixed overflow error, recursion counter was not included for parenthesis [#2199](https://github.com/apache/datafusion-sqlparser-rs/pull/2199) (LucaCappelletti94)
+- Add support for C-style comments [#2034](https://github.com/apache/datafusion-sqlparser-rs/pull/2034) (altmannmarcelo)
+- PostgreSQL: Support PostgreSQL ANALYZE with optional table and column [#2187](https://github.com/apache/datafusion-sqlparser-rs/pull/2187) (guan404ming)
+- Add Tokenizer custom token mapper support [#2184](https://github.com/apache/datafusion-sqlparser-rs/pull/2184) (askalt)
+- Fix MAP literals parsing [#2205](https://github.com/apache/datafusion-sqlparser-rs/pull/2205) (Samyak2)
+
+## Credits
+
+Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor.
+
+```
+     9	Luca Cappelletti
+     9	Yoav Cohen
+     8	Michael Victor Zink
+     7	xitep
+     6	Andy Grove
+     4	Andriy Romanov
+     2	Alexander Beedie
+     2	Andrew Lamb
+     2	Guan-Ming (Wesley) Chiu
+     2	James Vorderbruggen
+     2	Samyak Sarnayak
+     2	isaacparker0
+     2	jnlt3
+     1	Albert Skalt
+     1	Denis Goncharenko
+     1	Filipe Guerreiro
+     1	Louis Vialar
+     1	Marcelo Altmann
+     1	Martin Grigorov
+     1	Ophir LOJKINE
+     1	Simon Sawert
+     1	finchxxia
+```
+
+Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release.
+

derive/Cargo.toml

@@ -18,7 +18,7 @@
 [package]
 name = "sqlparser_derive"
 description = "Procedural (proc) macros for sqlparser"
-version = "0.4.0"
+version = "0.5.0"
 authors = ["sqlparser-rs authors"]
 homepage = "https://github.com/sqlparser-rs/sqlparser-rs"
 documentation = "https://docs.rs/sqlparser_derive/"

dev/release/release-tarball.sh

@@ -43,6 +43,13 @@ fi
 version=$1
 rc=$2
 
+read -r -p "Proceed to release tarball for ${version}-rc${rc}? [y/N]: " answer
+answer=${answer:-no}
+if [ "${answer}" != "y" ]; then
+  echo "Cancelled tarball release!"
+  exit 1
+fi
+
 tmp_dir=tmp-apache-datafusion-dist
 
 echo "Recreate temporary directory: ${tmp_dir}"

src/ast/mod.rs

@@ -3326,19 +3326,24 @@ impl Display for ExceptionWhen {
     }
 }
 
-/// ANALYZE TABLE statement (Hive-specific)
+/// ANALYZE statement
+///
+/// Supported syntax varies by dialect:
+/// - Hive: `ANALYZE TABLE t [PARTITION (...)] COMPUTE STATISTICS [NOSCAN] [FOR COLUMNS [col1, ...]] [CACHE METADATA]`
+/// - PostgreSQL: `ANALYZE [VERBOSE] [t [(col1, ...)]]` See <https://www.postgresql.org/docs/current/sql-analyze.html>
+/// - General: `ANALYZE [TABLE] t`
 #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
 pub struct Analyze {
     #[cfg_attr(feature = "visitor", visit(with = "visit_relation"))]
-    /// Name of the table to analyze.
-    pub table_name: ObjectName,
+    /// Name of the table to analyze. `None` for bare `ANALYZE`.
+    pub table_name: Option<ObjectName>,
     /// Optional partition expressions to restrict the analysis.
     pub partitions: Option<Vec<Expr>>,
-    /// `true` when analyzing specific columns.
+    /// `true` when analyzing specific columns (Hive `FOR COLUMNS` syntax).
     pub for_columns: bool,
-    /// Columns to analyze when `for_columns` is `true`.
+    /// Columns to analyze.
     pub columns: Vec<Ident>,
     /// Whether to cache metadata before analyzing.
     pub cache_metadata: bool,
@@ -3352,22 +3357,21 @@ pub struct Analyze {
 
 impl fmt::Display for Analyze {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            f,
-            "ANALYZE{}{table_name}",
+        write!(f, "ANALYZE")?;
+        if let Some(ref table_name) = self.table_name {
             if self.has_table_keyword {
-                " TABLE "
-            } else {
-                " "
-            },
-            table_name = self.table_name
-        )?;
+                write!(f, " TABLE")?;
+            }
+            write!(f, " {table_name}")?;
+        }
+        if !self.for_columns && !self.columns.is_empty() {
+            write!(f, " ({})", display_comma_separated(&self.columns))?;
+        }
         if let Some(ref parts) = self.partitions {
             if !parts.is_empty() {
                 write!(f, " PARTITION ({})", display_comma_separated(parts))?;
             }
         }
-
         if self.compute_statistics {
             write!(f, " COMPUTE STATISTICS")?;
         }
@@ -4574,22 +4578,40 @@ pub enum Statement {
         /// Legacy copy-style options.
         options: Vec<CopyLegacyOption>,
     },
+    /// ClickHouse:
     /// ```sql
     /// OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION ID 'partition_id'] [FINAL] [DEDUPLICATE [BY expression]]
     /// ```
-    ///
     /// See ClickHouse <https://clickhouse.com/docs/en/sql-reference/statements/optimize>
+    ///
+    /// Databricks:
+    /// ```sql
+    /// OPTIMIZE table_name [WHERE predicate] [ZORDER BY (col_name1 [, ...])]
+    /// ```
+    /// See Databricks <https://docs.databricks.com/en/sql/language-manual/delta-optimize.html>
     OptimizeTable {
         /// Table name to optimize.
         name: ObjectName,
+        /// Whether the `TABLE` keyword was present (ClickHouse uses `OPTIMIZE TABLE`, Databricks uses `OPTIMIZE`).
+        has_table_keyword: bool,
         /// Optional cluster identifier.
+        /// [ClickHouse](https://clickhouse.com/docs/en/sql-reference/statements/optimize)
         on_cluster: Option<Ident>,
         /// Optional partition spec.
+        /// [ClickHouse](https://clickhouse.com/docs/en/sql-reference/statements/optimize)
         partition: Option<Partition>,
         /// Whether `FINAL` was specified.
+        /// [ClickHouse](https://clickhouse.com/docs/en/sql-reference/statements/optimize)
         include_final: bool,
         /// Optional deduplication settings.
+        /// [ClickHouse](https://clickhouse.com/docs/en/sql-reference/statements/optimize)
         deduplicate: Option<Deduplicate>,
+        /// Optional WHERE predicate.
+        /// [Databricks](https://docs.databricks.com/en/sql/language-manual/delta-optimize.html)
+        predicate: Option<Expr>,
+        /// Optional ZORDER BY columns.
+        /// [Databricks](https://docs.databricks.com/en/sql/language-manual/delta-optimize.html)
+        zorder: Option<Vec<Expr>>,
     },
     /// ```sql
     /// LISTEN
@@ -6065,12 +6087,19 @@ impl fmt::Display for Statement {
             }
             Statement::OptimizeTable {
                 name,
+                has_table_keyword,
                 on_cluster,
                 partition,
                 include_final,
                 deduplicate,
+                predicate,
+                zorder,
             } => {
-                write!(f, "OPTIMIZE TABLE {name}")?;
+                write!(f, "OPTIMIZE")?;
+                if *has_table_keyword {
+                    write!(f, " TABLE")?;
+                }
+                write!(f, " {name}")?;
                 if let Some(on_cluster) = on_cluster {
                     write!(f, " ON CLUSTER {on_cluster}")?;
                 }
@@ -6083,6 +6112,12 @@ impl fmt::Display for Statement {
                 if let Some(deduplicate) = deduplicate {
                     write!(f, " {deduplicate}")?;
                 }
+                if let Some(predicate) = predicate {
+                    write!(f, " WHERE {predicate}")?;
+                }
+                if let Some(zorder) = zorder {
+                    write!(f, " ZORDER BY ({})", display_comma_separated(zorder))?;
+                }
                 Ok(())
             }
             Statement::LISTEN { channel } => {

src/ast/spans.rs

@@ -841,7 +841,9 @@ impl Spanned for ConstraintCharacteristics {
 impl Spanned for Analyze {
     fn span(&self) -> Span {
         union_spans(
-            core::iter::once(self.table_name.span())
+            self.table_name
+                .iter()
+                .map(|t| t.span())
                 .chain(
                     self.partitions
                         .iter()

src/dialect/ansi.rs

@@ -18,7 +18,8 @@
 use crate::dialect::Dialect;
 
 /// A [`Dialect`] for [ANSI SQL](https://en.wikipedia.org/wiki/SQL:2011).
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 pub struct AnsiDialect {}
 
 impl Dialect for AnsiDialect {

src/dialect/bigquery.rs

@@ -42,7 +42,8 @@ const RESERVED_FOR_COLUMN_ALIAS: &[Keyword] = &[
 ];
 
 /// A [`Dialect`] for [Google Bigquery](https://cloud.google.com/bigquery/)
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 pub struct BigQueryDialect;
 
 impl Dialect for BigQueryDialect {

src/dialect/clickhouse.rs

@@ -18,7 +18,8 @@
 use crate::dialect::Dialect;
 
 /// A [`Dialect`] for [ClickHouse](https://clickhouse.com/).
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 pub struct ClickHouseDialect {}
 
 impl Dialect for ClickHouseDialect {

src/dialect/databricks.rs

@@ -20,7 +20,8 @@ use crate::dialect::Dialect;
 /// A [`Dialect`] for [Databricks SQL](https://www.databricks.com/)
 ///
 /// See <https://docs.databricks.com/en/sql/language-manual/index.html>.
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 pub struct DatabricksDialect;
 
 impl Dialect for DatabricksDialect {
@@ -84,4 +85,9 @@ impl Dialect for DatabricksDialect {
     fn supports_values_as_table_factor(&self) -> bool {
         true
     }
+
+    /// See <https://docs.databricks.com/en/sql/language-manual/delta-optimize.html>
+    fn supports_optimize_table(&self) -> bool {
+        true
+    }
 }