Recursive CTEs: Stage 3 - add execution support

.github/actions/setup-rust-runtime/action.yaml

@@ -37,5 +37,5 @@ runs:
         echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV 
         echo "RUST_BACKTRACE=1" >> $GITHUB_ENV
         echo "RUST_MIN_STACK=3000000" >> $GITHUB_ENV
-        echo "RUST_FLAGS=-C debuginfo=line-tables-only -C incremental=false" >> $GITHUB_ENV                                   
+        echo "RUSTFLAGS=-C debuginfo=line-tables-only -C incremental=false" >> $GITHUB_ENV
 

.github/workflows/rust.yml

@@ -488,7 +488,7 @@ jobs:
 
   # Verify MSRV for the crates which are directly used by other projects.
   msrv:
-    name: Verify MSRV
+    name: Verify MSRV (Min Supported Rust Version)
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
@@ -500,7 +500,13 @@ jobs:
         run: cargo install cargo-msrv
       - name: Check datafusion
         working-directory: datafusion/core
-        run: cargo msrv verify
+        run: |
+          # If you encounter an error with any of the commands below
+          # it means some crate in your dependency tree has a higher 
+          # MSRV (Min Supported Rust Version) than the one specified 
+          # in the `rust-version` key of `Cargo.toml`. Check your 
+          # dependencies or update the version in `Cargo.toml`
+          cargo msrv verify
       - name: Check datafusion-substrait
         working-directory: datafusion/substrait
         run: cargo msrv verify

Cargo.toml

@@ -29,7 +29,7 @@ license = "Apache-2.0"
 readme = "README.md"
 repository = "https://github.com/apache/arrow-datafusion"
 rust-version = "1.70"
-version = "34.0.0"
+version = "35.0.0"
 
 [workspace.dependencies]
 arrow = { version = "50.0.0", features = ["prettyprint"] }
@@ -45,17 +45,17 @@ bytes = "1.4"
 chrono = { version = "0.4.31", default-features = false }
 ctor = "0.2.0"
 dashmap = "5.4.0"
-datafusion = { path = "datafusion/core", version = "34.0.0" }
-datafusion-common = { path = "datafusion/common", version = "34.0.0" }
-datafusion-execution = { path = "datafusion/execution", version = "34.0.0" }
-datafusion-expr = { path = "datafusion/expr", version = "34.0.0" }
-datafusion-optimizer = { path = "datafusion/optimizer", version = "34.0.0" }
-datafusion-physical-expr = { path = "datafusion/physical-expr", version = "34.0.0" }
-datafusion-physical-plan = { path = "datafusion/physical-plan", version = "34.0.0" }
-datafusion-proto = { path = "datafusion/proto", version = "34.0.0" }
-datafusion-sql = { path = "datafusion/sql", version = "34.0.0" }
-datafusion-sqllogictest = { path = "datafusion/sqllogictest", version = "34.0.0" }
-datafusion-substrait = { path = "datafusion/substrait", version = "34.0.0" }
+datafusion = { path = "datafusion/core", version = "35.0.0" }
+datafusion-common = { path = "datafusion/common", version = "35.0.0" }
+datafusion-execution = { path = "datafusion/execution", version = "35.0.0" }
+datafusion-expr = { path = "datafusion/expr", version = "35.0.0" }
+datafusion-optimizer = { path = "datafusion/optimizer", version = "35.0.0" }
+datafusion-physical-expr = { path = "datafusion/physical-expr", version = "35.0.0" }
+datafusion-physical-plan = { path = "datafusion/physical-plan", version = "35.0.0" }
+datafusion-proto = { path = "datafusion/proto", version = "35.0.0" }
+datafusion-sql = { path = "datafusion/sql", version = "35.0.0" }
+datafusion-sqllogictest = { path = "datafusion/sqllogictest", version = "35.0.0" }
+datafusion-substrait = { path = "datafusion/substrait", version = "35.0.0" }
 doc-comment = "0.3"
 env_logger = "0.10"
 futures = "0.3"
@@ -70,7 +70,7 @@ parquet = { version = "50.0.0", default-features = false, features = ["arrow", "
 rand = "0.8"
 rstest = "0.18.0"
 serde_json = "1"
-sqlparser = { version = "0.41.0", features = ["visitor"] }
+sqlparser = { version = "0.43.0", features = ["visitor"] }
 tempfile = "3"
 thiserror = "1.0.44"
 url = "2.2"

README.md

@@ -19,6 +19,25 @@
 
 # DataFusion
 
+[![Crates.io][crates-badge]][crates-url]
+[![Apache licensed][license-badge]][license-url]
+[![Build Status][actions-badge]][actions-url]
+[![Discord chat][discord-badge]][discord-url]
+
+[crates-badge]: https://img.shields.io/crates/v/datafusion.svg
+[crates-url]: https://crates.io/crates/datafusion
+[license-badge]: https://img.shields.io/badge/license-Apache%20v2-blue.svg
+[license-url]: https://github.com/apache/arrow-datafusion/blob/main/LICENSE.txt
+[actions-badge]: https://github.com/apache/arrow-datafusion/actions/workflows/rust.yml/badge.svg
+[actions-url]: https://github.com/apache/arrow-datafusion/actions?query=branch%3Amain
+[discord-badge]: https://img.shields.io/discord/885562378132000778.svg?logo=discord&style=flat-square
+[discord-url]: https://discord.com/invite/Qw5gKqHxUM
+
+[Website](https://github.com/apache/arrow-datafusion) |
+[Guides](https://github.com/apache/arrow-datafusion/tree/main/docs) |
+[API Docs](https://docs.rs/datafusion/latest/datafusion/) |
+[Chat](https://discord.com/channels/885562378132000778/885562378132000781)
+
 <img src="https://arrow.apache.org/datafusion/_images/DataFusion-Logo-Background-White.png" width="256" alt="logo"/>
 
 DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in

benchmarks/Cargo.toml

@@ -18,7 +18,7 @@
 [package]
 name = "datafusion-benchmarks"
 description = "DataFusion Benchmarks"
-version = "34.0.0"
+version = "35.0.0"
 edition = { workspace = true }
 authors = ["Apache Arrow <[email protected]>"]
 homepage = "https://github.com/apache/arrow-datafusion"
@@ -33,8 +33,8 @@ snmalloc = ["snmalloc-rs"]
 
 [dependencies]
 arrow = { workspace = true }
-datafusion = { path = "../datafusion/core", version = "34.0.0" }
-datafusion-common = { path = "../datafusion/common", version = "34.0.0" }
+datafusion = { path = "../datafusion/core", version = "35.0.0" }
+datafusion-common = { path = "../datafusion/common", version = "35.0.0" }
 env_logger = { workspace = true }
 futures = { workspace = true }
 log = { workspace = true }
@@ -49,4 +49,4 @@ test-utils = { path = "../test-utils/", version = "0.1.0" }
 tokio = { version = "^1.0", features = ["macros", "rt", "rt-multi-thread", "parking_lot"] }
 
 [dev-dependencies]
-datafusion-proto = { path = "../datafusion/proto", version = "34.0.0" }
+datafusion-proto = { path = "../datafusion/proto", version = "35.0.0" }

benchmarks/queries/clickbench/README.md

@@ -11,23 +11,180 @@ ClickBench is focused on aggregation and filtering performance (though it has no
 [ClickBench repository]: https://github.com/ClickHouse/ClickBench/blob/main/datafusion/queries.sql
 
 ## "Extended" Queries 
-The "extended" queries are not part of the official ClickBench benchmark. 
-Instead they are used to test other DataFusion features that are not 
-covered by the standard benchmark
 
-Each description below is for the corresponding line in `extended.sql` (line 1
-is `Q0`, line 2 is `Q1`, etc.)  
+The "extended" queries are not part of the official ClickBench benchmark.
+Instead they are used to test other DataFusion features that are not covered by
+the standard benchmark  Each description below is for the corresponding line in
+`extended.sql` (line 1 is `Q0`, line 2 is `Q1`, etc.)
+
+### Q0: Data Exploration
+
+**Question**: "How many distinct searches, mobile phones, and mobile phone models are there in the dataset?"
+
+**Important Query Properties**: multiple `COUNT DISTINCT`s, with low and high cardinality
+distinct string columns.
+
+```sql
+SELECT COUNT(DISTINCT "SearchPhrase"), COUNT(DISTINCT "MobilePhone"), COUNT(DISTINCT "MobilePhoneModel") 
+FROM hits;
+```
+
+
+### Q1: Data Exploration
+
+**Question**: "How many distinct "hit color", "browser country" and "language" are there in the dataset?"
+
+**Important Query Properties**: multiple `COUNT DISTINCT`s. All three are small strings (length either 1 or 2).
 
-### Q0
-Models initial Data exploration, to understand some statistics of data. 
-Import Query Properties: multiple `COUNT DISTINCT` on strings
 
 ```sql
-SELECT 
-    COUNT(DISTINCT "SearchPhrase"), COUNT(DISTINCT "MobilePhone"), COUNT(DISTINCT "MobilePhoneModel") 
+SELECT COUNT(DISTINCT "HitColor"), COUNT(DISTINCT "BrowserCountry"), COUNT(DISTINCT "BrowserLanguage")
 FROM hits;
 ```
 
+### Q2: Top 10 anaylsis
 
+**Question**: "Find the top 10 "browser country" by number of distinct "social network"s, 
+including the distinct counts of  "hit color", "browser language",
+and "social action"."
 
+**Important Query Properties**: GROUP BY short, string, multiple `COUNT DISTINCT`s. There are several small strings (length either 1 or 2).
 
+```sql
+SELECT "BrowserCountry",  COUNT(DISTINCT "SocialNetwork"), COUNT(DISTINCT "HitColor"), COUNT(DISTINCT "BrowserLanguage"), COUNT(DISTINCT "SocialAction")
+FROM hits 
+GROUP BY 1 
+ORDER BY 2 DESC 
+LIMIT 10;
+```
+
+
+## Data Notes
+
+Here are some interesting statistics about the data used in the queries
+Max length of `"SearchPhrase"` is 1113 characters
+```sql
+❯ select min(length("SearchPhrase")) as "SearchPhrase_len_min", max(length("SearchPhrase")) "SearchPhrase_len_max" from 'hits.parquet' limit 10;
++----------------------+----------------------+
+| SearchPhrase_len_min | SearchPhrase_len_max |
++----------------------+----------------------+
+| 0                    | 1113                 |
++----------------------+----------------------+
+```
+
+
+Here is the schema of the data
+```sql
+❯ describe 'hits.parquet';
++-----------------------+-----------+-------------+
+| column_name           | data_type | is_nullable |
++-----------------------+-----------+-------------+
+| WatchID               | Int64     | NO          |
+| JavaEnable            | Int16     | NO          |
+| Title                 | Utf8      | NO          |
+| GoodEvent             | Int16     | NO          |
+| EventTime             | Int64     | NO          |
+| EventDate             | UInt16    | NO          |
+| CounterID             | Int32     | NO          |
+| ClientIP              | Int32     | NO          |
+| RegionID              | Int32     | NO          |
+| UserID                | Int64     | NO          |
+| CounterClass          | Int16     | NO          |
+| OS                    | Int16     | NO          |
+| UserAgent             | Int16     | NO          |
+| URL                   | Utf8      | NO          |
+| Referer               | Utf8      | NO          |
+| IsRefresh             | Int16     | NO          |
+| RefererCategoryID     | Int16     | NO          |
+| RefererRegionID       | Int32     | NO          |
+| URLCategoryID         | Int16     | NO          |
+| URLRegionID           | Int32     | NO          |
+| ResolutionWidth       | Int16     | NO          |
+| ResolutionHeight      | Int16     | NO          |
+| ResolutionDepth       | Int16     | NO          |
+| FlashMajor            | Int16     | NO          |
+| FlashMinor            | Int16     | NO          |
+| FlashMinor2           | Utf8      | NO          |
+| NetMajor              | Int16     | NO          |
+| NetMinor              | Int16     | NO          |
+| UserAgentMajor        | Int16     | NO          |
+| UserAgentMinor        | Utf8      | NO          |
+| CookieEnable          | Int16     | NO          |
+| JavascriptEnable      | Int16     | NO          |
+| IsMobile              | Int16     | NO          |
+| MobilePhone           | Int16     | NO          |
+| MobilePhoneModel      | Utf8      | NO          |
+| Params                | Utf8      | NO          |
+| IPNetworkID           | Int32     | NO          |
+| TraficSourceID        | Int16     | NO          |
+| SearchEngineID        | Int16     | NO          |
+| SearchPhrase          | Utf8      | NO          |
+| AdvEngineID           | Int16     | NO          |
+| IsArtifical           | Int16     | NO          |
+| WindowClientWidth     | Int16     | NO          |
+| WindowClientHeight    | Int16     | NO          |
+| ClientTimeZone        | Int16     | NO          |
+| ClientEventTime       | Int64     | NO          |
+| SilverlightVersion1   | Int16     | NO          |
+| SilverlightVersion2   | Int16     | NO          |
+| SilverlightVersion3   | Int32     | NO          |
+| SilverlightVersion4   | Int16     | NO          |
+| PageCharset           | Utf8      | NO          |
+| CodeVersion           | Int32     | NO          |
+| IsLink                | Int16     | NO          |
+| IsDownload            | Int16     | NO          |
+| IsNotBounce           | Int16     | NO          |
+| FUniqID               | Int64     | NO          |
+| OriginalURL           | Utf8      | NO          |
+| HID                   | Int32     | NO          |
+| IsOldCounter          | Int16     | NO          |
+| IsEvent               | Int16     | NO          |
+| IsParameter           | Int16     | NO          |
+| DontCountHits         | Int16     | NO          |
+| WithHash              | Int16     | NO          |
+| HitColor              | Utf8      | NO          |
+| LocalEventTime        | Int64     | NO          |
+| Age                   | Int16     | NO          |
+| Sex                   | Int16     | NO          |
+| Income                | Int16     | NO          |
+| Interests             | Int16     | NO          |
+| Robotness             | Int16     | NO          |
+| RemoteIP              | Int32     | NO          |
+| WindowName            | Int32     | NO          |
+| OpenerName            | Int32     | NO          |
+| HistoryLength         | Int16     | NO          |
+| BrowserLanguage       | Utf8      | NO          |
+| BrowserCountry        | Utf8      | NO          |
+| SocialNetwork         | Utf8      | NO          |
+| SocialAction          | Utf8      | NO          |
+| HTTPError             | Int16     | NO          |
+| SendTiming            | Int32     | NO          |
+| DNSTiming             | Int32     | NO          |
+| ConnectTiming         | Int32     | NO          |
+| ResponseStartTiming   | Int32     | NO          |
+| ResponseEndTiming     | Int32     | NO          |
+| FetchTiming           | Int32     | NO          |
+| SocialSourceNetworkID | Int16     | NO          |
+| SocialSourcePage      | Utf8      | NO          |
+| ParamPrice            | Int64     | NO          |
+| ParamOrderID          | Utf8      | NO          |
+| ParamCurrency         | Utf8      | NO          |
+| ParamCurrencyID       | Int16     | NO          |
+| OpenstatServiceName   | Utf8      | NO          |
+| OpenstatCampaignID    | Utf8      | NO          |
+| OpenstatAdID          | Utf8      | NO          |
+| OpenstatSourceID      | Utf8      | NO          |
+| UTMSource             | Utf8      | NO          |
+| UTMMedium             | Utf8      | NO          |
+| UTMCampaign           | Utf8      | NO          |
+| UTMContent            | Utf8      | NO          |
+| UTMTerm               | Utf8      | NO          |
+| FromTag               | Utf8      | NO          |
+| HasGCLID              | Int16     | NO          |
+| RefererHash           | Int64     | NO          |
+| URLHash               | Int64     | NO          |
+| CLID                  | Int32     | NO          |
++-----------------------+-----------+-------------+
+105 rows in set. Query took 0.034 seconds.
+
+```

benchmarks/queries/clickbench/extended.sql

@@ -1 +1,3 @@
-SELECT COUNT(DISTINCT "SearchPhrase"), COUNT(DISTINCT "MobilePhone"), COUNT(DISTINCT "MobilePhoneModel") FROM hits;
+SELECT COUNT(DISTINCT "SearchPhrase"), COUNT(DISTINCT "MobilePhone"), COUNT(DISTINCT "MobilePhoneModel") FROM hits;
+SELECT COUNT(DISTINCT "HitColor"), COUNT(DISTINCT "BrowserCountry"), COUNT(DISTINCT "BrowserLanguage")  FROM hits;
+SELECT "BrowserCountry",  COUNT(DISTINCT "SocialNetwork"), COUNT(DISTINCT "HitColor"), COUNT(DISTINCT "BrowserLanguage"), COUNT(DISTINCT "SocialAction") FROM hits GROUP BY 1 ORDER BY 2 DESC LIMIT 10;