Update results to Datafusion 46

datafusion/README.md

@@ -2,37 +2,37 @@
 
 DataFusion is an extensible query execution framework, written in Rust, that uses Apache Arrow as its in-memory format. For more information, please check <https://arrow.apache.org/datafusion/user-guide/introduction.html>
 
-We use parquet file here and create an external table for it; and then do the queries.
+We use parquet file here and create an external table for it; and then execute the queries.
 
 ## Generate benchmark results
 
 The benchmark should be completed in under an hour. On-demand pricing is $0.6 per hour while spot pricing is only $0.2 to $0.3 per hour (us-east-2).
 
 1. manually start a AWS EC2 instance
     - `c6a.4xlarge`
-    - Amazon Linux 2 AMI
+    - Ubuntu 22.04 or later
     - Root 500GB gp2 SSD
     - no EBS optimized
     - no instance store
-1. wait for status check passed, then ssh to EC2 `ssh ec2-user@{ip}`
-1. `sudo yum update -y` and `sudo yum install gcc git -y`
+1. wait for status check passed, then ssh to EC2 `ssh ubuntu@{ip}`
 1. `git clone https://github.com/ClickHouse/ClickBench`
 1. `cd ClickBench/datafusion`
 1. `vi benchmark.sh` and modify following line to target Datafusion version
+
+    ```bash
+    git checkout 46.0.0
     ```
-    git checkout 45.0.0
-    ```
+
 1. `bash benchmark.sh`
 
-### Know Issues:
+### Know Issues
 
 1. importing parquet by `datafusion-cli` doesn't support schema, need to add some casting in queries.sql (e.g. converting EventTime from Int to Timestamp via `to_timestamp_seconds`)
 2. importing parquet by `datafusion-cli` make column name column name case-sensitive, i change all column name in queries.sql to double quoted literal (e.g. `EventTime` -> `"EventTime"`)
 3. `comparing binary with utf-8` and `group by binary` don't work in mac, if you run these queries in mac, you'll get some errors for queries contain binary format apache/arrow-datafusion#3050
 
-
 ## Generate full human readable results (for debugging)
 
 1. install datafusion-cli
 2. download the parquet ```wget --continue https://datasets.clickhouse.com/hits_compatible/hits.parquet```
-3. execute it ```datafusion-cli -f create.sh queries.sh``` or ```bash run2.sh```
+3. execute it ```datafusion-cli -f create_single.sql queries.sql``` or ```bash run2.sh```

datafusion/benchmark.sh

@@ -11,9 +11,9 @@ sudo apt-get install --yes gcc
 
 echo "Install DataFusion main branch"
 git clone https://github.com/apache/arrow-datafusion.git
-cd arrow-datafusion/datafusion-cli
-git checkout 45.0.0
-CARGO_PROFILE_RELEASE_LTO=true RUSTFLAGS="-C codegen-units=1" cargo build --release
+cd arrow-datafusion/
+git checkout 46.0.0
+CARGO_PROFILE_RELEASE_LTO=true RUSTFLAGS="-C codegen-units=1" cargo build --release --package datafusion-cli --bin datafusion-cli
 export PATH="`pwd`/target/release:$PATH"
 cd ../..
 

datafusion/results/partitioned.json

@@ -1,55 +1,55 @@
 {
     "system": "DataFusion (Parquet, partitioned)",
-    "date": "2024-03-29",
+    "date": "2024-04-18",
     "machine": "c6a.4xlarge, 500gb gp2",
     "cluster_size": 1,
-    "comment": "v45.0.0 (26058ac)",
+    "comment": "v46.0.0 (d5ca830)",
     "tags": ["Rust", "column-oriented", "embedded", "stateless"],
     "load_time": 0,
-    "data_size": 14779976446,
+    "data_size": 14737666736,
     "result": [
-[0.060, 0.022, 0.021],
-[0.109, 0.034, 0.035],
-[0.198, 0.085, 0.083],
-[0.391, 0.088, 0.084],
-[1.143, 0.846, 0.872],
-[1.020, 0.856, 0.855],
-[0.086, 0.032, 0.028],
-[0.118, 0.037, 0.037],
-[1.102, 0.962, 0.942],
-[1.353, 1.070, 1.045],
-[0.487, 0.260, 0.263],
-[0.663, 0.291, 0.286],
-[1.114, 0.893, 0.901],
-[2.596, 1.410, 1.360],
-[1.133, 0.860, 0.854],
-[1.132, 1.020, 1.001],
-[2.668, 1.835, 1.866],
-[2.557, 1.694, 1.704],
-[5.337, 3.714, 3.794],
-[0.263, 0.082, 0.082],
-[9.891, 1.109, 1.125],
-[11.284, 1.331, 1.348],
-[21.820, 2.617, 2.631],
-[55.448, 9.609, 9.630],
-[2.687, 0.452, 0.453],
-[0.804, 0.368, 0.364],
-[2.704, 0.517, 0.520],
-[9.662, 1.553, 1.507],
-[9.988, 9.801, 9.769],
-[0.526, 0.421, 0.403],
-[2.371, 0.802, 0.812],
-[5.944, 0.904, 0.903],
-[4.827, 3.645, 3.565],
-[10.196, 3.767, 3.792],
-[10.234, 3.823, 3.844],
-[1.397, 1.270, 1.303],
-[0.328, 0.146, 0.147],
-[0.196, 0.085, 0.105],
-[0.328, 0.147, 0.150],
-[0.482, 0.220, 0.219],
-[0.198, 0.076, 0.076],
-[0.189, 0.088, 0.076],
-[0.179, 0.064, 0.075]
+[0.062, 0.021, 0.021],
+[0.120, 0.037, 0.036],
+[0.212, 0.084, 0.084],
+[0.388, 0.089, 0.082],
+[1.037, 0.875, 0.879],
+[1.002, 0.863, 0.864],
+[0.082, 0.033, 0.032],
+[0.117, 0.039, 0.038],
+[1.120, 0.950, 0.961],
+[1.319, 1.056, 1.058],
+[0.509, 0.267, 0.261],
+[0.612, 0.290, 0.289],
+[1.105, 0.919, 0.922],
+[2.552, 1.418, 1.415],
+[1.100, 0.859, 0.877],
+[1.155, 1.016, 1.018],
+[2.650, 1.852, 1.855],
+[2.553, 1.718, 1.738],
+[5.359, 3.667, 3.813],
+[0.263, 0.084, 0.085],
+[9.892, 1.113, 1.136],
+[11.274, 1.352, 1.349],
+[21.836, 2.640, 2.605],
+[55.533, 9.673, 9.740],
+[2.690, 0.462, 0.462],
+[0.812, 0.381, 0.368],
+[2.697, 0.539, 0.533],
+[9.607, 1.547, 1.526],
+[10.160, 9.670, 9.757],
+[0.530, 0.432, 0.430],
+[2.388, 0.810, 0.836],
+[5.962, 0.921, 0.946],
+[4.766, 3.673, 3.741],
+[10.197, 3.789, 3.811],
+[10.207, 3.851, 3.917],
+[1.392, 1.259, 1.251],
+[0.331, 0.148, 0.150],
+[0.229, 0.085, 0.085],
+[0.323, 0.146, 0.162],
+[0.479, 0.222, 0.233],
+[0.215, 0.079, 0.079],
+[0.201, 0.073, 0.074],
+[0.182, 0.065, 0.064]
 ]
 }

datafusion/results/single.json

@@ -1,55 +1,55 @@
 {
     "system": "DataFusion (Parquet, single)",
-    "date": "2024-03-29",
+    "date": "2024-04-18",
     "machine": "c6a.4xlarge, 500gb gp2",
     "cluster_size": 1,
-    "comment": "v45.0.0 (26058ac)",
+    "comment": "v46.0.0 (d5ca830)",
     "tags": ["Rust", "column-oriented", "embedded", "stateless"],
     "load_time": 0,
     "data_size": 14779976446,
     "result": [
-[0.103, 0.063, 0.070],
-[0.136, 0.080, 0.079],
-[0.219, 0.117, 0.116],
-[0.348, 0.128, 0.128],
-[1.045, 0.913, 0.924],
-[1.102, 0.959, 0.964],
-[0.114, 0.066, 0.078],
-[0.152, 0.088, 0.087],
-[1.160, 1.021, 1.005],
-[1.324, 1.077, 1.110],
-[0.470, 0.297, 0.297],
-[0.567, 0.317, 0.312],
-[1.140, 0.984, 0.984],
-[2.681, 1.388, 1.459],
-[1.106, 0.952, 0.939],
-[1.185, 1.062, 1.067],
-[2.647, 1.943, 1.937],
-[2.524, 1.787, 1.787],
-[5.212, 3.749, 3.825],
-[0.272, 0.115, 0.122],
-[9.741, 1.205, 1.190],
-[11.298, 1.552, 1.497],
-[22.086, 3.670, 3.620],
-[55.936, 10.118, 10.120],
-[2.553, 0.572, 0.591],
-[0.792, 0.519, 0.512],
-[2.561, 0.639, 0.634],
-[9.600, 1.650, 1.682],
-[10.898, 10.343, 10.278],
-[0.556, 0.455, 0.459],
-[2.282, 0.938, 0.932],
-[5.685, 1.033, 1.025],
-[4.576, 3.773, 3.780],
-[10.309, 3.906, 3.927],
-[10.317, 3.969, 4.025],
-[1.395, 1.251, 1.253],
-[0.364, 0.202, 0.199],
-[0.284, 0.163, 0.164],
-[0.385, 0.216, 0.198],
-[0.541, 0.302, 0.295],
-[0.224, 0.115, 0.111],
-[0.215, 0.108, 0.111],
-[0.193, 0.102, 0.100]
+[0.103, 0.067, 0.060],
+[0.137, 0.083, 0.074],
+[0.228, 0.116, 0.118],
+[0.341, 0.124, 0.122],
+[1.033, 0.907, 0.936],
+[1.097, 0.981, 0.982],
+[0.118, 0.086, 0.083],
+[0.150, 0.082, 0.081],
+[1.141, 1.053, 1.024],
+[1.312, 1.103, 1.145],
+[0.483, 0.297, 0.300],
+[0.577, 0.330, 0.323],
+[1.157, 1.007, 1.004],
+[2.772, 1.420, 1.393],
+[1.115, 0.959, 0.973],
+[1.177, 1.081, 1.070],
+[2.626, 1.971, 1.971],
+[2.513, 1.802, 1.812],
+[5.277, 3.894, 3.846],
+[0.260, 0.124, 0.122],
+[9.730, 1.190, 1.210],
+[11.254, 1.453, 1.484],
+[22.102, 3.639, 3.597],
+[55.988, 10.171, 10.220],
+[2.557, 0.589, 0.577],
+[0.811, 0.536, 0.518],
+[2.574, 0.652, 0.657],
+[9.604, 1.652, 1.635],
+[10.793, 10.454, 10.628],
+[0.572, 0.448, 0.480],
+[2.283, 0.933, 0.961],
+[5.690, 1.038, 1.046],
+[4.496, 3.738, 3.841],
+[10.294, 3.946, 3.940],
+[10.197, 3.949, 4.030],
+[1.416, 1.289, 1.272],
+[0.382, 0.195, 0.212],
+[0.274, 0.184, 0.168],
+[0.369, 0.200, 0.196],
+[0.533, 0.296, 0.285],
+[0.243, 0.111, 0.111],
+[0.216, 0.112, 0.127],
+[0.193, 0.103, 0.102]
 ]
 }

datafusion/run.sh

@@ -31,7 +31,7 @@ cat queries.sql | while read -r query; do
         # 2. each query contains a "Query took xxx seconds", we just grep these 2 lines
         # 3. use sed to take the second line
         # 4. use awk to take the number we want
-        RES=`datafusion-cli -f $CREATE_SQL_FILE /tmp/query.sql 2>&1 | grep "Elapsed" |sed -n 2p | awk '{ print $2 }'`
+        RES=$(datafusion-cli -f $CREATE_SQL_FILE /tmp/query.sql 2>&1 | grep "Elapsed" |sed -n 2p | awk '{ print $2 }')
         [[ $RES != "" ]] && \
             echo -n "$RES" || \
             echo -n "null"