add tpch suite

Author: yew1eb

.rat-excludes

@@ -32,3 +32,8 @@
 NOTICE*
 docs/**
 build/apache-maven-*/**
+**/tpch-data-parquet/**
+**/tpch-plan-stability/**
+**/tpch-queries/**
+**/tpch-query-results/**
+

spark-extension-shims-spark/src/test/resources/tpch-data-parquet/customer/part-00000-3f8f9b4c-2c8a-4f5a-9d79-3c8c7f2b6e13-c000.snappy.parquet

@@ -0,0 +1,151 @@
+== Physical Plan ==
+AdaptiveSparkPlan (26)
++- == Final Plan ==
+   NativeSort (17)
+   +- InputAdapter (16)
+      +- AQEShuffleRead (15)
+         +- ShuffleQueryStage (14), Statistics(sizeInBytes=2.6 KiB, rowCount=4)
+            +- NativeShuffleExchange (13)
+               +- NativeProject (12)
+                  +- NativeHashAggregate (11)
+                     +- InputAdapter (10)
+                        +- AQEShuffleRead (9)
+                           +- ShuffleQueryStage (8), Statistics(sizeInBytes=1736.0 B, rowCount=4)
+                              +- NativeShuffleExchange (7)
+                                 +- NativeHashAggregate (6)
+                                    +- NativeProject (5)
+                                       +- NativeProject (4)
+                                          +- NativeFilter (3)
+                                             +- InputAdapter (2)
+                                                +- NativeParquetScan  (1)
++- == Initial Plan ==
+   Sort (25)
+   +- Exchange (24)
+      +- HashAggregate (23)
+         +- Exchange (22)
+            +- HashAggregate (21)
+               +- Project (20)
+                  +- Filter (19)
+                     +- Scan parquet  (18)
+
+
+(18) Scan parquet 
+Output [7]: [l_quantity#1, l_extendedprice#2, l_discount#3, l_tax#4, l_returnflag#5, l_linestatus#6, l_shipdate#7]
+Batched: true
+Location: InMemoryFileIndex [file:/<warehouse_dir>]
+PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)]
+ReadSchema: struct<l_quantity:decimal(12,2),l_extendedprice:decimal(12,2),l_discount:decimal(12,2),l_tax:decimal(12,2),l_returnflag:string,l_linestatus:string,l_shipdate:date>
+
+(2) InputAdapter
+Input [7]: [l_quantity#1, l_extendedprice#2, l_discount#3, l_tax#4, l_returnflag#5, l_linestatus#6, l_shipdate#7]
+Arguments: [#1, #2, #3, #4, #5, #6, #7]
+
+(3) NativeFilter
+Input [7]: [#1#1, #2#2, #3#3, #4#4, #5#5, #6#6, #7#7]
+Condition : (isnotnull(l_shipdate#7) AND (l_shipdate#7 <= 1998-09-02))
+
+(4) NativeProject
+Output [6]: [l_quantity#1, l_extendedprice#2, l_discount#3, l_tax#4, l_returnflag#5, l_linestatus#6]
+Input [7]: [#1#1, #2#2, #3#3, #4#4, #5#5, #6#6, #7#7]
+
+(5) NativeProject
+Output [7]: [l_returnflag#5 AS l_returnflag#5, l_linestatus#6 AS l_linestatus#6, l_quantity#1 AS _c2#8, l_extendedprice#2 AS _c3#9, (l_extendedprice#2 * (1 - l_discount#3)) AS _c4#10, ((l_extendedprice#2 * (1 - l_discount#3)) * (1 + l_tax#4)) AS _c5#11, l_discount#3 AS _c6#12]
+Input [6]: [l_quantity#1, l_extendedprice#2, l_discount#3, l_tax#4, l_returnflag#5, l_linestatus#6]
+
+(6) NativeHashAggregate
+Input [7]: [l_returnflag#5, l_linestatus#6, _c2#8, _c3#9, _c4#10, _c5#11, _c6#12]
+Keys [2]: [l_returnflag#5, l_linestatus#6]
+Functions [8]: [partial_sum(_c2#8), partial_sum(_c3#9), partial_sum(_c4#10), partial_sum(_c5#11), partial_avg(_c2#8), partial_avg(_c3#9), partial_avg(_c6#12), partial_count(1)]
+Aggregate Attributes [15]: [sum#13, isEmpty#14, sum#15, isEmpty#16, sum#17, isEmpty#18, sum#19, isEmpty#20, sum#21, count#22, sum#23, count#24, sum#25, count#26, count#27]
+Results [3]: [l_returnflag#5, l_linestatus#6, #28#28]
+
+(7) NativeShuffleExchange
+Input [3]: [l_returnflag#5, l_linestatus#6, #28#28]
+Arguments: hashpartitioning(l_returnflag#5, l_linestatus#6, 5), ENSURE_REQUIREMENTS, [plan_id=1]
+
+(8) ShuffleQueryStage
+Output [3]: [l_returnflag#5, l_linestatus#6, #28#28]
+Arguments: X
+
+(9) AQEShuffleRead
+Input [3]: [l_returnflag#5, l_linestatus#6, #28#28]
+Arguments: coalesced
+
+(10) InputAdapter
+Input [3]: [l_returnflag#5, l_linestatus#6, #28#28]
+
+(11) NativeHashAggregate
+Input [3]: [l_returnflag#5, l_linestatus#6, #28#28]
+Keys [2]: [l_returnflag#5, l_linestatus#6]
+Functions [8]: [sum(l_quantity#1), sum(l_extendedprice#2), sum((l_extendedprice#2 * (1 - l_discount#3))), sum(((l_extendedprice#2 * (1 - l_discount#3)) * (1 + l_tax#4))), avg(l_quantity#1), avg(l_extendedprice#2), avg(l_discount#3), count(1)]
+Aggregate Attributes [8]: [sum(l_quantity#1)#29, sum(l_extendedprice#2)#30, sum((l_extendedprice#2 * (1 - l_discount#3)))#31, sum(((l_extendedprice#2 * (1 - l_discount#3)) * (1 + l_tax#4)))#32, avg(l_quantity#1)#33, avg(l_extendedprice#2)#34, avg(l_discount#3)#35, count(1)#36]
+Results [10]: [l_returnflag#5, l_linestatus#6, sum(l_quantity#1)#29, sum(l_extendedprice#2)#30, sum((l_extendedprice#2 * (1 - l_discount#3)))#31, sum(((l_extendedprice#2 * (1 - l_discount#3)) * (1 + l_tax#4)))#32, avg(l_quantity#1)#33, avg(l_extendedprice#2)#34, avg(l_discount#3)#35, count(1)#36]
+
+(12) NativeProject
+Output [10]: [l_returnflag#5, l_linestatus#6, sum(l_quantity#1)#29 AS sum_qty#37, sum(l_extendedprice#2)#30 AS sum_base_price#38, sum((l_extendedprice#2 * (1 - l_discount#3)))#31 AS sum_disc_price#39, sum(((l_extendedprice#2 * (1 - l_discount#3)) * (1 + l_tax#4)))#32 AS sum_charge#40, avg(l_quantity#1)#33 AS avg_qty#41, avg(l_extendedprice#2)#34 AS avg_price#42, avg(l_discount#3)#35 AS avg_disc#43, count(1)#36 AS count_order#44]
+Input [10]: [l_returnflag#5, l_linestatus#6, sum(l_quantity#1)#29, sum(l_extendedprice#2)#30, sum((l_extendedprice#2 * (1 - l_discount#3)))#31, sum(((l_extendedprice#2 * (1 - l_discount#3)) * (1 + l_tax#4)))#32, avg(l_quantity#1)#33, avg(l_extendedprice#2)#34, avg(l_discount#3)#35, count(1)#36]
+
+(13) NativeShuffleExchange
+Input [10]: [l_returnflag#5, l_linestatus#6, sum_qty#37, sum_base_price#38, sum_disc_price#39, sum_charge#40, avg_qty#41, avg_price#42, avg_disc#43, count_order#44]
+Arguments: rangepartitioning(l_returnflag#5 ASC NULLS FIRST, l_linestatus#6 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=2]
+
+(14) ShuffleQueryStage
+Output [10]: [l_returnflag#5, l_linestatus#6, sum_qty#37, sum_base_price#38, sum_disc_price#39, sum_charge#40, avg_qty#41, avg_price#42, avg_disc#43, count_order#44]
+Arguments: X
+
+(15) AQEShuffleRead
+Input [10]: [l_returnflag#5, l_linestatus#6, sum_qty#37, sum_base_price#38, sum_disc_price#39, sum_charge#40, avg_qty#41, avg_price#42, avg_disc#43, count_order#44]
+Arguments: coalesced
+
+(16) InputAdapter
+Input [10]: [l_returnflag#5, l_linestatus#6, sum_qty#37, sum_base_price#38, sum_disc_price#39, sum_charge#40, avg_qty#41, avg_price#42, avg_disc#43, count_order#44]
+
+(17) NativeSort
+Input [10]: [l_returnflag#5, l_linestatus#6, sum_qty#37, sum_base_price#38, sum_disc_price#39, sum_charge#40, avg_qty#41, avg_price#42, avg_disc#43, count_order#44]
+Arguments: [l_returnflag#5 ASC NULLS FIRST, l_linestatus#6 ASC NULLS FIRST], true
+
+(18) Scan parquet 
+Output [7]: [l_quantity#1, l_extendedprice#2, l_discount#3, l_tax#4, l_returnflag#5, l_linestatus#6, l_shipdate#7]
+Batched: true
+Location: InMemoryFileIndex [file:/<warehouse_dir>]
+PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)]
+ReadSchema: struct<l_quantity:decimal(12,2),l_extendedprice:decimal(12,2),l_discount:decimal(12,2),l_tax:decimal(12,2),l_returnflag:string,l_linestatus:string,l_shipdate:date>
+
+(19) Filter
+Input [7]: [l_quantity#1, l_extendedprice#2, l_discount#3, l_tax#4, l_returnflag#5, l_linestatus#6, l_shipdate#7]
+Condition : (isnotnull(l_shipdate#7) AND (l_shipdate#7 <= 1998-09-02))
+
+(20) Project
+Output [6]: [l_quantity#1, l_extendedprice#2, l_discount#3, l_tax#4, l_returnflag#5, l_linestatus#6]
+Input [7]: [l_quantity#1, l_extendedprice#2, l_discount#3, l_tax#4, l_returnflag#5, l_linestatus#6, l_shipdate#7]
+
+(21) HashAggregate
+Input [6]: [l_quantity#1, l_extendedprice#2, l_discount#3, l_tax#4, l_returnflag#5, l_linestatus#6]
+Keys [2]: [l_returnflag#5, l_linestatus#6]
+Functions [8]: [partial_sum(l_quantity#1), partial_sum(l_extendedprice#2), partial_sum((l_extendedprice#2 * (1 - l_discount#3))), partial_sum(((l_extendedprice#2 * (1 - l_discount#3)) * (1 + l_tax#4))), partial_avg(l_quantity#1), partial_avg(l_extendedprice#2), partial_avg(l_discount#3), partial_count(1)]
+Aggregate Attributes [15]: [sum#13, isEmpty#14, sum#15, isEmpty#16, sum#17, isEmpty#18, sum#19, isEmpty#20, sum#21, count#22, sum#23, count#24, sum#25, count#26, count#27]
+Results [17]: [l_returnflag#5, l_linestatus#6, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, count#59]
+
+(22) Exchange
+Input [17]: [l_returnflag#5, l_linestatus#6, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, count#59]
+Arguments: hashpartitioning(l_returnflag#5, l_linestatus#6, 5), ENSURE_REQUIREMENTS, [plan_id=3]
+
+(23) HashAggregate
+Input [17]: [l_returnflag#5, l_linestatus#6, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, count#59]
+Keys [2]: [l_returnflag#5, l_linestatus#6]
+Functions [8]: [sum(l_quantity#1), sum(l_extendedprice#2), sum((l_extendedprice#2 * (1 - l_discount#3))), sum(((l_extendedprice#2 * (1 - l_discount#3)) * (1 + l_tax#4))), avg(l_quantity#1), avg(l_extendedprice#2), avg(l_discount#3), count(1)]
+Aggregate Attributes [8]: [sum(l_quantity#1)#29, sum(l_extendedprice#2)#30, sum((l_extendedprice#2 * (1 - l_discount#3)))#31, sum(((l_extendedprice#2 * (1 - l_discount#3)) * (1 + l_tax#4)))#32, avg(l_quantity#1)#33, avg(l_extendedprice#2)#34, avg(l_discount#3)#35, count(1)#36]
+Results [10]: [l_returnflag#5, l_linestatus#6, sum(l_quantity#1)#29 AS sum_qty#37, sum(l_extendedprice#2)#30 AS sum_base_price#38, sum((l_extendedprice#2 * (1 - l_discount#3)))#31 AS sum_disc_price#39, sum(((l_extendedprice#2 * (1 - l_discount#3)) * (1 + l_tax#4)))#32 AS sum_charge#40, avg(l_quantity#1)#33 AS avg_qty#41, avg(l_extendedprice#2)#34 AS avg_price#42, avg(l_discount#3)#35 AS avg_disc#43, count(1)#36 AS count_order#44]
+
+(24) Exchange
+Input [10]: [l_returnflag#5, l_linestatus#6, sum_qty#37, sum_base_price#38, sum_disc_price#39, sum_charge#40, avg_qty#41, avg_price#42, avg_disc#43, count_order#44]
+Arguments: rangepartitioning(l_returnflag#5 ASC NULLS FIRST, l_linestatus#6 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=4]
+
+(25) Sort
+Input [10]: [l_returnflag#5, l_linestatus#6, sum_qty#37, sum_base_price#38, sum_disc_price#39, sum_charge#40, avg_qty#41, avg_price#42, avg_disc#43, count_order#44]
+Arguments: [l_returnflag#5 ASC NULLS FIRST, l_linestatus#6 ASC NULLS FIRST], true, 0
+
+(26) AdaptiveSparkPlan
+Output [10]: [l_returnflag#5, l_linestatus#6, sum_qty#37, sum_base_price#38, sum_disc_price#39, sum_charge#40, avg_qty#41, avg_price#42, avg_disc#43, count_order#44]
+Arguments: isFinalPlan=true
+