diff --git a/cluster/src/dunit/scala/org/apache/spark/sql/store/SortedColumnDUnitTest.scala b/cluster/src/dunit/scala/org/apache/spark/sql/store/SortedColumnDUnitTest.scala new file mode 100644 index 0000000000..2a9c7e095a --- /dev/null +++ b/cluster/src/dunit/scala/org/apache/spark/sql/store/SortedColumnDUnitTest.scala @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package org.apache.spark.sql.store + +import scala.concurrent.duration.{FiniteDuration, MINUTES} + +import io.snappydata.cluster.ClusterManagerTestBase + +import org.apache.spark.sql.SnappyContext + +/** + * SortedColumnTests and SortedColumnPerformanceTests in DUnit. + */ +class SortedColumnDUnitTest(s: String) extends ClusterManagerTestBase(s) { + + def testDummy(): Unit = {} + + def disabled_testBasicInsert(): Unit = { + val snc = SnappyContext(sc).snappySession + val colTableName = "colDeltaTable" + val numElements = 551 + val numBuckets = 2 + + SortedColumnTests.verfiyInsertDataExists(snc, numElements) + SortedColumnTests.verfiyUpdateDataExists(snc, numElements) + SortedColumnTests.testBasicInsert(snc, colTableName, numBuckets, numElements) + } + + def disabled_testPointQueryPerformance() { + val snc = SnappyContext(sc).snappySession + val colTableName = "colDeltaTable" + val numElements = 999551 + val numBuckets = SortedColumnPerformanceTests.cores + val numIters = 100 + SortedColumnPerformanceTests.benchmarkMultiThreaded(snc, colTableName, numBuckets, numElements, + numIters, "PointQuery", numTimesInsert = 10, + doVerifyFullSize = true)(SortedColumnPerformanceTests.executeQuery_PointQuery_mt) + // while (true) {} + } + + def disabled_testPointQueryPerformanceMultithreaded() { + val snc = SnappyContext(sc).snappySession + val colTableName = "colDeltaTable" + val numElements = 999551 + val numBuckets = SortedColumnPerformanceTests.cores + val numIters = 100 + val totalNumThreads = SortedColumnPerformanceTests.cores + val totalTime: FiniteDuration = new FiniteDuration(5, MINUTES) + SortedColumnPerformanceTests.benchmarkMultiThreaded(snc, colTableName, numBuckets, numElements, + numIters, "PointQuery multithreaded", numTimesInsert = 10, isMultithreaded = true, + doVerifyFullSize = false, totalThreads = totalNumThreads, + runTime = totalTime)(SortedColumnPerformanceTests.executeQuery_PointQuery_mt) + // while (true) {} + } + + def disabled_testRangeQueryPerformance() { + val snc = SnappyContext(sc).snappySession + val colTableName = "colDeltaTable" + val numElements = 999551 + val numBuckets = SortedColumnPerformanceTests.cores + val numIters = 21 + SortedColumnPerformanceTests.benchmarkMultiThreaded(snc, colTableName, numBuckets, numElements, + numIters, "RangeQuery", numTimesInsert = 10, + doVerifyFullSize = true)(SortedColumnPerformanceTests.executeQuery_RangeQuery_mt) + // while (true) {} + } +} diff --git a/cluster/src/test/scala/org/apache/spark/sql/store/SortedColumnPerformanceTests.scala b/cluster/src/test/scala/org/apache/spark/sql/store/SortedColumnPerformanceTests.scala new file mode 100644 index 0000000000..cc83473cb0 --- /dev/null +++ b/cluster/src/test/scala/org/apache/spark/sql/store/SortedColumnPerformanceTests.scala @@ -0,0 +1,525 @@ +/* + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package org.apache.spark.sql.store + +import scala.concurrent.duration.FiniteDuration + +import io.snappydata.Property + +import org.apache.spark.SparkConf +import org.apache.spark.memory.SnappyUnifiedMemoryManager +import org.apache.spark.sql.execution.benchmark.ColumnCacheBenchmark +import org.apache.spark.sql.execution.columnar.ColumnTableScan +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.{DataFrame, DataFrameReader, SnappySession} +import org.apache.spark.util.{Benchmark, MultiThreadedBenchmark} +import scala.concurrent.duration._ + +/** + * Tests for column table having sorted columns. + */ +class SortedColumnPerformanceTests extends ColumnTablesTestBase { + + override def newSparkConf(addOn: SparkConf => SparkConf = null): SparkConf = { + val conf = new SparkConf() + .setIfMissing("spark.master", s"local[${SortedColumnPerformanceTests.cores}]") + .setAppName("microbenchmark") + conf.set("snappydata.store.critical-heap-percentage", "95") + if (SnappySession.isEnterpriseEdition) { + conf.set("snappydata.store.memory-size", "1200m") + } + conf.set("spark.memory.manager", classOf[SnappyUnifiedMemoryManager].getName) + conf.set("spark.serializer", "org.apache.spark.serializer.PooledKryoSerializer") + conf.set("spark.closure.serializer", "org.apache.spark.serializer.PooledKryoSerializer") + if (addOn != null) { + addOn(conf) + } + conf + } + + test("dummy") {} + + ignore("insert performance") { + val session = this.snc.snappySession + val colTableName = "colDeltaTable" + val numElements = 100000000 + val numTimesInsert = 1 + val numTimesUpdate = 1 + + val totalElements = (numElements * 0.6 * numTimesUpdate + + numElements * 0.4 * numTimesUpdate).toLong + val numBuckets = 4 + val numIters = 30 + + SortedColumnTests.verfiyInsertDataExists(session, numElements, numTimesInsert) + SortedColumnTests.verfiyUpdateDataExists(session, numElements, numTimesUpdate) + val dataFrameReader : DataFrameReader = session.read + val insertDF: DataFrame = dataFrameReader.load(SortedColumnTests.filePathInsert(numElements, + numTimesInsert)) + val updateDF: DataFrame = dataFrameReader.load(SortedColumnTests.filePathUpdate(numElements, + numTimesUpdate)) + + def prepare(): Unit = { + session.conf.set(Property.ColumnBatchSize.name, "24M") // default + session.conf.set(Property.ColumnMaxDeltaRows.name, "100") + } + + def testPrepare(): Unit = { + SortedColumnTests.createColumnTable(session, colTableName, numBuckets, numElements) + } + + def cleanup(): Unit = { + session.conf.unset(Property.ColumnBatchSize.name) + session.conf.unset(Property.ColumnMaxDeltaRows.name) + } + + def testCleanup(): Unit = { + SortedColumnTests.dropColumnTable(session, colTableName) + } + + val benchmark = new Benchmark("InsertQuery", totalElements) + var iter = 1 + ColumnCacheBenchmark.addCaseWithCleanup(benchmark, "Sorted", numIters, prepare, cleanup, + testCleanup, testPrepare) { _ => + var j = 0 + while (j < numTimesInsert) { + insertDF.write.insertInto(colTableName) + j += 1 + } + j = 0 + while (j < numTimesUpdate) { + updateDF.write.insertInto(colTableName) + j += 1 + } + iter += 1 + } + benchmark.run() + // Thread.sleep(50000000) + } + + ignore("PointQuery performance") { + val session = this.snc.snappySession + val colTableName = "colDeltaTable" + val numElements = 999551 + val numTimesInsert = 199 + val numTimesUpdate = 1 + + val totalElements = (numElements * 0.6 * numTimesUpdate + + numElements * 0.4 * numTimesUpdate).toLong + val numBuckets = 4 + val numIters = 1000 + + SortedColumnTests.verfiyInsertDataExists(session, numElements, numTimesInsert) + SortedColumnTests.verfiyUpdateDataExists(session, numElements, numTimesUpdate) + val dataFrameReader : DataFrameReader = session.read + val insertDF: DataFrame = dataFrameReader.load(SortedColumnTests.filePathInsert(numElements, + numTimesInsert)) + val updateDF: DataFrame = dataFrameReader.load(SortedColumnTests.filePathUpdate(numElements, + numTimesUpdate)) + + def prepare(): Unit = { + SortedColumnTests.createColumnTable(session, colTableName, numBuckets, numElements) + try { + session.conf.set(Property.ColumnBatchSize.name, "24M") // default + session.conf.set(Property.ColumnMaxDeltaRows.name, "100") + insertDF.write.insertInto(colTableName) + updateDF.write.insertInto(colTableName) + } finally { + session.conf.unset(Property.ColumnBatchSize.name) + session.conf.unset(Property.ColumnMaxDeltaRows.name) + } + } + + val benchmark = new Benchmark("PointQuery", totalElements) + var iter = 1 + benchmark.addCase("Sorted", numIters, prepare) { _ => + SortedColumnPerformanceTests.executeQuery_PointQuery(session, colTableName, iter, + numTimesInsert, numTimesUpdate = 1) + iter += 1 + } + benchmark.run() + // Thread.sleep(50000000) + } + + ignore("JoinQuery performance") { + val session = this.snc.snappySession + val colTableName = "colDeltaTable" + val joinTableName = "joinDeltaTable" + val numElements = 100000000 + val numTimesInsert = 1 + val numTimesUpdate = 1 + + val totalElements = (numElements * 0.6 * numTimesUpdate + + numElements * 0.4 * numTimesUpdate).toLong + val numBuckets = 4 + val numIters = 100 + + SortedColumnTests.verfiyInsertDataExists(session, numElements, numTimesInsert) + SortedColumnTests.verfiyUpdateDataExists(session, numElements, numTimesUpdate) + val dataFrameReader : DataFrameReader = session.read + val insertDF: DataFrame = dataFrameReader.load(SortedColumnTests.filePathInsert(numElements, + numTimesInsert)) + val updateDF: DataFrame = dataFrameReader.load(SortedColumnTests.filePathUpdate(numElements, + numTimesUpdate)) + + def prepare(): Unit = { + SortedColumnTests.createColumnTable(session, colTableName, numBuckets, numElements) + SortedColumnTests.createColumnTable(session, joinTableName, numBuckets, numElements, + Some(colTableName)) + try { + session.conf.set(Property.ColumnBatchSize.name, "24M") // default + session.conf.set(Property.ColumnMaxDeltaRows.name, "100") + insertDF.write.insertInto(colTableName) + insertDF.write.insertInto(joinTableName) + + updateDF.write.insertInto(colTableName) + updateDF.write.insertInto(joinTableName) + } finally { + session.conf.unset(Property.ColumnBatchSize.name) + session.conf.unset(Property.ColumnMaxDeltaRows.name) + } + } + + val benchmark = new Benchmark("JoinQuery", totalElements) + var iter = 1 + try { + // Force SMJ + session.conf.set(Property.HashJoinSize.name, "-1") + session.conf.set(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key, "-1") + benchmark.addCase("Sorted", numIters, prepare) { _ => + SortedColumnPerformanceTests.executeQuery_JoinQuery(session, colTableName, joinTableName, + iter, numTimesInsert, numTimesUpdate = 1) + iter += 1 + } + benchmark.run() + } finally { + session.conf.unset(Property.HashJoinSize.name) + session.conf.unset(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key) + } + // Thread.sleep(50000000) + } + + ignore("PointQuery performance multithreaded 1") { + val snc = this.snc.snappySession + SortedColumnPerformanceTests.mutiThreadedPointQuery(snc, numThreads = 1) + // Thread.sleep(5000000) + } + + ignore("PointQuery performance multithreaded 4") { + val snc = this.snc.snappySession + val totalNumThreads = SortedColumnPerformanceTests.cores + SortedColumnPerformanceTests.mutiThreadedPointQuery(snc, totalNumThreads) + // Thread.sleep(5000000) + } + + ignore("PointQuery performance multithreaded 8") { + val snc = this.snc.snappySession + val totalNumThreads = 2 * SortedColumnPerformanceTests.cores + SortedColumnPerformanceTests.mutiThreadedPointQuery(snc, totalNumThreads) + // Thread.sleep(5000000) + } + + ignore("PointQuery performance multithreaded 16") { + val snc = this.snc.snappySession + val totalNumThreads = 4 * SortedColumnPerformanceTests.cores + SortedColumnPerformanceTests.mutiThreadedPointQuery(snc, totalNumThreads) + // Thread.sleep(5000000) + } + + ignore("PointQuery performance multithreaded 32") { + val snc = this.snc.snappySession + val totalNumThreads = 4 * SortedColumnPerformanceTests.cores + SortedColumnPerformanceTests.mutiThreadedPointQuery(snc, totalNumThreads) + // Thread.sleep(5000000) + } + + ignore("RangeQuery performance") { + val snc = this.snc.snappySession + val colTableName = "colDeltaTable" + val numElements = 999551 + val numBuckets = 3 + val numIters = 21 + SortedColumnPerformanceTests.benchmarkMultiThreaded(snc, colTableName, numBuckets, numElements, + numIters, "RangeQuery", numTimesInsert = 10, + doVerifyFullSize = true)(SortedColumnPerformanceTests.executeQuery_RangeQuery_mt) + // Thread.sleep(5000000) + } +} + +object SortedColumnPerformanceTests { + val cores: Int = math.min(16, Runtime.getRuntime.availableProcessors()) + + def executeQuery_PointQuery(session: SnappySession, colTableName: String, iterCount: Int, + numTimesInsert: Int, numTimesUpdate: Int): Unit = { + val param = getParam(iterCount, params) + val query = s"select * from $colTableName where id = $param" + val expectedNumResults = if (param % 10 < 6) numTimesInsert else numTimesUpdate + val result = session.sql(query).collect() + val passed = result.length == expectedNumResults + // scalastyle:off + // println(s"Query = $query result=${result.length} $expectedNumResults $iterCount") + // scalastyle:on + } + + def executeQuery_JoinQuery(session: SnappySession, colTableName: String, joinTableName: String, + iterCount: Int, numTimesInsert: Int, numTimesUpdate: Int): Unit = { + val query = s"select AVG(A.id), COUNT(B.id) " + + s" from $colTableName A inner join $joinTableName B where A.id = B.id" + val result = session.sql(query).collect() + // scalastyle:off + if (iterCount < 5) { + println(s"Query = $query result=${result.length}") + result.foreach(r => { + val avg = r.getDouble(0) + val count = r.getLong(1) + print(s"[$avg, $count], ") + }) + println() + } + // scalastyle:on + } + + private def doGC(): Unit = { + System.gc() + System.runFinalization() + System.gc() + System.runFinalization() + } + + var lastFailedIteration: Int = Int.MinValue + + def executeQuery_PointQuery_mt(session: SnappySession, colTableName: String, + joinTableName: String, numIters: Int, iterCount: Int, numThreads: Int, threadId: Int, + isMultithreaded: Boolean, numTimesInsert: Int, numTimesUpdate: Int): Boolean = { + val param = if (iterCount != lastFailedIteration) { + getParam(iterCount, params) + } else MultiThreadedBenchmark.firstRandomValue + val query = s"select * from $colTableName where id = $param" + val expectedNumResults = if (param % 10 < 6) numTimesInsert else numTimesUpdate + val result = session.sql(query).collect() + val passed = result.length == expectedNumResults + if (!passed && iterCount != -1) { + lastFailedIteration = iterCount + } + // scalastyle:off + // println(s"Query = $query result=${result.length} $expectedNumResults $iterCount" + + // s" $numThreads $threadId") + // scalastyle:on + passed + } + + def executeQuery_RangeQuery_mt(session: SnappySession, colTableName: String, + joinTableName: String, numIters: Int, iterCount: Int, numThreads: Int, threadId: Int, + isMultithreaded: Boolean, numTimesInsert: Int, numTimesUpdate: Int): Boolean = { + val param1 = if (iterCount != lastFailedIteration) { + getParam(iterCount, params1) + } else MultiThreadedBenchmark.firstRandomValue + val param2 = if (iterCount != lastFailedIteration) { + getParam(iterCount, params2) + } else MultiThreadedBenchmark.secondRandomValue + val (low, high) = if (param1 < param2) { (param1, param2)} else (param2, param1) + val query = s"select * from $colTableName where id between $low and $high" + val expectedNumResults = getParam(iterCount, params3) + val result = session.sql(query).collect() + val passed = if (iterCount != lastFailedIteration) { + expectedNumResults == result.length + } else result.length > 0 + if (!passed && iterCount != -1) { + lastFailedIteration = iterCount + } + // scalastyle:off + // println(s"Query = $query result=${result.length} $passed $expectedNumResults") + // scalastyle:on + passed + } + + // scalastyle:off + def benchmarkMultiThreaded(session: SnappySession, colTableName: String, numBuckets: Int, + numElements: Long, numIters: Int, queryMark: String, isMultithreaded: Boolean = false, + doVerifyFullSize: Boolean = false, numTimesInsert: Int = 1, numTimesUpdate: Int = 1, + totalThreads: Int = 1, runTime: FiniteDuration = 2.seconds, + joinTableName: Option[String] = None) + // scalastyle:on + (f : (SnappySession, String, String, Int, Int, Int, Int, Boolean, Int, + Int) => Boolean): Unit = { + val benchmark = new MultiThreadedBenchmark(s"Benchmark $queryMark", isMultithreaded, + numElements, outputPerIteration = true, numThreads = totalThreads, minTime = runTime) + SortedColumnTests.verfiyInsertDataExists(session, numElements, numTimesInsert) + SortedColumnTests.verfiyUpdateDataExists(session, numElements, numTimesUpdate) + val dataFrameReader : DataFrameReader = session.read + val insertDF : DataFrame = dataFrameReader.load(SortedColumnTests.filePathInsert(numElements, + numTimesInsert)) + val updateDF : DataFrame = dataFrameReader.load(SortedColumnTests.filePathUpdate(numElements, + numTimesUpdate)) + val sessionArray = new Array[SnappySession](totalThreads) + sessionArray.indices.foreach(i => { + sessionArray(i) = session.newSession() + sessionArray(i).conf.set(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "true") + sessionArray(i).conf.set(SQLConf.WHOLESTAGE_FALLBACK.key, "false") + sessionArray(i).conf.set(Property.ForceLinkPartitionsToBuckets.name, "true") // remove ? + }) + + def addBenchmark(name: String, params: Map[String, String] = Map()): Unit = { + val defaults = params.keys.flatMap { + k => session.conf.getOption(k).map((k, _)) + } + + def prepare(): Unit = { + params.foreach { case (k, v) => session.conf.set(k, v) } + SortedColumnTests.createColumnTable(session, colTableName, numBuckets, numElements) + if (joinTableName.isDefined) { + SortedColumnTests.createColumnTable(session, joinTableName.get, numBuckets, numElements, + Some(colTableName)) + } + try { + session.conf.set(Property.ColumnBatchSize.name, "24M") // default + session.conf.set(Property.ColumnMaxDeltaRows.name, "100") + session.conf.set(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "true") + session.conf.set(SQLConf.WHOLESTAGE_FALLBACK.key, "false") + session.conf.set(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key, "-1") + insertDF.write.insertInto(colTableName) + if (joinTableName.isDefined) { + insertDF.write.insertInto(joinTableName.get) + } + updateDF.write.insertInto(colTableName) + if (joinTableName.isDefined) { + updateDF.write.insertInto(joinTableName.get) + } + if (doVerifyFullSize) { + SortedColumnTests.verifyTotalRows(session, colTableName, numElements, finalCall = true, + numTimesInsert, numTimesUpdate) + if (joinTableName.isDefined) { + SortedColumnTests.verifyTotalRows(session, joinTableName.get, numElements, + finalCall = true, numTimesInsert, numTimesUpdate) + } + } + } finally { + session.conf.unset(Property.ColumnBatchSize.name) + session.conf.unset(Property.ColumnMaxDeltaRows.name) + session.conf.unset(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key) + session.conf.unset(SQLConf.WHOLESTAGE_FALLBACK.key) + session.conf.unset(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key) + } + doGC() + } + + def cleanup(): Unit = { + sessionArray.indices.foreach(i => { + sessionArray(i).clear() + session.conf.unset(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key) + session.conf.unset(SQLConf.WHOLESTAGE_FALLBACK.key) + session.conf.unset(Property.ForceLinkPartitionsToBuckets.name) + }) + SnappySession.clearAllCache() + defaults.foreach { case (k, v) => session.conf.set(k, v) } + doGC() + } + + def testCleanup(): Unit = { + doGC() + } + + addCaseWithCleanup(benchmark, name, numIters, prepare, + cleanup, testCleanup, isMultithreaded) { (iteratorIndex, threadId) => + f(sessionArray(threadId), colTableName, joinTableName.getOrElse("TableIsNotAvailiable"), + numIters, iteratorIndex, totalThreads, threadId, isMultithreaded, numTimesInsert, + numTimesUpdate)} + } + + try { + session.conf.set(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "true") + session.conf.set(SQLConf.WHOLESTAGE_FALLBACK.key, "false") + session.conf.set(Property.ForceLinkPartitionsToBuckets.name, "true") // remove ? + + // Get numbers + addBenchmark(s"$queryMark", Map.empty) + benchmark.run() + } finally { + try { + session.sql(s"drop table $colTableName") + if (joinTableName != null) { + session.sql(s"drop table $joinTableName") + } + } catch { + case _: Throwable => + } + session.conf.unset(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key) + session.conf.unset(SQLConf.WHOLESTAGE_FALLBACK.key) + session.conf.unset(Property.ForceLinkPartitionsToBuckets.name) + } + } + + def mutiThreadedPointQuery(snc: SnappySession, numThreads: Int): Unit = { + val colTableName = "colDeltaTable" + val numElements = 999551 + val numBuckets = 3 + val numIters = 100 + val totalTime: FiniteDuration = new FiniteDuration(5, MINUTES) + SortedColumnPerformanceTests.benchmarkMultiThreaded(snc, colTableName, numBuckets, numElements, + numIters, "PointQuery multithreaded", numTimesInsert = 200, isMultithreaded = true, + doVerifyFullSize = false, totalThreads = numThreads, + runTime = totalTime)(SortedColumnPerformanceTests.executeQuery_PointQuery_mt) + } + + val params = Array (424281, 587515, 907730, 122421, 735695, 964648, 450150, 904625, 562060, + 496352, 745467, 823402, 988429, 311420, 394233, 30710, 653570, 236224, 987974, 653351, 826605, + 245093, 707312, 14213, 733602, 344160, 367710, 578064, 416602, 302421, 618862, 804150, 371841, + 402904, 691030, 246012, 156893, 379762, 775281, 109154, 693942, 121663, 762882, 367055, 836784, + 508941, 606644, 331100, 958543, 15944, 89403, 181845, 562542, 809723, 736823, 708541, 546835, + 384221, 899713, 689019, 946529, 679341, 953504, 420572, 52560, 845940, 541859, 33211, 63201, + 212861, 306901, 572094, 974953, 683232, 371095, 944829, 842675, 4273, 778735, 38911, 337234, + 975956, 648772, 103573, 381675, 153332, 682242, 269472, 940261, 989084, 569925, 922990, 65745, + 713571, 952867, 631447, 352805, 671402, 188913, 111165) + + val params1 = Array(435446, 668235, 698906, 9965, 923490, 970342, 971528, 924912, 210063, 514387, + 185010, 316700, 201191, 129476, 186458, 120609, 55514, 88575, 125345, 580302, 615387) + val params2 = Array(63648, 770312, 344177, 328320, 126064, 636422, 7245, 327093, 906825, 45465, + 93499, 285349, 807082, 290182, 872723, 752484, 562808, 243877, 194831, 737899, 465701) + val params3 = Array(2379519, 653292, 2270272, 2037464, 5103522, 2137098, 6171405, 3826048, + 4459294, 3001100, 585675, 200651, 3877716, 1028514, 4392106, 4044019, 3246679, 993932, 444706, + 1008620, 958004) + + def getParam(iterCount: Int, arr: Array[Int]): Int = { + val index = if (iterCount < 0) 0 else iterCount % arr.length + arr(index) + } + + def addCaseWithCleanup( + benchmark: MultiThreadedBenchmark, + name: String, + numIters: Int = 0, + prepare: () => Unit, + cleanup: () => Unit, + testCleanup: () => Unit, + isMultithreaded: Boolean, + testPrepare: () => Unit = () => Unit)(f: (Int, Int) => Boolean): Unit = { + val timedF = (timer: Benchmark.Timer, threadId: Int) => { + if (!isMultithreaded) { + testPrepare() + timer.startTiming() + } + val ret = f(timer.iteration, threadId) + if (!isMultithreaded) { + testCleanup() + timer.stopTiming() + } + ret + } + benchmark.benchmarks += MultiThreadedBenchmark.Case(name, timedF, numIters, prepare, cleanup) + } +} diff --git a/cluster/src/test/scala/org/apache/spark/sql/store/SortedColumnTests.scala b/cluster/src/test/scala/org/apache/spark/sql/store/SortedColumnTests.scala new file mode 100644 index 0000000000..2e187624e6 --- /dev/null +++ b/cluster/src/test/scala/org/apache/spark/sql/store/SortedColumnTests.scala @@ -0,0 +1,956 @@ +/* + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package org.apache.spark.sql.store + +import java.io.File + +import scala.collection.mutable + +import io.snappydata.Property + +import org.apache.spark.{Logging, SparkConf} +import org.apache.spark.memory.SnappyUnifiedMemoryManager +import org.apache.spark.sql.SnappySession +import org.apache.spark.sql.execution.columnar.ColumnTableScan +import org.apache.spark.sql.{DataFrame, DataFrameReader, SnappySession} +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.snappy._ +import org.apache.spark.util.Benchmark + +/** + * Tests for column table having sorted columns. + */ +class SortedColumnTests extends ColumnTablesTestBase { + + override def beforeAll(): Unit = { + super.beforeAll() + stopAll() + } + + override def afterAll(): Unit = { + super.afterAll() + stopAll() + } + + override protected def newSparkConf(addOn: (SparkConf) => SparkConf): SparkConf = { + val conf = new SparkConf() + conf.setIfMissing("spark.master", "local[*]") + .setAppName(getClass.getName) + conf.set("snappydata.store.critical-heap-percentage", "95") + if (SnappySession.isEnterpriseEdition) { + conf.set("snappydata.store.memory-size", "1200m") + } + conf.set("spark.memory.manager", classOf[SnappyUnifiedMemoryManager].getName) + conf.set("spark.serializer", "org.apache.spark.serializer.PooledKryoSerializer") + conf.set("spark.closure.serializer", "org.apache.spark.serializer.PooledKryoSerializer") + conf + } + + test("basic insert") { + val snc = this.snc.snappySession + val colTableName = "colDeltaTable" + val numElements = 551 + val numBuckets = 2 + + SortedColumnTests.verfiyInsertDataExists(snc, numElements) + SortedColumnTests.verfiyUpdateDataExists(snc, numElements) + SortedColumnTests.testBasicInsert(snc, colTableName, numBuckets, numElements) + } + + test("basic insert 2") { + val snc = this.snc.snappySession + val colTableName = "colDeltaTable" + val numElements = 551 + val numBuckets = 1 + + SortedColumnTests.testBasicInsert2(snc, colTableName, numBuckets, numElements) + // Thread.sleep(50000000) + } + + test("basic delete 1") { + val snc = this.snc.snappySession + val colTableName = "colDeltaTable" + val numElements = 551 + SortedColumnTests.testBasicInsertWithDelete(snc, colTableName, numBuckets = 1, numElements) + SortedColumnTests.testBasicInsertWithDelete(snc, colTableName, numBuckets = 2, numElements) + // Thread.sleep(50000000) + } + + test("multiple insert") { + val snc = this.snc.snappySession + val colTableName = "colDeltaTable" + val numElements = 300 + SortedColumnTests.testMultipleInsert(snc, colTableName, numBuckets = 1, numElements) + SortedColumnTests.testMultipleInsert(snc, colTableName, numBuckets = 2, numElements) + } + + test("update and insert") { + val snc = this.snc.snappySession + val colTableName = "colDeltaTable" + val numElements = 300 + SortedColumnTests.testUpdateAndInsert(snc, colTableName, numBuckets = 1, numElements) + SortedColumnTests.testUpdateAndInsert(snc, colTableName, numBuckets = 2, numElements) + } + + test("update and insert 2") { + val snc = this.snc.snappySession + val colTableName = "colDeltaTable" + val numElements = 400 + SortedColumnTests.testUpdateAndInsert2(snc, colTableName, numBuckets = 1, numElements) + SortedColumnTests.testUpdateAndInsert2(snc, colTableName, numBuckets = 2, numElements) + } + + test("join query") { + val session = this.snc.snappySession + val colTableName = "colDeltaTable" + val joinTableName = "joinDeltaTable" + val numBuckets = 4 + + SortedColumnTests.testColocatedJoin(session, colTableName, joinTableName, numBuckets, + // numElements = 10000000, expectedResCount = 1000000000, // 100 million - TODO VB failing + numElements = 1000000, expectedResCount = 100000000, // 10 million + numTimesInsert = 10, numTimesUpdate = 10) + SortedColumnTests.testColocatedJoin(session, colTableName, joinTableName, numBuckets, + // numElements = 100000000, expectedResCount = 100000000) // 100 million - TODO VB failing + numElements = 10000000, expectedResCount = 10000000) // 10 million + // Thread.sleep(50000000) + } +} + +object SortedColumnTests extends Logging { + private val baseDataPath = s"/home/vivek/work/testData/local_index" + + def filePathInsert(size: Long, multiple: Int) : String = s"$baseDataPath/insert${size}_$multiple" + def verfiyInsertDataExists(snc: SnappySession, size: Long, multiple: Int = 1) : Unit = { + val dataDirInsert = new File(SortedColumnTests.filePathInsert(size, multiple)) + if (!dataDirInsert.exists()) { + dataDirInsert.mkdir() + snc.sql(s"create EXTERNAL TABLE insert_table_${size}_$multiple(id int, addr string," + + s" status boolean)" + + s" USING parquet OPTIONS(path '${SortedColumnTests.filePathInsert(size, multiple)}')") + var j = 0 + while (j < multiple) { + snc.range(size).filter(_ % 10 < 6).selectExpr("id", "concat('addr'," + + "cast(id as string))", + "case when (id % 2) = 0 then true else false end").write. + insertInto(s"insert_table_${size}_$multiple") + j += 1 + } + } + } + + def filePathUpdate(size: Long, multiple: Int) : String = s"$baseDataPath/update${size}_$multiple" + def verfiyUpdateDataExists(snc: SnappySession, size: Long, multiple: Int = 1) : Unit = { + val dataDirUpdate = new File(SortedColumnTests.filePathUpdate(size, multiple)) + if (!dataDirUpdate.exists()) { + dataDirUpdate.mkdir() + snc.sql(s"create EXTERNAL TABLE update_table_${size}_$multiple(id int, addr string," + + s" status boolean)" + + s" USING parquet OPTIONS(path '${SortedColumnTests.filePathUpdate(size, multiple)}')") + var j = 0 + while (j < multiple) { + snc.range(size).filter(_ % 10 > 5).selectExpr("id", "concat('addr'," + + "cast(id as string))", + "case when (id % 2) = 0 then true else false end").write. + insertInto(s"update_table_${size}_$multiple") + j += 1 + } + } + } + + def verifyTotalRows(session: SnappySession, columnTable: String, numElements: Long, + finalCall: Boolean, numTimesInsert: Int, numTimesUpdate: Int): Unit = { + val colDf = session.sql(s"select * from $columnTable") + // scalastyle:off + // println(s"verifyTotalRows = ${colDf.collect().length}") + // scalastyle:on + val dataFrameReader: DataFrameReader = session.read + val insDF = dataFrameReader.parquet(filePathInsert(numElements, numTimesInsert)) + val verifyDF = if (finalCall) { + insDF.union(dataFrameReader.parquet(filePathUpdate(numElements, numTimesUpdate))) + } else insDF + val resCount = colDf.except(verifyDF).count() + assert(resCount == 0, resCount) + } + + def createColumnTable(session: SnappySession, colTableName: String, numBuckets: Int, + numElements: Long, colocateTableName: Option[String] = None): Unit = { + dropColumnTable(session, colTableName) + val additionalString = if (colocateTableName.isDefined) { + s", COLOCATE_WITH '${colocateTableName.get}'" + } else "" + session.sql(s"create table $colTableName (id int, addr string, status boolean) " + + s"using column options(buckets '$numBuckets', partition_by 'id SORTING ASC'" + + additionalString + s")") + } + + def createColumnTable2(session: SnappySession, colTableName: String, numBuckets: Int, + numElements: Long, colocateTableName: Option[String] = None): Unit = { + dropColumnTable(session, colTableName) + val additionalString = if (colocateTableName.isDefined) { + s", COLOCATE_WITH '${colocateTableName.get}'" + } else "" + session.sql(s"create table $colTableName (id int, addr int, status int) " + + s"using column options(buckets '$numBuckets', partition_by 'id SORTING ASC'" + + additionalString + s")") + } + + def dropColumnTable(session: SnappySession, colTableName: String): Unit = { + session.sql(s"drop table if exists $colTableName") + } + + def testBasicInsert(session: SnappySession, colTableName: String, numBuckets: Int, + numElements: Long): Unit = { + session.conf.set(Property.ColumnMaxDeltaRows.name, "100") + session.conf.set(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "true") + session.conf.set(SQLConf.WHOLESTAGE_FALLBACK.key, "false") + + createColumnTable(session, colTableName, numBuckets, numElements) + val dataFrameReader : DataFrameReader = session.read + val insertDF : DataFrame = dataFrameReader.load(filePathInsert(numElements, multiple = 1)) + insertDF.write.insertInto(colTableName) + val updateDF : DataFrame = dataFrameReader.load(filePathUpdate(numElements, multiple = 1)) + + try { + verifyTotalRows(session: SnappySession, colTableName, numElements, finalCall = false, + numTimesInsert = 1, numTimesUpdate = 1) + updateDF.write.insertInto(colTableName) + verifyTotalRows(session: SnappySession, colTableName, numElements, finalCall = true, + numTimesInsert = 1, numTimesUpdate = 1) + } catch { + case t: Throwable => + logError(t.getMessage, t) + throw t + } + + // Disable verifying rows in sorted order + // def sorted(l: List[Row]) = l.isEmpty || + // l.view.zip(l.tail).forall(x => x._1.getInt(0) <= x._2.getInt(0)) + // assert(sorted(rs2.toList)) + + session.sql(s"drop table $colTableName") + session.conf.unset(Property.ColumnBatchSize.name) + session.conf.unset(Property.ColumnMaxDeltaRows.name) + session.conf.unset(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key) + session.conf.unset(SQLConf.WHOLESTAGE_FALLBACK.key) + } + + def testBasicInsert2(session: SnappySession, colTableName: String, numBuckets: Int, + numElements: Long): Unit = { + session.conf.set(Property.ColumnMaxDeltaRows.name, "100") + session.conf.set(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "true") + session.conf.set(SQLConf.WHOLESTAGE_FALLBACK.key, "false") + + val testName = "testBasicInsert2" + val dataFile_1 = s"${testName}_1" + SortedColumnTests.createFixedData2(session, numElements, dataFile_1)(i => { + i % 10 < 6 + }) + val dataFile_2 = s"${testName}_2" + SortedColumnTests.createFixedData2(session, numElements, dataFile_2)(i => { + i % 10 > 5 + }) + + def doIncrementalInsert(fileName: String, dataFrameReader: DataFrameReader): Unit = { + // scalastyle:off + println(s"$testName start loading $fileName") + // scalastyle:on + dataFrameReader.load(fixedFilePath(fileName)).write.insertInto(colTableName) + // scalastyle:off + println(s"$testName loaded $fileName") + // scalastyle:on + } + + def verifySelect(expectedCount: Int): Unit = { + val select_query = s"select * from $colTableName" + val colDf = session.sql(select_query) + val res = colDf.collect() + var i = 0 + res.foreach(r => { + val col0 = r.getInt(0) + val col1 = r.getInt(1) + val col2 = r.getInt(2) + // scalastyle:off + println(s"verifySelect-$expectedCount-$i [$col0 $col1 $col2]") + // scalastyle:on + i += 1 + }) + assert(i == expectedCount, s"$i : $expectedCount") + } + + try { + createColumnTable2(session, colTableName, numBuckets, numElements) + + // scalastyle:off + println(s"$testName start loading $dataFile_1") + // scalastyle:on + val dataFrameReader: DataFrameReader = session.read + dataFrameReader.load(fixedFilePath(dataFile_1)).write.insertInto(colTableName) + // scalastyle:off + println(s"$testName loaded $dataFile_1") + // scalastyle:on + + doIncrementalInsert(dataFile_2, dataFrameReader) + + // ColumnTableScan.setDebugMode(true) + verifySelect(numElements.toInt) + } catch { + case t: Throwable => + logError(t.getMessage, t) + throw t + } + + session.sql(s"drop table $colTableName") + session.conf.unset(Property.ColumnBatchSize.name) + session.conf.unset(Property.ColumnMaxDeltaRows.name) + session.conf.unset(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key) + session.conf.unset(SQLConf.WHOLESTAGE_FALLBACK.key) + } + + def testBasicInsertWithDelete(session: SnappySession, colTableName: String, numBuckets: Int, + numElements: Long): Unit = { + session.conf.set(Property.ColumnMaxDeltaRows.name, "100") + session.conf.set(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "true") + session.conf.set(SQLConf.WHOLESTAGE_FALLBACK.key, "false") + + val testName = "testBasicInsertWithDelete" + val dataFile_1 = s"${testName}_1" + SortedColumnTests.createFixedData2(session, numElements, dataFile_1)(i => { + i % 10 < 6 + }) + val dataFile_2 = s"${testName}_2" + SortedColumnTests.createFixedData2(session, numElements, dataFile_2)(i => { + i % 10 > 5 && i % 10 < 10 + }) + val dataFile_3 = s"${testName}_3" + SortedColumnTests.createFixedData2(session, numElements, dataFile_3)(i => { + i % 10 == 3 || i % 10 == 8 + }) + val expected = new mutable.HashSet[Int] + + + def doInsert(fileName: String, dataFrameReader: DataFrameReader): Unit = { + // scalastyle:off + println(s"$testName start loading $fileName") + // scalastyle:on + dataFrameReader.load(fixedFilePath(fileName)).write.insertInto(colTableName) + // scalastyle:off + println(s"$testName loaded $fileName") + // scalastyle:on + } + + def doIncrementalInsert(fileName: String, dataFrameReader: DataFrameReader): Unit = { + // scalastyle:off + println(s"$testName start loading $fileName") + // scalastyle:on + dataFrameReader.load(fixedFilePath(fileName)).write.insertInto(colTableName) + // scalastyle:off + println(s"$testName loaded $fileName") + // scalastyle:on + } + + def verifySelect(expectedCount: Int, doPrint: Boolean = false): Unit = { + val dataSet = expected.clone() + val dataSetSize = dataSet.size + // scalastyle:off + println(s"$testName started verifySelect $dataSetSize") + // scalastyle:on + val select_query = s"select * from $colTableName" + val colDf = session.sql(select_query) + val res = colDf.collect() + var i = 0 + res.foreach(r => { + val col0 = r.getInt(0) + val col1 = r.getInt(1) + val col2 = r.getInt(2) + if (doPrint) { + // scalastyle:off + println(s"verifySelect-$expectedCount-$i [$col0 $col1 $col2]") + // scalastyle:on + } + assert(dataSet.contains(col0)) + dataSet.remove(col0) + i += 1 + }) + // assert(i == expectedCount, s"$i : $expectedCount") + // assert(dataSet.isEmpty) + // scalastyle:off + println(s"$testName done verifySelect $dataSetSize") + // scalastyle:on + } + + def doDelete(whereClause: String = ""): Unit = { + val delete_query = s"delete from $colTableName where id in $whereClause" + // scalastyle:off + println(s"$testName started DELETE $delete_query") + // scalastyle:on + val upd = session.sql(delete_query) + // scalastyle:off + println(s"$testName done DELETE $delete_query") + // scalastyle:on + } + + try { + createColumnTable2(session, colTableName, numBuckets, numElements) + + val dataFrameReader: DataFrameReader = session.read + doInsert(dataFile_1, dataFrameReader) + (0 until numElements.toInt).filter(i => i % 10 < 6).foreach(i => expected.add(i)) + + var numDeletes1 = 1 + var deleteWhereCaluse1: StringBuilder = new StringBuilder("(3") + (10 to numElements.toInt).foreach(i => { + if (i % 10 == 3) { + deleteWhereCaluse1.append(s", $i") + numDeletes1 += 1 + } + }) + deleteWhereCaluse1.append(s")") + doDelete(deleteWhereCaluse1.result()) + (0 until numElements.toInt).filter(i => i % 10 == 3).foreach(i => expected.remove(i)) + + doIncrementalInsert(dataFile_2, dataFrameReader) + (0 until numElements.toInt).filter(i => i % 10 > 5 && i % 10 < 10). + foreach(i => expected.add(i)) + verifySelect(numElements.toInt - numDeletes1) + + var numDeletes2 = 1 + var deleteWhereCaluse2: StringBuilder = new StringBuilder("(8") + (10 to numElements.toInt).foreach(i => { + if (i % 10 == 8) { + deleteWhereCaluse2.append(s", $i") + numDeletes2 += 1 + } + }) + deleteWhereCaluse2.append(s")") + doDelete(deleteWhereCaluse2.result()) + (0 until numElements.toInt).filter(i => i % 10 == 8).foreach(i => expected.remove(i)) + verifySelect(numElements.toInt - numDeletes1 - numDeletes2) + + // ColumnTableScan.setDebugMode(true) + doIncrementalInsert(dataFile_3, dataFrameReader) + (0 until numElements.toInt).filter(i => i % 10 == 3 || i % 10 == 8). + foreach(i => expected.add(i)) + verifySelect(numElements.toInt, doPrint = false) + } catch { + case t: Throwable => + logError(t.getMessage, t) + throw t + } + + session.sql(s"drop table $colTableName") + session.conf.unset(Property.ColumnBatchSize.name) + session.conf.unset(Property.ColumnMaxDeltaRows.name) + session.conf.unset(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key) + session.conf.unset(SQLConf.WHOLESTAGE_FALLBACK.key) + } + + def fixedFilePath(fileName: String): String = s"$baseDataPath/$fileName" + + def createFixedData(snc: SnappySession, size: Long, fileName: String) + (f: (Long) => Boolean): Unit = { + val dataDir = new File(fixedFilePath(fileName)) + if (dataDir.exists()) { + def deleteRecursively(file: File): Unit = { + if (file.isDirectory) { + file.listFiles.foreach(deleteRecursively) + } + if (file.exists && !file.delete) { + throw new Exception(s"Unable to delete ${file.getAbsolutePath}") + } + } + deleteRecursively(dataDir) + } + dataDir.mkdir() + snc.sql(s"drop TABLE if exists insert_table_$fileName") + snc.sql(s"create EXTERNAL TABLE insert_table_$fileName(id int, addr string," + + s" status boolean)" + + s" USING parquet OPTIONS(path '${fixedFilePath(fileName)}')") + snc.range(size).filter(f(_)).selectExpr("id", "concat('addr'," + + "cast(id as string))", + "case when (id % 2) = 0 then true else false end").write. + insertInto(s"insert_table_$fileName") + } + + def createFixedData2(snc: SnappySession, size: Long, fileName: String) + (f: (Long) => Boolean): Unit = { + val dataDir = new File(fixedFilePath(fileName)) + if (dataDir.exists()) { + def deleteRecursively(file: File): Unit = { + if (file.isDirectory) { + file.listFiles.foreach(deleteRecursively) + } + if (file.exists && !file.delete) { + throw new Exception(s"Unable to delete ${file.getAbsolutePath}") + } + } + deleteRecursively(dataDir) + } + dataDir.mkdir() + snc.sql(s"drop TABLE if exists insert_table_$fileName") + snc.sql(s"create EXTERNAL TABLE insert_table_$fileName(id int, addr int," + + s" status int)" + + s" USING parquet OPTIONS(path '${fixedFilePath(fileName)}')") + snc.range(size).filter(f(_)).selectExpr("id", "10000", + "case when (id % 2) = 0 then 111111 else 222222 end").write. + insertInto(s"insert_table_$fileName") + } + + def testMultipleInsert(session: SnappySession, colTableName: String, numBuckets: Int, + numElements: Long): Unit = { + val testName = "testMultipleInsert" + val dataFile_1 = s"${testName}_1" + SortedColumnTests.createFixedData(session, numElements, dataFile_1)(i => { + i == 0 || i == 99 || i == 200 || i == 299 + }) + val dataFile_2 = s"${testName}_2" + SortedColumnTests.createFixedData(session, numElements, dataFile_2)(i => { + i == 100 || i == 199 + }) + val dataFile_3 = s"${testName}_3" + SortedColumnTests.createFixedData(session, numElements, dataFile_3)(i => { + i == 50 || i == 250 + }) + val dataFile_4 = s"${testName}_4" + SortedColumnTests.createFixedData(session, numElements, dataFile_4)(i => { + i == 25 || i == 175 + }) + val dataFile_5 = s"${testName}_5" + SortedColumnTests.createFixedData(session, numElements, dataFile_5)(i => { + i == 125 || i == 275 + }) + val dataFile_6 = s"${testName}_6" + SortedColumnTests.createFixedData(session, numElements, dataFile_6)(i => { + i == 150 || i == 225 + }) + + session.conf.set(Property.ColumnMaxDeltaRows.name, "100") + session.conf.set(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "true") + session.conf.set(SQLConf.WHOLESTAGE_FALLBACK.key, "false") + + try { + createColumnTable(session, colTableName, numBuckets, numElements) + val dataFrameReader : DataFrameReader = session.read + dataFrameReader.load(fixedFilePath(dataFile_1)).write.insertInto(colTableName) + // scalastyle:off + println(s"$testName loaded $dataFile_1") + // scalastyle:on + + dataFrameReader.load(fixedFilePath(dataFile_2)).write.insertInto(colTableName) + // scalastyle:off + println(s"$testName loaded $dataFile_2") + // scalastyle:on + dataFrameReader.load(fixedFilePath(dataFile_3)).write.insertInto(colTableName) + // scalastyle:off + println(s"$testName loaded $dataFile_3") + // scalastyle:on + dataFrameReader.load(fixedFilePath(dataFile_4)).write.insertInto(colTableName) + // scalastyle:off + println(s"$testName loaded $dataFile_4") + // scalastyle:on + dataFrameReader.load(fixedFilePath(dataFile_5)).write.insertInto(colTableName) + // scalastyle:off + println(s"$testName loaded $dataFile_5") + // scalastyle:on + dataFrameReader.load(fixedFilePath(dataFile_6)).write.insertInto(colTableName) + // scalastyle:off + println(s"$testName loaded $dataFile_6") + // scalastyle:on + + val colDf = session.sql(s"select * from $colTableName") + val res = colDf.collect() + val expected = Array(0, 25, 50, 99, 100, 125, 150, 175, 199, 200, 225, 250, 275, 299) + assert(res.length == expected.length) + // scalastyle:off + // println(s"verifyTotalRows = ${colDf.collect().length}") + // scalastyle:on + if (numBuckets == 1) { + var i = 0 + res.foreach(r => { + val col1 = r.getInt(0) + assert(col1 == expected(i), s"$i : $col1") + i += 1 + }) + } + } catch { + case t: Throwable => + logError(t.getMessage, t) + throw t + } + + // Disable verifying rows in sorted order + // def sorted(l: List[Row]) = l.isEmpty || + // l.view.zip(l.tail).forall(x => x._1.getInt(0) <= x._2.getInt(0)) + // assert(sorted(rs2.toList)) + + session.sql(s"drop table $colTableName") + session.conf.unset(Property.ColumnBatchSize.name) + session.conf.unset(Property.ColumnMaxDeltaRows.name) + session.conf.unset(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key) + session.conf.unset(SQLConf.WHOLESTAGE_FALLBACK.key) + } + + def testUpdateAndInsert(session: SnappySession, colTableName: String, numBuckets: Int, + numElements: Long): Unit = { + val testName = "testUpdateAndInsert" + val dataFile_1 = s"${testName}_1" + SortedColumnTests.createFixedData(session, numElements, dataFile_1)(i => { + i == 0 || i == 99 || i == 200 || i == 299 + }) + val dataFile_2 = s"${testName}_2" + SortedColumnTests.createFixedData(session, numElements, dataFile_2)(i => { + i == 100 || i == 199 + }) + val dataFile_3 = s"${testName}_3" + SortedColumnTests.createFixedData(session, numElements, dataFile_3)(i => { + i == 50 || i == 250 + }) + val dataFile_4 = s"${testName}_4" + SortedColumnTests.createFixedData(session, numElements, dataFile_4)(i => { + i == 25 || i == 175 + }) + val dataFile_5 = s"${testName}_5" + SortedColumnTests.createFixedData(session, numElements, dataFile_5)(i => { + i == 125 || i == 275 + }) + val dataFile_6 = s"${testName}_6" + SortedColumnTests.createFixedData(session, numElements, dataFile_6)(i => { + i == 150 || i == 225 + }) + + session.conf.set(Property.ColumnMaxDeltaRows.name, "100") + session.conf.set(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "true") + session.conf.set(SQLConf.WHOLESTAGE_FALLBACK.key, "false") + + def doUpdate(queryStr: String, whereClause: String = ""): String = { + val update_query = s"update $colTableName set addr = '$queryStr' $whereClause" + // scalastyle:off + println(s"$testName started UPDATE $update_query") + // scalastyle:on + val upd = session.sql(update_query) + // scalastyle:off + println(s"$testName done UPDATE $update_query") + // scalastyle:on + queryStr + } + + def doIncrementalInsert(fileName: String, dataFrameReader: DataFrameReader): Unit = { + // scalastyle:off + println(s"$testName start loading $fileName") + // scalastyle:on + dataFrameReader.load(fixedFilePath(fileName)).write.insertInto(colTableName) + // scalastyle:off + println(s"$testName loaded $fileName") + // scalastyle:on + } + + def verifyUpdate(expected: String, expectedCount: Int): Unit = { + val select_query = s"select * from $colTableName" + val colDf = session.sql(select_query) + val res = colDf.collect() + var i = 0 + res.foreach(r => { + val col0 = r.getInt(0) + val col1 = r.getString(1) + // scalastyle:off + println(s"verifyUpdate-$expected-$expectedCount $col0 $col1") + // scalastyle:on + assert(col1.equalsIgnoreCase(expected), s"$col1 : $expected") + i += 1 + }) + assert(i == expectedCount, s"$i : $expectedCount") + } + + try { + createColumnTable(session, colTableName, numBuckets, numElements) + + // scalastyle:off + println(s"$testName start loading $dataFile_1") + // scalastyle:on + val dataFrameReader: DataFrameReader = session.read + dataFrameReader.load(fixedFilePath(dataFile_1)).write.insertInto(colTableName) + // scalastyle:off + println(s"$testName loaded $dataFile_1") + // scalastyle:on + verifyUpdate(doUpdate("updated1"), 4) + + doIncrementalInsert(dataFile_2, dataFrameReader) + verifyUpdate(doUpdate("updated2"), 6) + + doIncrementalInsert(dataFile_3, dataFrameReader) + verifyUpdate(doUpdate("updated3"), 8) + + doIncrementalInsert(dataFile_4, dataFrameReader) + verifyUpdate(doUpdate("updated4"), 10) + + doIncrementalInsert(dataFile_5, dataFrameReader) + verifyUpdate(doUpdate("updated5"), 12) + + doIncrementalInsert(dataFile_6, dataFrameReader) + verifyUpdate(doUpdate("updated6"), 14) + + val select_query = s"select * from $colTableName" + // scalastyle:off + println(s"$testName started SELECT $select_query") + // scalastyle:on + val colDf = session.sql(select_query) + val res = colDf.collect() + val expected = Array(0, 25, 50, 99, 100, 125, 150, 175, 199, 200, 225, 250, 275, 299) + assert(res.length == expected.length, s"output: ${res.length}, expected=${expected.length}") + // scalastyle:off + println(s"$testName SELECT = ${res.length} / ${expected.length}") + // scalastyle:on + if (numBuckets == 1) { + var i = 0 + res.foreach(r => { + val col1 = r.getInt(0) + assert(col1 == expected(i), s"$i: output: $col1, expected=${expected(i)}") + i += 1 + }) + } + } catch { + case t: Throwable => + logError(t.getMessage, t) + throw t + } + + // Disable verifying rows in sorted order + // def sorted(l: List[Row]) = l.isEmpty || + // l.view.zip(l.tail).forall(x => x._1.getInt(0) <= x._2.getInt(0)) + // assert(sorted(rs2.toList)) + + session.sql(s"drop table $colTableName") + session.conf.unset(Property.ColumnBatchSize.name) + session.conf.unset(Property.ColumnMaxDeltaRows.name) + session.conf.unset(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key) + session.conf.unset(SQLConf.WHOLESTAGE_FALLBACK.key) + } + + def testUpdateAndInsert2(session: SnappySession, colTableName: String, numBuckets: Int, + numElements: Long): Unit = { + val testName = "testUpdateAndInsert" + val dataFile_1 = s"${testName}_1" + SortedColumnTests.createFixedData2(session, numElements, dataFile_1)(i => { + i == 0 || i == 99 || i == 200 || i == 299 + }) + val dataFile_2 = s"${testName}_2" + SortedColumnTests.createFixedData2(session, numElements, dataFile_2)(i => { + i == 100 || i == 199 + }) + val dataFile_3 = s"${testName}_3" + SortedColumnTests.createFixedData2(session, numElements, dataFile_3)(i => { + i == 50 || i == 250 + }) + val dataFile_4 = s"${testName}_4" + SortedColumnTests.createFixedData2(session, numElements, dataFile_4)(i => { + i == 25 || i == 175 + }) + val dataFile_5 = s"${testName}_5" + SortedColumnTests.createFixedData2(session, numElements, dataFile_5)(i => { + i == 125 || i == 275 + }) + val dataFile_6 = s"${testName}_6" + SortedColumnTests.createFixedData2(session, numElements, dataFile_6)(i => { + i == 150 || i == 225 + }) + + session.conf.set(Property.ColumnMaxDeltaRows.name, "100") + session.conf.set(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "true") + session.conf.set(SQLConf.WHOLESTAGE_FALLBACK.key, "false") + + def doUpdate(queryStr: Int, whereClause: String = ""): Int = { + val update_query = s"update $colTableName set addr = '$queryStr' $whereClause" + // scalastyle:off + println(s"$testName started UPDATE $update_query") + // scalastyle:on + val upd = session.sql(update_query) + // scalastyle:off + println(s"$testName done UPDATE $update_query") + // scalastyle:on + queryStr + } + + def doIncrementalInsert(fileName: String, dataFrameReader: DataFrameReader): Unit = { + // scalastyle:off + println(s"$testName start loading $fileName") + // scalastyle:on + dataFrameReader.load(fixedFilePath(fileName)).write.insertInto(colTableName) + // scalastyle:off + println(s"$testName loaded $fileName") + // scalastyle:on + } + + def verifySelect(expectedCount: Int): Unit = { + val select_query = s"select * from $colTableName" + val colDf = session.sql(select_query) + val res = colDf.collect() + var i = 0 + res.foreach(r => { + val col0 = r.getInt(0) + val col1 = r.getInt(1) + // scalastyle:off + println(s"verifySelect-$expectedCount $col0 $col1") + // scalastyle:on + i += 1 + }) + assert(i == expectedCount, s"$i : $expectedCount") + } + + def verifyUpdate(expected: Int, expectedCount: Int): Unit = { + val select_query = s"select * from $colTableName" + val colDf = session.sql(select_query) + val res = colDf.collect() + var i = 0 + res.foreach(r => { + val col0 = r.getInt(0) + val col1 = r.getInt(1) + // scalastyle:off + println(s"verifyUpdate-$expected-$expectedCount $col0 $col1") + // scalastyle:on + assert(col1 == expected, s"$col1 : $expected") + i += 1 + }) + assert(i == expectedCount, s"$i : $expectedCount") + } + + try { + createColumnTable2(session, colTableName, numBuckets, numElements) + + // scalastyle:off + println(s"$testName start loading $dataFile_1") + // scalastyle:on + val dataFrameReader: DataFrameReader = session.read + dataFrameReader.load(fixedFilePath(dataFile_1)).write.insertInto(colTableName) + // scalastyle:off + println(s"$testName loaded $dataFile_1") + // scalastyle:on + + verifySelect(4) + verifyUpdate(doUpdate(10001), 4) + + doIncrementalInsert(dataFile_2, dataFrameReader) + verifySelect(6) + verifyUpdate(doUpdate(10002), 6) + + doIncrementalInsert(dataFile_3, dataFrameReader) + verifySelect(8) + verifyUpdate(doUpdate(10003), 8) + + doIncrementalInsert(dataFile_4, dataFrameReader) + verifySelect(10) + verifyUpdate(doUpdate(10004), 10) + + doIncrementalInsert(dataFile_5, dataFrameReader) + verifySelect(12) + verifyUpdate(doUpdate(10005), 12) + + doIncrementalInsert(dataFile_6, dataFrameReader) + verifySelect(14) + verifyUpdate(doUpdate(10006), 14) + + val select_query = s"select * from $colTableName" + // scalastyle:off + println(s"$testName started SELECT $select_query") + // scalastyle:on + + val colDf = session.sql(select_query) + val res = colDf.collect() + val expected = Array(0, 25, 50, 99, 100, 125, 150, 175, 199, 200, 225, 250, 275, 299) + assert(res.length == expected.length, s"output: ${res.length}, expected=${expected.length}") + // scalastyle:off + println(s"$testName SELECT = ${res.length} / ${expected.length}") + // scalastyle:on + if (numBuckets == 1) { + var i = 0 + res.foreach(r => { + val col1 = r.getInt(0) + assert(col1 == expected(i), s"$i: output: $col1, expected=${expected(i)}") + i += 1 + }) + } + } catch { + case t: Throwable => + logError(t.getMessage, t) + throw t + } + + // Disable verifying rows in sorted order + // def sorted(l: List[Row]) = l.isEmpty || + // l.view.zip(l.tail).forall(x => x._1.getInt(0) <= x._2.getInt(0)) + // assert(sorted(rs2.toList)) + + session.sql(s"drop table $colTableName") + session.conf.unset(Property.ColumnBatchSize.name) + session.conf.unset(Property.ColumnMaxDeltaRows.name) + session.conf.unset(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key) + session.conf.unset(SQLConf.WHOLESTAGE_FALLBACK.key) + } + + def testColocatedJoin(session: SnappySession, colTableName: String, joinTableName: String, + numBuckets: Int, numElements: Long, expectedResCount: Int, numTimesInsert: Int = 1, + numTimesUpdate: Int = 1): Unit = { + val totalElements = (numElements * 0.6 * numTimesUpdate + + numElements * 0.4 * numTimesUpdate).toLong + SortedColumnTests.verfiyInsertDataExists(session, numElements, numTimesInsert) + SortedColumnTests.verfiyUpdateDataExists(session, numElements, numTimesUpdate) + val dataFrameReader : DataFrameReader = session.read + val insertDF: DataFrame = dataFrameReader.load(SortedColumnTests.filePathInsert(numElements, + numTimesInsert)) + val updateDF: DataFrame = dataFrameReader.load(SortedColumnTests.filePathUpdate(numElements, + numTimesUpdate)) + + SortedColumnTests.createColumnTable(session, colTableName, numBuckets, numElements) + SortedColumnTests.createColumnTable(session, joinTableName, numBuckets, numElements, + Some(colTableName)) + try { + session.conf.set(Property.ColumnBatchSize.name, "24M") // default + session.conf.set(Property.ColumnMaxDeltaRows.name, "100") + insertDF.write.insertInto(colTableName) + insertDF.write.insertInto(joinTableName) + + updateDF.write.insertInto(colTableName) + updateDF.write.insertInto(joinTableName) + } finally { + session.conf.unset(Property.ColumnBatchSize.name) + session.conf.unset(Property.ColumnMaxDeltaRows.name) + } + + try { + // Force SMJ + session.conf.set(Property.HashJoinSize.name, "-1") + session.conf.set(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key, "-1") + val query = s"select AVG(A.id), COUNT(B.id) " + + s" from $colTableName A inner join $joinTableName B where A.id = B.id" + val result = session.sql(query).collect() + // scalastyle:off + println(s"Query = $query result=${result.length}") + result.foreach(r => { + val avg = r.getDouble(0) + val count = r.getLong(1) + println(s"[$avg, $count], ") + assert(count == expectedResCount) + }) + // scalastyle:on + } finally { + session.sql(s"drop TABLE if exists $joinTableName") + session.sql(s"drop TABLE if exists $colTableName") + session.conf.unset(Property.HashJoinSize.name) + session.conf.unset(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key) + } + } +} diff --git a/core/src/main/java/io/snappydata/impl/SnappyHiveCatalog.java b/core/src/main/java/io/snappydata/impl/SnappyHiveCatalog.java index b5f9a78020..926861bf33 100644 --- a/core/src/main/java/io/snappydata/impl/SnappyHiveCatalog.java +++ b/core/src/main/java/io/snappydata/impl/SnappyHiveCatalog.java @@ -424,7 +424,7 @@ public Object call() throws Exception { Utils.toUpperCase(table.getDbName()), tableType, null, -1, -1, null, null, null, null, - tblDataSourcePath, driverClass); + tblDataSourcePath, "", driverClass); metaData.provider = table.getParameters().get( SnappyStoreHiveCatalog.HIVE_PROVIDER()); metaData.shortProvider = SnappyContext.getProviderShortName(metaData.provider); @@ -496,11 +496,29 @@ public Object call() throws Exception { value = parameters.get(ExternalStoreUtils.DEPENDENT_RELATIONS()); String[] dependentRelations = value != null ? value.toString().split(",") : null; - int columnBatchSize = ExternalStoreUtils.sizeAsBytes(parameters.get( - ExternalStoreUtils.COLUMN_BATCH_SIZE()), ExternalStoreUtils.COLUMN_BATCH_SIZE()); - int columnMaxDeltaRows = ExternalStoreUtils.checkPositiveNum(Integer.parseInt( - parameters.get(ExternalStoreUtils.COLUMN_MAX_DELTA_ROWS())), - ExternalStoreUtils.COLUMN_MAX_DELTA_ROWS()); + final int columnBatchSize; + String columnBatchSizeStr = parameters.get(ExternalStoreUtils.COLUMN_BATCH_SIZE()); + if (columnBatchSizeStr != null) { + columnBatchSize = ExternalStoreUtils.sizeAsBytes(columnBatchSizeStr, + ExternalStoreUtils.COLUMN_BATCH_SIZE()); + } else { + columnBatchSize = -1; + } + final int columnMaxDeltaRows; + String columnMaxDeltaRowsStr = parameters.get(ExternalStoreUtils.COLUMN_MAX_DELTA_ROWS()); + if (columnMaxDeltaRowsStr != null) { + columnMaxDeltaRows = ExternalStoreUtils.checkPositiveNum(Integer. + parseInt(columnMaxDeltaRowsStr), ExternalStoreUtils.COLUMN_MAX_DELTA_ROWS()); + } else { + columnMaxDeltaRows = -1; + } + final String columnBatchSorting; + String columnBatchSortingStr = parameters.get(StoreUtils.COLUMN_BATCH_SORTED()); + if (columnBatchSortingStr != null) { + columnBatchSorting = columnBatchSortingStr; + } else { + columnBatchSorting = ""; + } value = parameters.get(ExternalStoreUtils.COMPRESSION_CODEC()); String compressionCodec = value == null ? Constant.DEFAULT_CODEC() : value.toString(); String tableType = ExternalTableType.getTableType(table); @@ -521,6 +539,7 @@ public Object call() throws Exception { dmls, dependentRelations, tblDataSourcePath, + columnBatchSorting, driverClass); case CLOSE_HMC: diff --git a/core/src/main/scala/org/apache/spark/sql/SnappyStrategies.scala b/core/src/main/scala/org/apache/spark/sql/SnappyStrategies.scala index 3eb55b9612..8a2ddc177d 100644 --- a/core/src/main/scala/org/apache/spark/sql/SnappyStrategies.scala +++ b/core/src/main/scala/org/apache/spark/sql/SnappyStrategies.scala @@ -16,11 +16,11 @@ */ package org.apache.spark.sql -import scala.annotation.tailrec import scala.util.control.NonFatal import io.snappydata.Property +import org.apache.spark.rdd.ZippedPartitionsRDD2 import org.apache.spark.sql.JoinStrategy._ import org.apache.spark.sql.catalyst.analysis import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, AggregateFunction, Complete, Final, ImperativeAggregate, Partial, PartialMerge} @@ -28,17 +28,18 @@ import org.apache.spark.sql.catalyst.expressions.{Alias, Expression, NamedExpres import org.apache.spark.sql.catalyst.planning.{ExtractEquiJoinKeys, PhysicalAggregation} import org.apache.spark.sql.catalyst.plans.logical.{Join, LogicalPlan, ReturnAnswer} import org.apache.spark.sql.catalyst.plans.physical.{ClusteredDistribution, HashPartitioning} -import org.apache.spark.sql.catalyst.plans.{ExistenceJoin, Inner, JoinType, LeftAnti, LeftOuter, LeftSemi, RightOuter} +import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.collection.Utils import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.aggregate.{AggUtils, CollectAggregateExec, SnappyHashAggregateExec} -import org.apache.spark.sql.execution.columnar.ExternalStoreUtils +import org.apache.spark.sql.execution.columnar.impl.ColumnarStorePartitionedRDD +import org.apache.spark.sql.execution.columnar.{ColumnTableScan, DeltaInsertExec, DirectInsertExec, ExternalStoreUtils} import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.execution.exchange.{EnsureRequirements, Exchange, ShuffleExchange} -import org.apache.spark.sql.execution.joins.{BuildLeft, BuildRight} +import org.apache.spark.sql.execution.joins.{BuildLeft, BuildRight, SortMergeJoinExec} import org.apache.spark.sql.execution.sources.PhysicalScan -import org.apache.spark.sql.internal.{DefaultPlanner, JoinQueryPlanning, SQLConf} +import org.apache.spark.sql.internal.{DefaultPlanner, DeltaInsertNode, JoinQueryPlanning, SQLConf} import org.apache.spark.sql.streaming._ /** @@ -75,6 +76,19 @@ private[sql] trait SnappyStrategies { } } + object DeltaInsertOnSortMergeJoinStrategies extends Strategy { + def apply(plan: LogicalPlan): Seq[SparkPlan] = if (isDisabled) { + Nil + } else { + plan match { + case DeltaInsertNode(child, isDirectInsert) => + if (isDirectInsert) DirectInsertExec(planLater(child)) :: Nil + else DeltaInsertExec(planLater(child)) :: Nil + case _ => Nil + } + } + } + object HashJoinStrategies extends Strategy with JoinQueryPlanning { def apply(plan: LogicalPlan): Seq[SparkPlan] = if (isDisabled) { Nil diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnInsertExec.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnInsertExec.scala index 360c3d44e0..4cfb354ddd 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnInsertExec.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnInsertExec.scala @@ -23,12 +23,14 @@ import io.snappydata.{Constant, Property} import org.apache.spark.TaskContext import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode, GenerateUnsafeProjection} -import org.apache.spark.sql.catalyst.expressions.{Attribute, BoundReference, Expression, Literal} +import org.apache.spark.sql.catalyst.expressions.{Attribute, BoundReference, Expression, Literal, SortOrder} import org.apache.spark.sql.catalyst.util.{SerializedArray, SerializedMap, SerializedRow} import org.apache.spark.sql.collection.Utils import org.apache.spark.sql.execution.columnar.encoding.{BitSet, ColumnEncoder, ColumnEncoding, ColumnStatsSchema} +import org.apache.spark.sql.execution.columnar.impl.ColumnFormatRelation import org.apache.spark.sql.execution.{SparkPlan, TableExec} import org.apache.spark.sql.sources.DestroyRelation +import org.apache.spark.sql.store.StoreUtils import org.apache.spark.sql.store.CompressionCodecId import org.apache.spark.sql.types._ import org.apache.spark.util.TaskCompletionListener @@ -88,6 +90,26 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], override protected def opType: String = "Inserted" override protected def isInsert: Boolean = true + private val isColumnBatchSorted: Boolean = relation.isDefined && (relation.get match { + case cfr: ColumnFormatRelation => + StoreUtils.isColumnBatchSortedAscending(cfr.columnSortedOrder) || + StoreUtils.isColumnBatchSortedDescending(cfr.columnSortedOrder) + case _ => false + }) + + // Require per-partition sort on partitioning column + override def requiredChildOrdering: Seq[Seq[SortOrder]] = if (isColumnBatchSorted + && partitionExpressions.nonEmpty) { + // Seq(Seq(StoreUtils.getColumnUpdateDeleteOrdering(partitionExpressions.head.toAttribute))) + // For partitionColumns find the matching child columns + val schema = tableSchema + val childOutput = child.output + // for inserts the column names can be different and need to match + // by index else search in child output by name + val childPartitioningAttributes = partitionColumns.map(partColumn => + childOutput(schema.indexWhere(_.name.equalsIgnoreCase(partColumn)))) + Seq(childPartitioningAttributes.map(cpa => StoreUtils.getColumnUpdateDeleteOrdering(cpa))) + } else super.requiredChildOrdering /** Frequency of rows to check for total size exceeding batch size. */ private val (checkFrequency, checkMask) = { @@ -571,7 +593,7 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], | $batchSizeTerm, $buffers, $statsRow.getBytes(), null); | $externalStoreTerm.storeColumnBatch($tableName, $columnBatch, | $partitionIdCode, $batchUUID.longValue(), $maxDeltaRowsTerm, - | ${compressionCodec.id}, $conn); + | ${compressionCodec.id}, $isColumnBatchSorted, $conn); | $numInsertions += $batchSizeTerm; |} """.stripMargin) @@ -708,7 +730,7 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], | $batchSizeTerm, $buffers, $statsRow.getBytes(), null); | $externalStoreTerm.storeColumnBatch($tableName, $columnBatch, | $partitionIdCode, $batchUUID.longValue(), $maxDeltaRowsTerm, - | ${compressionCodec.id}, $conn); + | ${compressionCodec.id}, $isColumnBatchSorted, $conn); | $numInsertions += $batchSizeTerm; |} """.stripMargin) diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnPutIntoExec.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnPutIntoExec.scala index ab05ccd29f..ec131be7cf 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnPutIntoExec.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnPutIntoExec.scala @@ -23,7 +23,7 @@ import org.apache.spark.sql.execution.{BinaryExecNode, SparkPlan} import org.apache.spark.sql.types.LongType -case class ColumnPutIntoExec(insertPlan: SparkPlan, +abstract class BaseColumnPutIntoExec(insertPlan: SparkPlan, updatePlan: SparkPlan) extends BinaryExecNode { override lazy val output: Seq[Attribute] = AttributeReference( @@ -50,3 +50,10 @@ case class ColumnPutIntoExec(insertPlan: SparkPlan, Array(resultRow) } } + +case class ColumnPutIntoExec(insertPlan: SparkPlan, updatePlan: SparkPlan) extends + BaseColumnPutIntoExec(insertPlan, updatePlan) + +case class ColumnTableInsertExec(insertPlan: SparkPlan, updatePlan: SparkPlan) extends + BaseColumnPutIntoExec(insertPlan, updatePlan) { +} diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnTableScan.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnTableScan.scala index dca6423166..0a455e2c4f 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnTableScan.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnTableScan.scala @@ -46,10 +46,10 @@ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode, ExpressionCanonicalizer} -import org.apache.spark.sql.collection.Utils +import org.apache.spark.sql.collection.{ToolsCallbackInit, Utils} import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.columnar.encoding._ -import org.apache.spark.sql.execution.columnar.impl.{BaseColumnFormatRelation, ColumnDelta} +import org.apache.spark.sql.execution.columnar.impl.{BaseColumnFormatRelation, ColumnDelta, ColumnFormatRelation} import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} import org.apache.spark.sql.execution.row.{ResultSetDecoder, ResultSetTraversal, UnsafeRowDecoder, UnsafeRowHolder} import org.apache.spark.sql.sources.BaseRelation @@ -84,8 +84,25 @@ private[sql] final case class ColumnTableScan( override val nodeName: String = "ColumnTableScan" @transient private val MAX_SCHEMA_LENGTH = 40 + val (isColumnBatchSorted, ascendingOrder): (Boolean, Boolean) = baseRelation match { + case cfr: ColumnFormatRelation => + val isAscending = StoreUtils.isColumnBatchSortedAscending(cfr.columnSortedOrder) + (isAscending || StoreUtils.isColumnBatchSortedDescending(cfr.columnSortedOrder), isAscending) + case _ => (false, false) + } - override lazy val outputOrdering: Seq[SortOrder] = { + override lazy val outputOrdering: Seq[SortOrder] = if (isColumnBatchSorted) { + val buffer = new ArrayBuffer[SortOrder](partitionColumns.size) + if (ascendingOrder) { + partitionColumns.map(buffer += SortOrder(_, Ascending)) + } else { + partitionColumns.map(buffer += SortOrder(_, Descending)) + } + // TODO VB: To meet requirement of ColumnFormatIterator to be used in sorted mode only in case + // of SMJ, Order By or delta insert, need a flag in this calss that can be set true from here. + // (This will only be called if sorted output is required) + buffer + } else { val buffer = new ArrayBuffer[SortOrder](2) // sorted on [batchId, ordinal (position within batch)] for update/delete output.foreach { @@ -261,9 +278,12 @@ private[sql] final case class ColumnTableScan( val numFullRows = s"${batch}NumFullRows" val numDeltaRows = s"${batch}NumDeltaRows" val batchIndex = s"${batch}Index" + val batchDictionaryIndex = s"${batch}DictionaryIndex" val buffers = s"${batch}Buffers" val numRows = ctx.freshName("numRows") val batchOrdinal = ctx.freshName("batchOrdinal") + val lastRowFromDictionary = ctx.freshName("lastRowFromDictionary") + val isDeletedEntry = ctx.freshName("isDeletedEntry") val deletedDecoder = s"${batch}Deleted" val deletedDecoderLocal = s"${deletedDecoder}Local" var deletedDeclaration = "" @@ -274,8 +294,11 @@ private[sql] final case class ColumnTableScan( ctx.addMutableState("java.nio.ByteBuffer", buffers, "") ctx.addMutableState("int", numBatchRows, "") ctx.addMutableState("int", batchIndex, "") + ctx.addMutableState("int", batchDictionaryIndex, "") ctx.addMutableState(deletedDecoderClass, deletedDecoder, "") ctx.addMutableState("int", deletedCount, "") + ctx.addMutableState("boolean", lastRowFromDictionary, s"") + ctx.addMutableState("boolean", isDeletedEntry, s"") // need DataType and nullable to get decoder in generated code // shipping as StructType for efficient serialization @@ -416,10 +439,12 @@ private[sql] final case class ColumnTableScan( if (!isWideSchema) { genCodeColumnBuffer(ctx, decoderLocal, updatedDecoderLocal, decoder, updatedDecoder, - bufferVar, batchOrdinal, numNullsVar, attr, weightVarName) + bufferVar, batchOrdinal, numNullsVar, attr, weightVarName, lastRowFromDictionary, + batchDictionaryIndex, isDeletedEntry) } else { val ev = genCodeColumnBuffer(ctx, decoder, updatedDecoder, decoder, updatedDecoder, - bufferVar, batchOrdinal, numNullsVar, attr, weightVarName) + bufferVar, batchOrdinal, numNullsVar, attr, weightVarName, lastRowFromDictionary, + batchDictionaryIndex, isDeletedEntry) convertExprToMethodCall(ctx, ev, attr, index, batchOrdinal) } } @@ -454,8 +479,13 @@ private[sql] final case class ColumnTableScan( s"$deletedDecoder = $colInput.getDeletedColumnDecoder();$incrementDeletedBatchCount\n") deletedDeclaration = s"final $deletedDecoderClass $deletedDecoderLocal = $deletedDecoder;\n" - deletedCheck = s"if ($deletedDecoderLocal != null && " + - s"$deletedDecoderLocal.deleted($batchOrdinal)) continue;" + if (isColumnBatchSorted) { + deletedCheck = s"$isDeletedEntry = ($deletedDecoderLocal != null && " + + s"$deletedDecoderLocal.deleted($batchOrdinal));" + } else { + deletedCheck = s"if ($deletedDecoderLocal != null && " + + s"$deletedDecoderLocal.deleted($batchOrdinal)) continue;" + } } if (isWideSchema) { @@ -502,6 +532,9 @@ private[sql] final case class ColumnTableScan( int $numDeltaRows = $deltaStatsRow != null ? $deltaStatsRow.getInt( $countIndexInSchema) : 0; $numBatchRows = $numFullRows + $numDeltaRows; + // TODO VB: Remove this. For debugging + // System.out.println("VB: ColumnTableScan numBatchRows=" + $numBatchRows + + // " ,numFullRows=" + $numFullRows + " ,numDeltaRows=" + $numDeltaRows); // TODO: don't have the update count here (only insert count) $numDeltaRows = $numBatchRows; $incrementBatchCount @@ -577,6 +610,9 @@ private[sql] final case class ColumnTableScan( | $columnBufferInit | } | $batchIndex = 0; + | $batchDictionaryIndex = 0; + | $lastRowFromDictionary = false; + | $isDeletedEntry = false; | return true; |} """.stripMargin) @@ -624,6 +660,13 @@ private[sql] final case class ColumnTableScan( | final int $numRows = $numBatchRows$deletedCountCheck; | for (int $batchOrdinal = $batchIndex; $batchOrdinal < $numRows; | $batchOrdinal++) { + | if ($isColumnBatchSorted) { + | if ($lastRowFromDictionary) { + | $batchDictionaryIndex++; + | $lastRowFromDictionary = false; + | } + | $isDeletedEntry = false; + | } | $deletedCheck | $assignOrdinalId | $consumeCode @@ -658,11 +701,17 @@ private[sql] final case class ColumnTableScan( } } + // scalastyle:off private def genCodeColumnBuffer(ctx: CodegenContext, decoder: String, updateDecoder: String, decoderGlobal: String, mutableDecoderGlobal: String, buffer: String, batchOrdinal: String, - numNullsVar: String, attr: Attribute, weightVar: String): ExprCode = { - val nonNullPosition = if (attr.nullable) s"$batchOrdinal - $numNullsVar" else batchOrdinal + numNullsVar: String, attr: Attribute, weightVar: String, lastRowFromDictionary: String, + batchDictionaryIndex: String, isDeletedEntry: String): ExprCode = { + // scalastyle:on + val nonNullPosition = if (isColumnBatchSorted) { + if (attr.nullable) s"$batchDictionaryIndex - $numNullsVar" else s"$batchDictionaryIndex" + } else if (attr.nullable) s"$batchOrdinal - $numNullsVar" else batchOrdinal val col = ctx.freshName("col") + val unchangedByte = ctx.freshName("unchangedByte") val sqlType = Utils.getSQLDataType(attr.dataType) val jt = ctx.javaType(sqlType) var colAssign = "" @@ -730,12 +779,41 @@ private[sql] final case class ColumnTableScan( updatedAssign = s"read$typeName()" } updatedAssign = s"$col = $updateDecoder.getCurrentDeltaBuffer().$updatedAssign;" - - val unchangedCode = s"$updateDecoder == null || $updateDecoder.unchanged($batchOrdinal)" + val unchangedCode = if (isColumnBatchSorted) { + s"""$unchangedByte = $updateDecoder == null ? ${ColumnTableScan.NOT_IN_DELTA} : + | $updateDecoder.unchangedByte($batchOrdinal);""".stripMargin + } else s"$updateDecoder == null || $updateDecoder.unchanged($batchOrdinal)" if (attr.nullable) { val isNullVar = ctx.freshName("isNull") val defaultValue = ctx.defaultValue(jt) - val code = + val code = if (isColumnBatchSorted) { + s""" + |final $jt $col; + |final byte $unchangedByte; + |boolean $isNullVar = false; + |$unchangedCode + |if ($unchangedByte == ${ColumnTableScan.NOT_IN_DELTA}) { + | $lastRowFromDictionary = true; + |} + |// If entry is deleted, return from here + |if ($isDeletedEntry) { + | continue; + |} + |if ($unchangedByte == ${ColumnTableScan.NOT_IN_DELTA}) { + | ${genIfNonNullCode(ctx, decoder, buffer, batchOrdinal, numNullsVar)} { + | $colAssign + | } else { + | $col = $defaultValue; + | $isNullVar = true; + | } + |} else if ($updateDecoder.readNotNull()) { + | $updatedAssign + |} else { + | $col = $defaultValue; + | $isNullVar = true; + |} + """.stripMargin + } else { s""" |final $jt $col; |boolean $isNullVar = false; @@ -753,14 +831,31 @@ private[sql] final case class ColumnTableScan( | $isNullVar = true; |} """.stripMargin + } ExprCode(code, isNullVar, col) } else { - var code = + var code = if (isColumnBatchSorted) { + s""" + |final $jt $col; + |final byte $unchangedByte; + |$unchangedCode + |if ($unchangedByte == ${ColumnTableScan.NOT_IN_DELTA}) { + | $lastRowFromDictionary = true; + |} + |// If entry is deleted, return from here + |if ($isDeletedEntry) { + | continue; + |} + |if ($unchangedByte == ${ColumnTableScan.NOT_IN_DELTA}) $colAssign + |else $updatedAssign + """.stripMargin + } else { s""" |final $jt $col; |if ($unchangedCode) $colAssign |else $updatedAssign """.stripMargin + } if (weightVar != null && attr.name == Utils.WEIGHTAGE_COLUMN_NAME) { code += s"if ($col == 1) $col = $weightVar;\n" } @@ -793,6 +888,11 @@ private[sql] final case class ColumnTableScan( } object ColumnTableScan extends Logging { + // Handle inverted bytes that denote incremental insert + def getPositive(p: Int): Int = if (p < 0) ~p else p + val NOT_IN_DELTA: Byte = 1 + val INSERT_IN_DELTA: Byte = 0 + val UPDATE_IN_DELTA: Byte = -1 def generateStatPredicate(ctx: CodegenContext, isColumnTable: Boolean, schemaAttrs: Seq[AttributeReference], allFilters: Seq[Expression], numRowsTerm: String, diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnUpdateExec.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnUpdateExec.scala index 7f131c023b..688bd33e23 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnUpdateExec.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnUpdateExec.scala @@ -24,13 +24,14 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, BindReferences, Exp import org.apache.spark.sql.collection.Utils import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.columnar.encoding.{ColumnDeltaEncoder, ColumnEncoder, ColumnStatsSchema} -import org.apache.spark.sql.execution.columnar.impl.ColumnDelta +import org.apache.spark.sql.execution.columnar.impl.{ColumnDelta, ColumnFormatRelation} import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} import org.apache.spark.sql.execution.row.RowExec import org.apache.spark.sql.sources.JdbcExtendedUtils.quotedName import org.apache.spark.sql.sources.{ConnectionProperties, DestroyRelation, JdbcExtendedUtils} import org.apache.spark.sql.store.{CompressionCodecId, StoreUtils} import org.apache.spark.sql.types.StructType +import org.apache.spark.unsafe.Platform /** * Generated code plan for updates into a column table. @@ -41,11 +42,18 @@ case class ColumnUpdateExec(child: SparkPlan, columnTable: String, isPartitioned: Boolean, tableSchema: StructType, externalStore: ExternalStore, appendableRelation: JDBCAppendableRelation, updateColumns: Seq[Attribute], updateExpressions: Seq[Expression], keyColumns: Seq[Attribute], - connProps: ConnectionProperties, onExecutor: Boolean) extends ColumnExec { + connProps: ConnectionProperties, onExecutor: Boolean, caseOfDeltaInsert: Boolean) + extends ColumnExec { assert(updateColumns.length == updateExpressions.length) override def relation: Option[DestroyRelation] = Some(appendableRelation) + private val isColumnBatchSorted: Boolean = relation.isDefined && (relation.get match { + case cfr: ColumnFormatRelation => + StoreUtils.isColumnBatchSortedAscending(cfr.columnSortedOrder) || + StoreUtils.isColumnBatchSortedDescending(cfr.columnSortedOrder) + case _ => false + }) val compressionCodec: CompressionCodecId.Type = CompressionCodecId.fromName( appendableRelation.getCompressionCodec) @@ -105,6 +113,7 @@ case class ColumnUpdateExec(child: SparkPlan, columnTable: String, s"expressions=$updateExpressions compression=$compressionCodec" @transient private var batchOrdinal: String = _ + @transient private var deltaInsertOrdinal: String = _ @transient private var finishUpdate: String = _ @transient private var updateMetric: String = _ @transient protected var txId: String = _ @@ -142,6 +151,7 @@ case class ColumnUpdateExec(child: SparkPlan, columnTable: String, val cursors = ctx.freshName("cursors") val index = ctx.freshName("index") batchOrdinal = ctx.freshName("batchOrdinal") + deltaInsertOrdinal = ctx.freshName("deltaInsertOrdinal") val lastColumnBatchId = ctx.freshName("lastColumnBatchId") val lastBucketId = ctx.freshName("lastBucketId") val lastNumRows = ctx.freshName("lastNumRows") @@ -169,6 +179,7 @@ case class ColumnUpdateExec(child: SparkPlan, columnTable: String, |$initializeEncoders(); """.stripMargin) ctx.addMutableState("int", batchOrdinal, "") + ctx.addMutableState("int", deltaInsertOrdinal, "") ctx.addMutableState("long", lastColumnBatchId, s"$lastColumnBatchId = $invalidUUID;") ctx.addMutableState("int", lastBucketId, "") ctx.addMutableState("int", lastNumRows, "") @@ -233,7 +244,15 @@ case class ColumnUpdateExec(child: SparkPlan, columnTable: String, | boolean $isNull, ${ctx.javaType(dataType)} $field) { | final $deltaEncoderClass $encoderTerm = $deltaEncoders[$i]; | final $encoderClass $realEncoderTerm = $encoderTerm.getRealEncoder(); - | $encoderTerm.setUpdatePosition($ordinalIdVar); + | final int updatedOrdinalIdVar; + | if ($caseOfDeltaInsert) { + | // +ordinal is to adjust all inserts in delta so far + | // +1 since ordinalIdVar is of the last position + | updatedOrdinalIdVar = ~($ordinalIdVar + $ordinal + 1); + | } else { + | updatedOrdinalIdVar = $ordinalIdVar; + | } + | $encoderTerm.setUpdatePosition(updatedOrdinalIdVar); | ${ColumnWriter.genCodeColumnWrite(ctx, dataType, col.nullable, realEncoderTerm, encoderTerm, cursorTerm, ev.copy(isNull = isNull, value = field), ordinal)} |} @@ -278,7 +297,7 @@ case class ColumnUpdateExec(child: SparkPlan, columnTable: String, // GenerateUnsafeProjection will automatically split stats expressions into separate // methods if required so no need to add separate functions explicitly. // Count is hardcoded as zero which will change for "insert" index deltas. - val statsEv = ColumnWriter.genStatsRow(ctx, "0", stats, statsSchema) + val statsEv = ColumnWriter.genStatsRow(ctx, deltaInsertOrdinal, stats, statsSchema) ctx.addNewFunction(finishUpdate, s""" |private void $finishUpdate(long batchId, int bucketId, int numRows) { @@ -302,7 +321,8 @@ case class ColumnUpdateExec(child: SparkPlan, columnTable: String, | $batchOrdinal, buffers, ${statsEv.value}.getBytes(), $deltaIndexes); | // maxDeltaRows is -1 so that insert into row buffer is never considered | $externalStoreTerm.storeColumnBatch($tableName, columnBatch, $lastBucketId, - | $lastColumnBatchId, -1, ${compressionCodec.id}, new scala.Some($connTerm)); + | $lastColumnBatchId, -1, ${compressionCodec.id}, $isColumnBatchSorted, + | new scala.Some($connTerm)); | $result += $batchOrdinal; | ${if (updateMetric eq null) "" else s"$updateMetric.${metricAdd(batchOrdinal)};"} | $initializeEncoders(); @@ -310,6 +330,7 @@ case class ColumnUpdateExec(child: SparkPlan, columnTable: String, | $lastBucketId = bucketId; | $lastNumRows = numRows; | $batchOrdinal = 0; + | $deltaInsertOrdinal = 0; | } |} """.stripMargin) @@ -324,6 +345,9 @@ case class ColumnUpdateExec(child: SparkPlan, columnTable: String, | // write to the encoders | $callEncoders | $batchOrdinal++; + | if ($caseOfDeltaInsert) { + | $deltaInsertOrdinal++; + | } |} else { | $rowConsume |} diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/DeltaInsertExec.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/DeltaInsertExec.scala new file mode 100644 index 0000000000..f055dadf6e --- /dev/null +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/DeltaInsertExec.scala @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package org.apache.spark.sql.execution.columnar + +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.{Attribute, SortOrder} +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext +import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, PartitioningCollection, SinglePartition} +import org.apache.spark.sql.execution.columnar.impl.ColumnDelta +import org.apache.spark.sql.execution.joins.SortMergeJoinExec +import org.apache.spark.sql.execution.metric.SQLMetrics +import org.apache.spark.sql.execution.{CodegenSupport, ProjectExec, SparkPlan, UnaryExecNode} + +/** + * On top of sort merge join of two child relations. + */ +abstract class BaseDeltaInsertExec(child: SparkPlan) extends UnaryExecNode with CodegenSupport { + + override def output: Seq[Attribute] = child.output + + /** Specifies how data is partitioned across different nodes in the cluster. */ + override def outputPartitioning: Partitioning = child match { + case smj: SortMergeJoinExec => PartitioningCollection(Seq(smj.left.outputPartitioning, + smj.right.outputPartitioning)) + case prj: ProjectExec => prj.child match { + case smj: SortMergeJoinExec => PartitioningCollection(Seq(smj.left.outputPartitioning, + smj.right.outputPartitioning)) + case _ => child.outputPartitioning + } + case _ => child.outputPartitioning + } + + /** Specifies any partition requirements on the input data for this operator. */ + // override def requiredChildDistribution: Seq[Distribution] = + // Seq.fill(children.size)(UnspecifiedDistribution) + + /** Specifies how data is ordered in each partition. */ + override def outputOrdering: Seq[SortOrder] = child.outputOrdering + + /** Specifies sort order for each partition requirements on the input data for this operator. */ + // override def requiredChildOrdering: Seq[Seq[SortOrder]] = Seq.fill(children.size)(Nil) + + override lazy val metrics = Map( + "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows")) + + override def supportCodegen: Boolean = false + + override def inputRDDs(): Seq[RDD[InternalRow]] = if (child.isInstanceOf[SortMergeJoinExec]) { + child.asInstanceOf[SortMergeJoinExec].inputRDDs() + } else Nil + + override def doProduce(ctx: CodegenContext): String = if (child.isInstanceOf[SortMergeJoinExec]) { + child.asInstanceOf[SortMergeJoinExec].doProduce(ctx) + } else "" +} + +case class DeltaInsertExec(child: SparkPlan) extends BaseDeltaInsertExec(child) { + + protected override def doExecute(): RDD[InternalRow] = { + val numOutputRows = longMetric("numOutputRows") + val keyAttributes = ColumnDelta.mutableKeyAttributes.map(_.name) + val out = output.map(_.name) + val keyAttributeIndices = Seq(out.indexOf(keyAttributes.head), out.indexOf(keyAttributes(1)), + out.indexOf(keyAttributes(2)), out.indexOf(keyAttributes(3))) + + child.execute().mapPartitionsWithIndexInternal { (index, iter) => + var lastRowOrdinal: Long = Long.MinValue + var lastBatchId: Long = Long.MinValue + var lastBucketOrdinal: Integer = Int.MinValue + var lastBatchNumrows: Integer = Int.MinValue + iter.dropWhile { row => + keyAttributeIndices.forall(i => row.isNullAt(i)) + }.filter { row => + val allNulls = keyAttributeIndices.forall(i => row.isNullAt(i)) + if (!allNulls) { + lastRowOrdinal = row.getLong(keyAttributeIndices.head) + lastBatchId = row.getLong(keyAttributeIndices(1)) + lastBucketOrdinal = row.getInt(keyAttributeIndices(2)) + lastBatchNumrows = row.getInt(keyAttributeIndices(3)) + } + allNulls + }.map { row => + numOutputRows += 1 + row.setLong(keyAttributeIndices.head, lastRowOrdinal) + row.setLong(keyAttributeIndices(1), lastBatchId) + row.setInt(keyAttributeIndices(2), lastBucketOrdinal) + row.setInt(keyAttributeIndices(3), lastBatchNumrows) + row + } + } + } +} + +case class DirectInsertExec(child: SparkPlan) extends BaseDeltaInsertExec(child) { + + protected override def doExecute(): RDD[InternalRow] = { + val numOutputRows = longMetric("numOutputRows") + child.execute().mapPartitionsWithIndexInternal { (index, iter) => + iter.takeWhile { row => + val allNulls = output.indices.forall(i => row.isNullAt(i)) + if (!allNulls) { + numOutputRows += 1 + } + !allNulls + } + } + } +} diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/ExternalStore.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/ExternalStore.scala index 38cd2c75af..c40b2c0fe3 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/ExternalStore.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/ExternalStore.scala @@ -39,7 +39,7 @@ trait ExternalStore extends Serializable with Logging { def storeColumnBatch(tableName: String, batch: ColumnBatch, partitionId: Int, batchId: Long, maxDeltaRows: Int, - compressionCodecId: Int, conn: Option[Connection]): Unit + compressionCodecId: Int, isSorted: Boolean, conn: Option[Connection]): Unit def storeDelete(tableName: String, buffer: ByteBuffer, partitionId: Int, batchId: Long, compressionCodecId: Int, conn: Option[Connection]): Unit diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/encoding/ColumnDeleteEncoder.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/encoding/ColumnDeleteEncoder.scala index d91cbf4629..0d80e4eeab 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/encoding/ColumnDeleteEncoder.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/encoding/ColumnDeleteEncoder.scala @@ -25,9 +25,11 @@ import com.gemstone.gemfire.internal.cache.versions.{VersionSource, VersionTag} import com.gemstone.gemfire.internal.cache.{DiskEntry, EntryEventImpl} import com.gemstone.gemfire.internal.shared.{BufferAllocator, FetchRequest} import com.pivotal.gemfirexd.internal.engine.GfxdSerializable +import com.pivotal.gemfirexd.internal.engine.store.GemFireContainer -import org.apache.spark.sql.execution.columnar.impl.ColumnFormatValue -import org.apache.spark.sql.types.{DataType, IntegerType} +import org.apache.spark.sql.execution.columnar.ColumnTableScan +import org.apache.spark.sql.execution.columnar.impl.{ColumnDelta, ColumnFormatKey, ColumnFormatValue} +import org.apache.spark.sql.types.{DataType, IntegerType, StructField, StructType} import org.apache.spark.unsafe.Platform /** @@ -179,6 +181,73 @@ final class ColumnDeleteEncoder extends ColumnEncoder { createFinalBuffer(position, numBaseRows) } + def adjust(newValue: ByteBuffer, existingValue: ByteBuffer, field: StructField): ByteBuffer = { + deletedPositions = new Array[Int](16) + var position = 0 + var cursor1: Long = 0 + var numBaseRows: Int = 0 + var numPositions: Int = 0 + def initializeDecoder(columnBytes: AnyRef, cursor: Long): Long = { + // read the number of base rows + numBaseRows = ColumnEncoding.readInt(columnBytes, cursor) + // read the positions + numPositions = ColumnEncoding.readInt(columnBytes, cursor + 4) + cursor1 = cursor + 8 + val positionEndCursor = cursor1 + (numPositions << 2) + // round to nearest word to get data start position + ((positionEndCursor + 7) >> 3) << 3 + } + + val (decoder1, columnBytes1) = ColumnEncoding.getColumnDecoderAndBuffer( + newValue, field, initializeDecoder) + val endOffset1 = cursor1 + newValue.remaining() + var position1 = ColumnEncoding.readInt(columnBytes1, cursor1) + + val allocator2 = ColumnEncoding.getAllocator(existingValue) + val columnBytes2 = allocator2.baseObject(existingValue) + var cursor2 = allocator2.baseOffset(existingValue) + existingValue.position() + val endOffset2 = cursor2 + existingValue.remaining() + // skip 12 byte header (4 byte + number of base rows + number of elements) + cursor2 += 12 + var position2 = ColumnEncoding.readInt(columnBytes2, cursor2) + + var insertCount = 0 + def insertAdjustedPosition(pos: Int) = if (pos < 0) pos - insertCount else pos + insertCount + + // Adjust delete index with delta inserts + var doProcess = cursor1 < endOffset1 && cursor2 < endOffset2 + while (doProcess) { + val adjustedPosition2 = insertAdjustedPosition(position2) + if (ColumnTableScan.getPositive(position1) > ColumnTableScan.getPositive(adjustedPosition2)) { + // consume position2 and move + position = writeInt(position, adjustedPosition2) + cursor2 += 4 + if (cursor2 < endOffset2) { + position2 = ColumnEncoding.readInt(columnBytes2, cursor2) + } else { + doProcess = false + } + } else { + // consume position1 and move + cursor1 += 4 + if (cursor1 < endOffset1) { + if (position1 < 0) insertCount += 1 + position1 = ColumnEncoding.readInt(columnBytes1, cursor1) + } else { + doProcess = false + } + } + } + // consume any remaining of deletes + while (cursor2 < endOffset2) { + position2 = ColumnEncoding.readInt(columnBytes2, cursor2) + position = writeInt(position, insertAdjustedPosition(position2)) + cursor2 += 4 + } + + createFinalBuffer(position, numBaseRows) + } + override def finish(cursor: Long): ByteBuffer = { throw new UnsupportedOperationException( "ColumnDeleteEncoder.finish(cursor) not expected to be called") @@ -257,3 +326,74 @@ final class ColumnDeleteDelta extends ColumnFormatValue with Delta { override protected def className: String = "ColumnDeleteDelta" } + +/** Simple delta that merges the deleted positions */ +final class ColumnDeleteChange extends ColumnFormatValue with Delta { + + val columnIndex = 0 // TODO VB: Adjust columnIndex + + def this(buffer: ByteBuffer, codecId: Int, isCompressed: Boolean, + changeOwnerToStorage: Boolean = true) = { + this() + setBuffer(buffer, codecId, isCompressed, changeOwnerToStorage) + } + + override protected def copy(buffer: ByteBuffer, isCompressed: Boolean, + changeOwnerToStorage: Boolean): ColumnDeleteChange = synchronized { + new ColumnDeleteChange(buffer, compressionCodecId, isCompressed, changeOwnerToStorage) + } + + override def apply(putEvent: EntryEvent[_, _]): AnyRef = { + val event = putEvent.asInstanceOf[EntryEventImpl] + apply(event.getRegion, event.getKey, event.getOldValueAsOffHeapDeserializedOrRaw, + event.getTransactionId == null) + } + + override def apply(region: Region[_, _], key: AnyRef, oldValue: AnyRef, + prepareForOffHeap: Boolean): AnyRef = synchronized { + if (oldValue eq null) { + null + } else { + // Adjust existing delete list with incoming delta buffer + val encoder = new ColumnDeleteEncoder + val oldColumnValue = oldValue.asInstanceOf[ColumnFormatValue].getValueRetain( + FetchRequest.DECOMPRESS) + val existingBuffer = oldColumnValue.getBuffer + val newValue = getValueRetain(FetchRequest.DECOMPRESS) + val schema = region.getUserAttribute.asInstanceOf[GemFireContainer] + .fetchHiveMetaData(false) match { + case null => throw new IllegalStateException( + s"Table for region ${region.getFullPath} not found in hive metadata") + case m => m.schema.asInstanceOf[StructType] + } + try { + new ColumnFormatValue(encoder.adjust(newValue.getBuffer, existingBuffer, + schema(columnIndex)), oldColumnValue.compressionCodecId, isCompressed = false) + } finally { + oldColumnValue.release() + // Do Not release newValue that is delta buffer + // release own buffer too and delta should be unusable now + release() + } + } + } + + /** first delta update for a column will be put as is into the region */ + override def allowCreate(): Boolean = true + + override def merge(region: Region[_, _], toMerge: Delta): Delta = + throw new UnsupportedOperationException("Unexpected call to ColumnDeleteChange.merge") + + override def cloneDelta(): Delta = + throw new UnsupportedOperationException("Unexpected call to ColumnDeleteChange.cloneDelta") + + override def setVersionTag(versionTag: VersionTag[_ <: VersionSource[_]]): Unit = + throw new UnsupportedOperationException("Unexpected call to ColumnDeleteChange.setVersionTag") + + override def getVersionTag: VersionTag[_ <: VersionSource[_]] = + throw new UnsupportedOperationException("Unexpected call to ColumnDeleteChange.getVersionTag") + + override def getGfxdID: Byte = GfxdSerializable.COLUMN_DELETE_CHANGE + + override protected def className: String = "ColumnDeleteChange" +} diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/encoding/ColumnDeltaEncoder.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/encoding/ColumnDeltaEncoder.scala index d3ff2cabd7..180183f596 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/encoding/ColumnDeltaEncoder.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/encoding/ColumnDeltaEncoder.scala @@ -26,6 +26,7 @@ import org.codehaus.janino.CompilerFactory import org.apache.spark.sql.catalyst.util.{SerializedArray, SerializedMap, SerializedRow} import org.apache.spark.sql.collection.Utils +import org.apache.spark.sql.execution.columnar.ColumnTableScan import org.apache.spark.sql.execution.columnar.impl.{ColumnDelta, ColumnFormatValue} import org.apache.spark.sql.store.CodeGeneration import org.apache.spark.sql.types._ @@ -316,7 +317,7 @@ final class ColumnDeltaEncoder(val hierarchyDepth: Int) extends ColumnEncoder { } def merge(newValue: ByteBuffer, existingValue: ByteBuffer, - existingIsDelta: Boolean, field: StructField): ByteBuffer = { + existingIsDelta: Boolean, field: StructField, isColumnBatchSorted: Boolean): ByteBuffer = { // TODO: PERF: delta encoder should create a "merged" dictionary i.e. having // only elements beyond the main dictionary so that the overall decoder can be // dictionary enabled. As of now delta decoder does not have an overall dictionary @@ -398,14 +399,29 @@ final class ColumnDeltaEncoder(val hierarchyDepth: Int) extends ColumnEncoder { var relativePosition2 = 0 var encoderPosition = -1 + var insertCount = 0 + def insertAdjustedPosition(pos: Int) = if (pos < 0) pos - insertCount else pos + insertCount + var doProcess = numPositions1 > 0 && numPositions2 > 0 while (doProcess) { encoderPosition += 1 - val areEqual = position1 == position2 - val isGreater = position1 > position2 + val adjustedPosition2 = insertAdjustedPosition(position2) + // areEqual would be false if position1 is negative + val areEqual = if (isColumnBatchSorted) { + position1 == ColumnTableScan.getPositive(adjustedPosition2) + } else position1 == position2 + val isGreater = if (isColumnBatchSorted) { + ColumnTableScan.getPositive(position1) > ColumnTableScan.getPositive(adjustedPosition2) + } else position1 > position2 if (isGreater || areEqual) { // set next update position to be from second - if (existingIsDelta && !areEqual) positionsArray(encoderPosition) = position2 + if (existingIsDelta && !areEqual) { + if (isColumnBatchSorted) { + positionsArray(encoderPosition) = adjustedPosition2 + } else { + positionsArray(encoderPosition) = position2 + } + } // consume data at position2 and move it if position2 is smaller // else if they are equal then newValue gets precedence cursor = consumeDecoder(decoder2, if (nullable2) relativePosition2 else -1, @@ -426,7 +442,12 @@ final class ColumnDeltaEncoder(val hierarchyDepth: Int) extends ColumnEncoder { // write for the second was skipped in the first block above if (!isGreater) { // set next update position to be from first - if (existingIsDelta) positionsArray(encoderPosition) = position1 + if (existingIsDelta) { + positionsArray(encoderPosition) = position1 + if (isColumnBatchSorted) { + if (position1 < 0) insertCount += 1 + } + } // consume data at position1 and move it cursor = consumeDecoder(decoder1, if (nullable1) relativePosition1 else -1, columnBytes1, writer, cursor, encoderPosition) @@ -446,7 +467,13 @@ final class ColumnDeltaEncoder(val hierarchyDepth: Int) extends ColumnEncoder { encoderPosition += 1 // set next update position to be from first if (existingIsDelta) { - positionsArray(encoderPosition) = ColumnEncoding.readInt(columnBytes1, positionCursor1) + if (isColumnBatchSorted) { + val pos1 = ColumnEncoding.readInt(columnBytes1, positionCursor1) + positionsArray(encoderPosition) = pos1 + if (pos1 < 0) insertCount += 1 + } else { + positionsArray(encoderPosition) = ColumnEncoding.readInt(columnBytes1, positionCursor1) + } positionCursor1 += 4 } cursor = consumeDecoder(decoder1, if (nullable1) relativePosition1 else -1, @@ -458,7 +485,12 @@ final class ColumnDeltaEncoder(val hierarchyDepth: Int) extends ColumnEncoder { encoderPosition += 1 // set next update position to be from second if (existingIsDelta) { - positionsArray(encoderPosition) = ColumnEncoding.readInt(columnBytes2, positionCursor2) + if (isColumnBatchSorted) { + positionsArray(encoderPosition) = + insertAdjustedPosition(ColumnEncoding.readInt(columnBytes2, positionCursor2)) + } else { + positionsArray(encoderPosition) = ColumnEncoding.readInt(columnBytes2, positionCursor2) + } positionCursor2 += 4 } cursor = consumeDecoder(decoder2, if (nullable2) relativePosition2 else -1, diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/encoding/UpdatedColumnDecoder.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/encoding/UpdatedColumnDecoder.scala index b069a2fde9..b8b67843e8 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/encoding/UpdatedColumnDecoder.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/encoding/UpdatedColumnDecoder.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.execution.columnar.encoding import java.nio.ByteBuffer +import org.apache.spark.sql.execution.columnar.ColumnTableScan import org.apache.spark.sql.types._ /** @@ -127,6 +128,30 @@ abstract class UpdatedColumnDecoderBase(decoder: ColumnDecoder, field: StructFie else skipUntil(ordinal) } + private def skipUntilByte(ordinal: Int): Byte = { + while (true) { + // update the cursor and keep on till ordinal is not reached + nextUpdatedPosition = moveToNextUpdatedPosition() + if (ColumnTableScan.getPositive(nextUpdatedPosition) > ordinal) { + return ColumnTableScan.NOT_IN_DELTA + } + if (ColumnTableScan.getPositive(nextUpdatedPosition) == ordinal) { + return isInsertOrUpdate(ordinal) + } + } + ColumnTableScan.NOT_IN_DELTA // never reached + } + + private def isInsertOrUpdate(ordinal: Int): Byte = if (nextUpdatedPosition < 0) { + ColumnTableScan.INSERT_IN_DELTA + } else ColumnTableScan.UPDATE_IN_DELTA + + final def unchangedByte(ordinal: Int): Byte = { + if (ColumnTableScan.getPositive(nextUpdatedPosition) > ordinal) ColumnTableScan.NOT_IN_DELTA + else if (ColumnTableScan.getPositive(nextUpdatedPosition) == ordinal) isInsertOrUpdate(ordinal) + else skipUntilByte(ordinal) + } + def readNotNull: Boolean // TODO: SW: need to create a combined delta+full value dictionary for this to work diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/ColumnDelta.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/ColumnDelta.scala index 2088ebd4fc..b764df6f79 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/ColumnDelta.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/ColumnDelta.scala @@ -30,7 +30,9 @@ import com.pivotal.gemfirexd.internal.engine.store.GemFireContainer import org.apache.spark.sql.catalyst.expressions.{Add, AttributeReference, BoundReference, GenericInternalRow, UnsafeProjection} import org.apache.spark.sql.catalyst.util.TypeUtils import org.apache.spark.sql.collection.Utils +import org.apache.spark.sql.execution.columnar.ColumnTableScan import org.apache.spark.sql.execution.columnar.encoding.{ColumnDeltaEncoder, ColumnEncoding, ColumnStatsSchema} +import org.apache.spark.sql.store.StoreUtils import org.apache.spark.sql.types.{IntegerType, LongType, StructField, StructType} /** @@ -84,13 +86,16 @@ final class ColumnDelta extends ColumnFormatValue with Delta { val existingBuffer = oldColValue.getBuffer val newValue = getValueRetain(FetchRequest.DECOMPRESS) val newBuffer = newValue.getBuffer + var isColumnBatchSorted = false try { - val schema = region.getUserAttribute.asInstanceOf[GemFireContainer] + val (schema, columnTableSorting) = region.getUserAttribute.asInstanceOf[GemFireContainer] .fetchHiveMetaData(false) match { case null => throw new IllegalStateException( s"Table for region ${region.getFullPath} not found in hive metadata") - case m => m.schema.asInstanceOf[StructType] + case m => (m.schema.asInstanceOf[StructType], m.columnTableSortOrder) } + isColumnBatchSorted = StoreUtils.isColumnBatchSortedAscending(columnTableSorting) || + StoreUtils.isColumnBatchSortedDescending(columnTableSorting) val columnIndex = key.asInstanceOf[ColumnFormatKey].columnIndex // TODO: SW: if old value itself is returned, then avoid any put at GemFire layer // (perhaps throw some exception that can be caught and ignored in virtualPut) @@ -109,12 +114,15 @@ final class ColumnDelta extends ColumnFormatValue with Delta { val tableColumnIndex = ColumnDelta.tableColumnIndex(columnIndex) - 1 val encoder = new ColumnDeltaEncoder(ColumnDelta.deltaHierarchyDepth(columnIndex)) new ColumnFormatValue(encoder.merge(newBuffer, existingBuffer, - columnIndex < ColumnFormatEntry.DELETE_MASK_COL_INDEX, schema(tableColumnIndex)), - oldColValue.compressionCodecId, isCompressed = false) + columnIndex < ColumnFormatEntry.DELETE_MASK_COL_INDEX, schema(tableColumnIndex), + isColumnBatchSorted), oldColValue.compressionCodecId, isCompressed = false) } } finally { oldColValue.release() - newValue.release() + // TODO VB: Do not release delta buffer if case of delta insert + if (!isColumnBatchSorted) { + newValue.release() + } // release own buffer too and delta should be unusable now release() } @@ -135,7 +143,7 @@ final class ColumnDelta extends ColumnFormatValue with Delta { values(ColumnStatsSchema.COUNT_INDEX_IN_SCHEMA) = oldCount + newCount statsSchema(ColumnStatsSchema.COUNT_INDEX_IN_SCHEMA) = StructField("count", IntegerType, nullable = false) - var hasChange = false + var hasChange = oldCount != newCount // non-generated code for evaluation since this is only for one row // (besides binding to two separate rows will need custom code) for (i <- schema.indices) { diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/ColumnFormatEntry.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/ColumnFormatEntry.scala index c74d74baf4..5c7cbc17e8 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/ColumnFormatEntry.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/ColumnFormatEntry.scala @@ -43,7 +43,7 @@ import io.snappydata.Constant import org.apache.spark.memory.MemoryManagerCallback import org.apache.spark.sql.collection.Utils -import org.apache.spark.sql.execution.columnar.encoding.{ColumnDeleteDelta, ColumnEncoding, ColumnStatsSchema} +import org.apache.spark.sql.execution.columnar.encoding.{ColumnDeleteChange, ColumnDeleteDelta, ColumnEncoding, ColumnStatsSchema} import org.apache.spark.sql.execution.columnar.impl.ColumnFormatEntry.alignedSize import org.apache.spark.sql.store.{CompressionCodecId, CompressionUtils} @@ -70,6 +70,10 @@ object ColumnFormatEntry { new Supplier[GfxdDSFID] { override def get(): GfxdDSFID = new ColumnDeleteDelta() }) + DSFIDFactory.registerGemFireXDClass(GfxdSerializable.COLUMN_DELETE_CHANGE, + new Supplier[GfxdDSFID] { + override def get(): GfxdDSFID = new ColumnDeleteChange() + }) } /** diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/ColumnFormatIterator.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/ColumnFormatIterator.scala index 310190534b..ddb0f5f4ea 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/ColumnFormatIterator.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/ColumnFormatIterator.scala @@ -18,20 +18,32 @@ package org.apache.spark.sql.execution.columnar.impl import java.util.function.LongFunction +import scala.collection.mutable.ArrayBuffer + import com.gemstone.gemfire.cache.RegionDestroyedException import com.gemstone.gemfire.internal.cache.DiskBlockSortManager.DiskBlockSorter import com.gemstone.gemfire.internal.cache.DistributedRegion.{DiskEntryPage, DiskPosition} import com.gemstone.gemfire.internal.cache._ import com.gemstone.gemfire.internal.cache.store.SerializedDiskBuffer import com.gemstone.gemfire.internal.concurrent.CustomEntryConcurrentHashMap +import com.gemstone.gemfire.internal.shared.FetchRequest import com.google.common.primitives.Ints import com.koloboke.function.LongObjPredicate +import com.pivotal.gemfirexd.internal.engine.Misc +import com.pivotal.gemfirexd.internal.engine.ddl.resolver.GfxdPartitionByExpressionResolver +import com.pivotal.gemfirexd.internal.engine.distributed.utils.GemFireXDUtils +import com.pivotal.gemfirexd.internal.engine.store.GemFireContainer import com.pivotal.gemfirexd.internal.iapi.util.ReuseFactory import io.snappydata.collection.LongObjectHashMap -import org.apache.spark.sql.catalyst.expressions.UnsafeRow -import org.apache.spark.sql.execution.columnar.encoding.BitSet +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.codegen.GenerateOrdering +import org.apache.spark.sql.catalyst.expressions.{Ascending, AttributeReference, AttributeSeq, BindReferences, BoundReference, Descending, Expression, InterpretedOrdering, SortOrder, UnsafeProjection, UnsafeRow} +import org.apache.spark.sql.collection.Utils +import org.apache.spark.sql.execution.columnar.encoding.{BitSet, ColumnStatsSchema} import org.apache.spark.sql.execution.columnar.impl.ColumnFormatEntry._ +import org.apache.spark.sql.store.StoreUtils +import org.apache.spark.sql.types.StructType import org.apache.spark.unsafe.Platform /** @@ -66,10 +78,47 @@ final class ColumnFormatIterator(baseRegion: LocalRegion, projection: Array[Int] */ private val inMemoryBatches = new java.util.ArrayList[LongObjectHashMap[AnyRef]](4) private var inMemoryBatchIndex: Int = _ + private var inMemorySortedBatches: Array[(InternalRow, LongObjectHashMap[AnyRef])] = _ + + private val container = distributedRegion.getUserAttribute + .asInstanceOf[GemFireContainer] + private val columnTableSorting = container.fetchHiveMetaData(false).columnTableSortOrder + // TODO VB: Only enable case of sorted scan for join, insert and group by queries + // See comments in ColumnTableScan + private val isColumnBatchSorted = (StoreUtils.isColumnBatchSortedAscending(columnTableSorting) || + StoreUtils.isColumnBatchSortedDescending(columnTableSorting)) - private val canOverflow = + private val canOverflow = !isColumnBatchSorted && distributedRegion.isOverflowEnabled && distributedRegion.getDataPolicy.withPersistence() + private val (partitioningProjection, statsLen, partitioningOrdering) = if (isColumnBatchSorted) { + val rowBufferTable = GemFireContainer.getRowBufferTableName(container.getQualifiedTableName) + val rowBufferRegion = Misc.getRegionForTable(rowBufferTable, true).asInstanceOf[LocalRegion] + val paritioningPositions = GemFireXDUtils.getResolver(rowBufferRegion) + .asInstanceOf[GfxdPartitionByExpressionResolver].getColumnPositions + val tableSchema = container.fetchHiveMetaData(false).schema.asInstanceOf[StructType] + val statsSchema = tableSchema.map(f => + ColumnStatsSchema(f.name, f.dataType, nullCountNullable = true)) + val fullStatsSchema = ColumnStatsSchema.COUNT_ATTRIBUTE +: statsSchema.flatMap(_.schema) + val partUnboundExprs = paritioningPositions.map(pos => pos -1). + map(pos => statsSchema(pos).lowerBound) + val partitioningExprs = partUnboundExprs.map(ae => { + BindReferences.bindReference(ae.asInstanceOf[Expression], fullStatsSchema). + asInstanceOf[BoundReference] + }) + val partExprsSchema: AttributeSeq = partUnboundExprs.toSeq + val ordering = if (StoreUtils.isColumnBatchSortedAscending(columnTableSorting)) { + val sortOrdering = partUnboundExprs.map(ae => SortOrder(BindReferences.bindReference( + ae, partExprsSchema), Ascending)) + GenerateOrdering.generate(sortOrdering) + } else { + val sortOrdering = partUnboundExprs.map(ae => SortOrder(BindReferences.bindReference( + ae, partExprsSchema), Descending)) + GenerateOrdering.generate(sortOrdering) + } + (UnsafeProjection.create(partitioningExprs), fullStatsSchema.length, ordering) + } else (null, 0, null) + private val projectionBitSet = { if (projection.length > 0) { val maxProjection = Ints.max(projection: _*) @@ -123,7 +172,9 @@ final class ColumnFormatIterator(baseRegion: LocalRegion, projection: Array[Int] checkRegion(region) currentRegion = region entryIterator = region.entries.regionEntries().iterator().asInstanceOf[MapValueIterator] - advanceToNextBatchSet() + if (isColumnBatchSorted) { + inMemorySortedBatches = initSortedBatchSets() + } else advanceToNextBatchSet() } override def initDiskIterator(): Boolean = { @@ -144,13 +195,22 @@ final class ColumnFormatIterator(baseRegion: LocalRegion, projection: Array[Int] } override def hasNext: Boolean = { - if (entryIterator ne null) { + if (isColumnBatchSorted) { + inMemoryBatchIndex + 1 < inMemorySortedBatches.length + } else if (entryIterator ne null) { if (inMemoryBatchIndex + 1 < inMemoryBatches.size()) true else advanceToNextBatchSet() } else nextDiskBatch ne null } override def next(): RegionEntry = { - if (entryIterator ne null) { + if (isColumnBatchSorted) { + inMemoryBatchIndex += 1 + if (inMemoryBatchIndex >= inMemorySortedBatches.length) { + if (!advanceToNextBatchSet()) throw new NoSuchElementException + } + val map = inMemorySortedBatches(inMemoryBatchIndex) + map._2.getGlobalState.asInstanceOf[RegionEntry] + } else if (entryIterator ne null) { inMemoryBatchIndex += 1 if (inMemoryBatchIndex >= inMemoryBatches.size()) { if (!advanceToNextBatchSet()) throw new NoSuchElementException @@ -170,7 +230,9 @@ final class ColumnFormatIterator(baseRegion: LocalRegion, projection: Array[Int] override def getColumnValue(columnIndex: Int): AnyRef = { val column = columnIndex & 0xffffffffL - if (entryIterator ne null) inMemoryBatches.get(inMemoryBatchIndex).get(column) + if (isColumnBatchSorted) { + inMemorySortedBatches(inMemoryBatchIndex)._2.get(column) + } else if (entryIterator ne null) inMemoryBatches.get(inMemoryBatchIndex).get(column) else if (columnIndex == DELTA_STATROW_COL_INDEX) currentDiskBatch.getDeltaStatsValue else currentDiskBatch.entryMap.get(column) } @@ -304,6 +366,77 @@ final class ColumnFormatIterator(baseRegion: LocalRegion, projection: Array[Int] } false } + + def initSortedBatchSets(): Array[(InternalRow, LongObjectHashMap[AnyRef])] = { + val inMemorySortedBatchBuffer = new ArrayBuffer[(InternalRow, LongObjectHashMap[AnyRef])]() + inMemoryBatchIndex = -1 + while (entryIterator.hasNext) { + /** + * Maintains the current set of batches that are being iterated. + * When all columns provided in the projectionBitSet have been marked as + * [[inMemorySortedBatchBuffer]] then the batch is cleared from the map. + */ + val activeBatches = LongObjectHashMap.withExpectedSize[LongObjectHashMap[AnyRef]](4) + val partitionRows = LongObjectHashMap.withExpectedSize[InternalRow](4) + + // iterate till next map index since all columns of the same batch + // are guaranteed to be in the same index + val mapIndex = entryIterator.getMapTableIndex + while (entryIterator.hasNext && mapIndex == entryIterator.getMapTableIndex) { + val aEntry = entryIterator.next() + var entry: RegionEntry = aEntry + val key = aEntry.getRawKey.asInstanceOf[ColumnFormatKey] + // check if it is one of required projection columns, their deltas or meta-columns + val columnIndex = key.columnIndex + if ((columnIndex < 0 && columnIndex >= DELETE_MASK_COL_INDEX) || { + val tableColumn = ColumnDelta.tableColumnIndex(columnIndex) + tableColumn > 0 && + BitSet.isSet(projectionBitSet, Platform.LONG_ARRAY_OFFSET, + tableColumn, projectionBitSet.length) + }) { + // note that the map used below uses value==0 to indicate free, so the + // column indexes have to be 1-based (and negative for deltas/meta-data) + // and so the same values as that stored in ColumnFormatKey are used + val uuidMap = activeBatches.computeIfAbsent(key.uuid, newMapCreator) + // set the stats entry in the state + if (columnIndex == STATROW_COL_INDEX) { + if (uuidMap.getGlobalState eq null) uuidMap.setGlobalState(entry) + val statsValue = entry.getValue(currentRegion).asInstanceOf[ColumnFormatValue] + val statsVal = statsValue.getValueRetain(FetchRequest.DECOMPRESS) + try { + val statsRow = Utils.toUnsafeRow(statsVal.getBuffer, statsLen) + partitionRows.justPut(key.uuid, partitioningProjection(statsRow).copy()) + } finally { + statsValue.release() + } + } else { + // fetch the TX snapshot entry; the stats row entry is skipped here + // since that will be done by higher-level PR iterator that returns + // the stats row entry + if (txState ne null) { + entry = txState.getLocalEntry(distributedRegion, currentRegion, + -1 /* not used */ , aEntry, false).asInstanceOf[RegionEntry] + } + setValue(entry, columnIndex, uuidMap) + } + } + } + + // if there are entries that are overflowed, then pass them to the disk sorter + // while entries that are fully in memory are stored and returned + if (activeBatches.size() > 0) { + activeBatches.forEachWhile(new LongObjPredicate[LongObjectHashMap[AnyRef]] { + override def test(uuid: Long, map: LongObjectHashMap[AnyRef]): Boolean = { + if (map.getGlobalState ne null) { + inMemorySortedBatchBuffer += partitionRows.get(uuid) -> map + } + true + } + }) + } + } + inMemorySortedBatchBuffer.toArray.sortBy(_._1)(partitioningOrdering) + } } /** diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/ColumnFormatRelation.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/ColumnFormatRelation.scala index 3e6a60cdd9..ffc31cade9 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/ColumnFormatRelation.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/ColumnFormatRelation.scala @@ -124,6 +124,8 @@ abstract class BaseColumnFormatRelation( partitioningColumns } + override def getSortingOrder: String = "" + override private[sql] lazy val externalColumnTableName: String = ColumnFormatRelation.columnBatchTableName(table, Some(() => sqlContext.sparkSession.asInstanceOf[SnappySession])) @@ -268,10 +270,11 @@ abstract class BaseColumnFormatRelation( */ override def getUpdatePlan(relation: LogicalRelation, child: SparkPlan, updateColumns: Seq[Attribute], updateExpressions: Seq[Expression], - keyColumns: Seq[Attribute]): SparkPlan = { + keyColumns: Seq[Attribute], isDeltaInsert: Boolean): SparkPlan = { ColumnUpdateExec(child, externalColumnTableName, partitionColumns, partitionExpressions(relation), numBuckets, isPartitioned, schema, externalStore, this, - updateColumns, updateExpressions, keyColumns, connProperties, onExecutor = false) + updateColumns, updateExpressions, keyColumns, connProperties, onExecutor = false, + caseOfDeltaInsert = isDeltaInsert) } /** @@ -517,7 +520,9 @@ class ColumnFormatRelation( _origOptions: Map[String, String], _externalStore: ExternalStore, _partitioningColumns: Seq[String], - _context: SQLContext) + _context: SQLContext, + val columnSortedOrder: String = "", + val allowInsertWhileScan: Boolean = false) extends BaseColumnFormatRelation( _table, _provider, @@ -529,7 +534,7 @@ class ColumnFormatRelation( _externalStore, _partitioningColumns, _context) - with ParentRelation with DependentRelation with BulkPutRelation { + with ParentRelation with DependentRelation with BulkPutRelation with ColumnTableInsertRelation { val tableOptions = new CaseInsensitiveMutableHashMap(_origOptions) override def withKeyColumns(relation: LogicalRelation, @@ -543,12 +548,15 @@ class ColumnFormatRelation( val schema = StructType(cr.schema ++ ColumnDelta.mutableKeyFields) val newRelation = new ColumnFormatRelation(cr.table, cr.provider, cr.mode, schema, cr.schemaExtensions, cr.ddlExtensionForShadowTable, - cr.origOptions, cr.externalStore, cr.partitioningColumns, cr.sqlContext) + cr.origOptions, cr.externalStore, cr.partitioningColumns, cr.sqlContext, + cr.columnSortedOrder) newRelation.delayRollover = true relation.copy(relation = newRelation, expectedOutputAttributes = Some(relation.output ++ ColumnDelta.mutableKeyAttributes)) } + override def getSortingOrder: String = columnSortedOrder + override def addDependent(dependent: DependentRelation, catalog: SnappyStoreHiveCatalog): Boolean = DependencyCatalog.addDependent(table, dependent.name) @@ -677,6 +685,10 @@ class ColumnFormatRelation( /** Name of this relation in the catalog. */ override def name: String = table + override def getColumnTableInsertPlan(insertPlan: SparkPlan, updatePlan: SparkPlan): SparkPlan = { + ColumnTableInsertExec(insertPlan, updatePlan) + } + /** * Get a spark plan for puts. If the row is already present, it gets updated * otherwise it gets inserted into the table represented by this relation. @@ -693,6 +705,19 @@ class ColumnFormatRelation( case None => None } } + + override def equals(that: Any): Boolean = { + val se = super.equals(that) + // Handle InsertIntoTable rule of PreWriteCheck + if (se && (StoreUtils.isColumnBatchSortedAscending(columnSortedOrder) || + StoreUtils.isColumnBatchSortedDescending(columnSortedOrder))) { + that match { + case cfr: ColumnFormatRelation if cfr.allowInsertWhileScan => + allowInsertWhileScan + case _ => se + } + } else se + } } /** @@ -797,10 +822,11 @@ final class DefaultSource extends SchemaRelationProvider val table = Utils.toUpperCase(ExternalStoreUtils.removeInternalProps(parameters)) val partitions = ExternalStoreUtils.getAndSetTotalPartitions( Some(sqlContext.sparkContext), parameters, forManagedTable = true) + val (partitioningColumns, columnSorting) = StoreUtils.getPartitioningColumns(parameters) + parameters.put(StoreUtils.COLUMN_BATCH_SORTED, columnSorting) val tableOptions = new CaseInsensitiveMap(parameters.toMap) val parametersForShadowTable = new CaseInsensitiveMutableHashMap(parameters) - val partitioningColumns = StoreUtils.getPartitioningColumns(parameters) // change the schema to use VARCHAR for StringType for partitioning columns // so that the row buffer table can use it as part of primary key val (primaryKeyClause, stringPKCols) = StoreUtils.getPrimaryKeyClause( @@ -869,7 +895,8 @@ final class DefaultSource extends SchemaRelationProvider tableOptions, externalStore, partitioningColumns, - sqlContext) + sqlContext, + columnSorting) } val isRelationforSample = parameters.get(ExternalStoreUtils.RELATION_FOR_SAMPLE) .exists(_.toBoolean) diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/JDBCSourceAsColumnarStore.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/JDBCSourceAsColumnarStore.scala index a6f4e7ceff..cbc36d56fc 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/JDBCSourceAsColumnarStore.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/JDBCSourceAsColumnarStore.scala @@ -44,7 +44,7 @@ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.collection._ import org.apache.spark.sql.execution.columnar._ -import org.apache.spark.sql.execution.columnar.encoding.ColumnDeleteDelta +import org.apache.spark.sql.execution.columnar.encoding.{ColumnDeleteChange, ColumnDeleteDelta} import org.apache.spark.sql.execution.row.{ResultSetTraversal, RowFormatScanRDD, RowInsertExec} import org.apache.spark.sql.execution.sources.StoreDataSourceStrategy.translateToFilter import org.apache.spark.sql.execution.{BufferedRowIterator, ConnectionPool, RDDKryo, WholeStageCodegenExec} @@ -96,18 +96,18 @@ class JDBCSourceAsColumnarStore(private var _connProperties: ConnectionPropertie override def storeColumnBatch(columnTableName: String, batch: ColumnBatch, partitionId: Int, batchId: Long, maxDeltaRows: Int, - compressionCodecId: Int, conn: Option[Connection]): Unit = { + compressionCodecId: Int, isSorted: Boolean, conn: Option[Connection]): Unit = { // check for task cancellation before further processing checkTaskCancellation() if (partitionId >= 0) { doInsertOrPut(columnTableName, batch, batchId, partitionId, maxDeltaRows, - compressionCodecId, conn) + compressionCodecId, conn, isSorted) } else { val (bucketId, br, batchSize) = getPartitionID(columnTableName, () => batch.buffers.foldLeft(0L)(_ + _.capacity())) try { doInsertOrPut(columnTableName, batch, batchId, bucketId, maxDeltaRows, - compressionCodecId, conn) + compressionCodecId, conn, isSorted) } finally br match { case None => case Some(bucket) => bucket.updateInProgressSize(-batchSize) @@ -346,7 +346,8 @@ class JDBCSourceAsColumnarStore(private var _connProperties: ConnectionPropertie * batches for now. */ private def doSnappyInsertOrPut(region: LocalRegion, batch: ColumnBatch, - batchId: Long, partitionId: Int, maxDeltaRows: Int, compressionCodecId: Int): Unit = { + batchId: Long, partitionId: Int, maxDeltaRows: Int, compressionCodecId: Int, + isSorted: Boolean): Unit = { val deltaUpdate = batch.deltaIndexes ne null val statRowIndex = if (deltaUpdate) ColumnFormatEntry.DELTA_STATROW_COL_INDEX else ColumnFormatEntry.STATROW_COL_INDEX @@ -373,6 +374,18 @@ class JDBCSourceAsColumnarStore(private var _connProperties: ConnectionPropertie } else new ColumnFormatValue(statsBuffer, compressionCodecId, isCompressed = false) keyValues.put(key, value) + // update the delete indexes for delta inserts + if (deltaUpdate && isSorted) { + val deleteKey = new ColumnFormatKey(batchId, partitionId, + ColumnFormatEntry.DELETE_MASK_COL_INDEX) + if (region.get(deleteKey) != null) { + // should always buffers(0) go? + val deleteChange = new ColumnDeleteChange(batch.buffers(0), compressionCodecId, + isCompressed = false) + keyValues.put(deleteKey, deleteChange) + } + } + // do a putAll of the key-value map with create=true val startPut = CachePerfStats.getStatTime val putAllOp = region.newPutAllOperation(keyValues) @@ -531,9 +544,9 @@ class JDBCSourceAsColumnarStore(private var _connProperties: ConnectionPropertie private def doInsertOrPut(columnTableName: String, batch: ColumnBatch, batchId: Long, partitionId: Int, maxDeltaRows: Int, compressionCodecId: Int, - conn: Option[Connection] = None): Unit = { + conn: Option[Connection] = None, isSorted: Boolean): Unit = { // split the batch and put into row buffer if it is small - if (maxDeltaRows > 0 && batch.numRows < math.min(maxDeltaRows, + if (!isSorted && maxDeltaRows > 0 && batch.numRows < math.min(maxDeltaRows, math.max(maxDeltaRows >>> 1, SystemProperties.SNAPPY_MIN_COLUMN_DELTA_ROWS))) { // noinspection RedundantDefaultArgument tryExecute(tableName, closeOnSuccessOrFailure = false /* batch.deltaIndexes ne null */ , @@ -547,7 +560,8 @@ class JDBCSourceAsColumnarStore(private var _connProperties: ConnectionPropertie // all other callers (ColumnFormatEncoder, BucketRegion) use the same val uuid = if (BucketRegion.isValidUUID(batchId)) batchId else region.getColocatedWithRegion.newUUID(false) - doSnappyInsertOrPut(region, batch, uuid, partitionId, maxDeltaRows, compressionCodecId) + doSnappyInsertOrPut(region, batch, uuid, partitionId, maxDeltaRows, compressionCodecId, + isSorted) case _ => // noinspection RedundantDefaultArgument diff --git a/core/src/main/scala/org/apache/spark/sql/internal/ColumnTableBulkOps.scala b/core/src/main/scala/org/apache/spark/sql/internal/ColumnTableBulkOps.scala index 48c7273842..280e32b1dd 100644 --- a/core/src/main/scala/org/apache/spark/sql/internal/ColumnTableBulkOps.scala +++ b/core/src/main/scala/org/apache/spark/sql/internal/ColumnTableBulkOps.scala @@ -19,13 +19,15 @@ package org.apache.spark.sql.internal import io.snappydata.Property import org.apache.spark.sql.catalyst.encoders.RowEncoder -import org.apache.spark.sql.catalyst.expressions.{And, Attribute, AttributeReference, EqualTo, Expression} -import org.apache.spark.sql.catalyst.plans.logical.{BinaryNode, Join, LogicalPlan, OverwriteOptions, Project} -import org.apache.spark.sql.catalyst.plans.{Inner, LeftAnti} +import org.apache.spark.sql.catalyst.expressions.{And, Attribute, AttributeReference, EqualTo, Expression, PredicateHelper} +import org.apache.spark.sql.catalyst.plans.logical.{BinaryNode, InsertIntoTable, Join, LogicalPlan, OverwriteOptions, Project, UnaryNode} +import org.apache.spark.sql.catalyst.plans.{FullOuter, Inner, LeftAnti} import org.apache.spark.sql.collection.Utils import org.apache.spark.sql.execution.columnar.ExternalStoreUtils +import org.apache.spark.sql.execution.columnar.impl.{ColumnDelta, ColumnFormatRelation} import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.sources._ +import org.apache.spark.sql.store.StoreUtils import org.apache.spark.sql.types.{DataType, LongType} import org.apache.spark.sql.{AnalysisException, Dataset, SnappySession, SparkSession} @@ -36,7 +38,70 @@ import org.apache.spark.sql.{AnalysisException, Dataset, SnappySession, SparkSes */ object ColumnTableBulkOps { + def transformInsertPlan(sparkSession: SparkSession, + originalPlan: InsertIntoTable): LogicalPlan = { + val table = originalPlan.table + var transFormedPlan: LogicalPlan = originalPlan + table.collectFirst { + case lr@LogicalRelation(mutable: MutableRelation, _, _) => + if (StoreUtils.isColumnBatchSortedAscending(mutable.getSortingOrder) || + StoreUtils.isColumnBatchSortedDescending(mutable.getSortingOrder)) { + val partitionColumns = mutable.partitionColumns + if (partitionColumns.isEmpty) { + throw new AnalysisException( + s"Insert in sorted column table requires partitioning column(s)" + + s" but got empty string") + } + val newTableOption = table match { + case LogicalRelation(cr: ColumnFormatRelation, b, a) => + Some(LogicalRelation(new ColumnFormatRelation(cr.table, cr.provider, + cr.mode, originalPlan.table.schema, cr.schemaExtensions, + cr.ddlExtensionForShadowTable, cr.origOptions, cr.externalStore, + cr.partitioningColumns, cr.sqlContext, cr.columnSortedOrder, + allowInsertWhileScan = true), b, a)) + case _ => None + } + if (newTableOption.isDefined) { + val newTable = newTableOption.get + val subQuery = originalPlan.child + val condition = prepareCondition(sparkSession, table, subQuery, partitionColumns, + changeCondition = true) + val joinSubQuery: LogicalPlan = Join(table, subQuery, FullOuter, condition) + // Only enable in case of proven benefit using performance testing + // val joinDS = new Dataset(sparkSession, joinSubQuery, RowEncoder(joinSubQuery.schema)) + // joinDS.cache() + // val analyzedJoin = joinDS.queryExecution.analyzed.asInstanceOf[Join] + // Below use analyzedJoin in place of joinSubQuery + + val insertSubQuery: LogicalPlan = DeltaInsertNode(joinSubQuery, isDirectInsert = true) + val insertPlan = new Insert(newTable, Map.empty[String, + Option[String]], Project(subQuery.output, insertSubQuery), + OverwriteOptions(enabled = false), ifNotExists = false) + + // TODO VB: Any cheaper way to find table is empty or not? + // TODO VB: What if somebody else would insert in parallel? + val tabEmpty = new Dataset(sparkSession, table, RowEncoder(table.schema)).count() == 0 + transFormedPlan = if (!tabEmpty) { + val updateSubQuery: LogicalPlan = DeltaInsertNode(joinSubQuery, + isDirectInsert = false) + val updatePlan = Update(table, updateSubQuery, Seq.empty, table.output, + subQuery.output, isDeltaInsert = true) + val columnTableInsertPlan = ColumnTableInsert(table, insertPlan, updatePlan) + val columnTableInsertDS = new Dataset(sparkSession, columnTableInsertPlan, + RowEncoder(columnTableInsertPlan.schema)) + columnTableInsertDS.queryExecution.analyzed.asInstanceOf[ColumnTableInsert] + } else { + val modifiedInsertDS = new Dataset(sparkSession, insertPlan, + RowEncoder(insertPlan.schema)) + modifiedInsertDS.queryExecution.analyzed.asInstanceOf[Insert] + } + } + } + case _ => // Do nothing, original insert plan is enough + } + transFormedPlan + } def transformPutPlan(sparkSession: SparkSession, originalPlan: PutIntoTable): LogicalPlan = { validateOp(originalPlan) @@ -118,7 +183,8 @@ object ColumnTableBulkOps { private def prepareCondition(sparkSession: SparkSession, table: LogicalPlan, child: LogicalPlan, - columnNames: Seq[String]): Option[Expression] = { + columnNames: Seq[String], + changeCondition: Boolean = false): Option[Expression] = { val analyzer = sparkSession.sessionState.analyzer val leftKeys = columnNames.map { keyName => table.output.find(attr => analyzer.resolver(attr.name, keyName)).getOrElse { @@ -139,7 +205,12 @@ object ColumnTableBulkOps { } } val joinPairs = leftKeys.zip(rightKeys) - val newCondition = joinPairs.map(EqualTo.tupled).reduceOption(And) + val newCondition = if (changeCondition) { + val newCondition1 = joinPairs.map(EqualTo.tupled) + val newCondition2 = joinPairs.map(a => + org.apache.spark.sql.catalyst.expressions.Not(EqualTo(a._1, a._2))) + (newCondition1 ++ newCondition2).reduceOption(And) + } else joinPairs.map(EqualTo.tupled).reduceOption(And) newCondition } @@ -185,7 +256,7 @@ object ColumnTableBulkOps { } } -case class PutIntoColumnTable(table: LogicalPlan, +abstract class BasePutIntoColumnTable(table: LogicalPlan, insert: Insert, update: Update) extends BinaryNode { override lazy val output: Seq[Attribute] = AttributeReference( @@ -202,3 +273,14 @@ case class PutIntoColumnTable(table: LogicalPlan, override def right: LogicalPlan = insert } + +case class PutIntoColumnTable(table: LogicalPlan, insert: Insert, update: Update) extends + BasePutIntoColumnTable(table, insert, update) + +case class ColumnTableInsert(table: LogicalPlan, insert: Insert, update: Update) extends + BasePutIntoColumnTable(table, insert, update) + +case class DeltaInsertNode(child: LogicalPlan, isDirectInsert: Boolean) extends UnaryNode with + PredicateHelper { + override def output: Seq[Attribute] = child.output +} diff --git a/core/src/main/scala/org/apache/spark/sql/internal/SnappySessionState.scala b/core/src/main/scala/org/apache/spark/sql/internal/SnappySessionState.scala index 69b987a1fb..ea014f976f 100644 --- a/core/src/main/scala/org/apache/spark/sql/internal/SnappySessionState.scala +++ b/core/src/main/scala/org/apache/spark/sql/internal/SnappySessionState.scala @@ -576,7 +576,7 @@ class SnappySessionState(snappySession: SnappySession) case c: DMLExternalTable if !c.query.resolved => c.copy(query = analyzeQuery(c.query)) - case u@Update(table, child, keyColumns, updateCols, updateExprs) + case u@Update(table, child, keyColumns, updateCols, updateExprs, isDeltaInsert) if keyColumns.isEmpty && u.resolved && child.resolved => // add the key columns to the plan val (keyAttrs, newChild, relation) = getKeyAttributes(table, child, u) @@ -595,7 +595,7 @@ class SnappySessionState(snappySession: SnappySession) throw new AnalysisException(s"Could not resolve update column ${c.name}")) } val colName = Utils.toUpperCase(c.name) - if (nonUpdatableColumns.contains(colName)) { + if (!isDeltaInsert && nonUpdatableColumns.contains(colName)) { throw new AnalysisException("Cannot update partitioning/key column " + s"of the table for $colName (among [${nonUpdatableColumns.mkString(", ")}])") } @@ -635,6 +635,12 @@ class SnappySessionState(snappySession: SnappySession) ColumnTableBulkOps.transformDeletePlan(sparkSession, d) case p@PutIntoTable(_, child) if child.resolved => ColumnTableBulkOps.transformPutPlan(sparkSession, p) + case i@InsertIntoTable(table: LogicalPlan, _, child, _, _) if child.resolved + && (child find { + case d: DeltaInsertNode => true + case _ => false + }).isEmpty => + ColumnTableBulkOps.transformInsertPlan(sparkSession, i) } private def analyzeQuery(query: LogicalPlan): LogicalPlan = { @@ -1091,7 +1097,8 @@ class DefaultPlanner(val snappySession: SnappySession, conf: SQLConf, } private val storeOptimizedRules: Seq[Strategy] = - Seq(StoreDataSourceStrategy, SnappyAggregation, HashJoinStrategies) + Seq(StoreDataSourceStrategy, SnappyAggregation, HashJoinStrategies, + DeltaInsertOnSortMergeJoinStrategies) override def strategies: Seq[Strategy] = Seq(SnappyStrategies, diff --git a/core/src/main/scala/org/apache/spark/sql/row/JDBCMutableRelation.scala b/core/src/main/scala/org/apache/spark/sql/row/JDBCMutableRelation.scala index 15c664a63c..c6776e1208 100644 --- a/core/src/main/scala/org/apache/spark/sql/row/JDBCMutableRelation.scala +++ b/core/src/main/scala/org/apache/spark/sql/row/JDBCMutableRelation.scala @@ -95,6 +95,8 @@ case class JDBCMutableRelation( def partitionColumns: Seq[String] = Nil + override def getSortingOrder: String = "" + def partitionExpressions(relation: LogicalRelation): Seq[Expression] = Nil def numBuckets: Int = -1 @@ -210,7 +212,7 @@ case class JDBCMutableRelation( */ override def getUpdatePlan(relation: LogicalRelation, child: SparkPlan, updateColumns: Seq[Attribute], updateExpressions: Seq[Expression], - keyColumns: Seq[Attribute]): SparkPlan = { + keyColumns: Seq[Attribute], isDeltaInsert: Boolean): SparkPlan = { RowUpdateExec(child, resolvedName, partitionColumns, partitionExpressions(relation), numBuckets, isPartitioned, schema, Some(this), updateColumns, updateExpressions, keyColumns, connProperties, onExecutor = false) diff --git a/core/src/main/scala/org/apache/spark/sql/sources/StoreStrategy.scala b/core/src/main/scala/org/apache/spark/sql/sources/StoreStrategy.scala index f124cfc246..57e94c7c62 100644 --- a/core/src/main/scala/org/apache/spark/sql/sources/StoreStrategy.scala +++ b/core/src/main/scala/org/apache/spark/sql/sources/StoreStrategy.scala @@ -22,7 +22,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoTable, LogicalPlan import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.command.{ExecutedCommandExec, RunnableCommand} import org.apache.spark.sql.execution.datasources.{CreateTable, LogicalRelation} -import org.apache.spark.sql.internal.{BypassRowLevelSecurity, PutIntoColumnTable} +import org.apache.spark.sql.internal.{BypassRowLevelSecurity, PutIntoColumnTable, ColumnTableInsert} import org.apache.spark.sql.types.{DataType, LongType, StructType} import org.apache.spark.sql.{Strategy, _} @@ -131,10 +131,13 @@ object StoreStrategy extends Strategy { case PutIntoColumnTable(LogicalRelation(p: BulkPutRelation, _, _), left, right) => ExecutePlan(p.getPutPlan(planLater(left), planLater(right))) :: Nil + case ColumnTableInsert(LogicalRelation(p: ColumnTableInsertRelation, _, _), left, right) => + ExecutePlan(p.getColumnTableInsertPlan(planLater(left), planLater(right))) :: Nil + case Update(l@LogicalRelation(u: MutableRelation, _, _), child, - keyColumns, updateColumns, updateExpressions) => + keyColumns, updateColumns, updateExpressions, isDeltaInsert) => ExecutePlan(u.getUpdatePlan(l, planLater(child), updateColumns, - updateExpressions, keyColumns)) :: Nil + updateExpressions, keyColumns, isDeltaInsert = isDeltaInsert)) :: Nil case Delete(l@LogicalRelation(d: MutableRelation, _, _), child, keyColumns) => ExecutePlan(d.getDeletePlan(l, planLater(child), keyColumns)) :: Nil @@ -208,7 +211,8 @@ final class Insert( case class Update(table: LogicalPlan, child: LogicalPlan, keyColumns: Seq[Attribute], updateColumns: Seq[Attribute], - updateExpressions: Seq[Expression]) extends LogicalPlan with TableMutationPlan { + updateExpressions: Seq[Expression], isDeltaInsert: Boolean = false) extends LogicalPlan + with TableMutationPlan { assert(updateColumns.length == updateExpressions.length, s"Internal error: updateColumns=${updateColumns.length} " + diff --git a/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala b/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala index ef593ec0cd..d211d9cb4b 100644 --- a/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala +++ b/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala @@ -88,6 +88,11 @@ trait BulkPutRelation extends DestroyRelation { def getPutPlan(insertPlan: SparkPlan, updatePlan: SparkPlan): SparkPlan } +trait ColumnTableInsertRelation extends DestroyRelation { + + def getColumnTableInsertPlan(insertPlan: SparkPlan, updatePlan: SparkPlan): SparkPlan +} + @DeveloperApi trait SingleRowInsertableRelation { /** @@ -128,7 +133,7 @@ trait MutableRelation extends DestroyRelation { */ def getUpdatePlan(relation: LogicalRelation, child: SparkPlan, updateColumns: Seq[Attribute], updateExpressions: Seq[Expression], - keyColumns: Seq[Attribute]): SparkPlan + keyColumns: Seq[Attribute], isDeltaInsert: Boolean): SparkPlan /** * Get a spark plan to delete rows the relation. The result of SparkPlan @@ -136,6 +141,8 @@ trait MutableRelation extends DestroyRelation { */ def getDeletePlan(relation: LogicalRelation, child: SparkPlan, keyColumns: Seq[Attribute]): SparkPlan + + def getSortingOrder: String } /** diff --git a/core/src/main/scala/org/apache/spark/sql/store/StoreUtils.scala b/core/src/main/scala/org/apache/spark/sql/store/StoreUtils.scala index 48d9e75f19..72093b340d 100644 --- a/core/src/main/scala/org/apache/spark/sql/store/StoreUtils.scala +++ b/core/src/main/scala/org/apache/spark/sql/store/StoreUtils.scala @@ -73,6 +73,7 @@ object StoreUtils { val PRIMARY_KEY = "PRIMARY KEY" val LRUCOUNT = "LRUCOUNT" val GEM_INDEXED_TABLE = "INDEXED_TABLE" + val COLUMN_BATCH_SORTED = "COLUMN_BATCH_SORTED" // int values for Spark SQL types for efficient switching avoiding reflection val STRING_TYPE = 0 @@ -95,7 +96,7 @@ object StoreUtils { val ddlOptions: Seq[String] = Seq(PARTITION_BY, REPLICATE, BUCKETS, PARTITIONER, COLOCATE_WITH, REDUNDANCY, RECOVERYDELAY, MAXPARTSIZE, EVICTION_BY, PERSISTENCE, PERSISTENT, SERVER_GROUPS, EXPIRE, OVERFLOW, COMPRESSION_CODEC_DEPRECATED, - GEM_INDEXED_TABLE) ++ ExternalStoreUtils.ddlOptions + GEM_INDEXED_TABLE, COLUMN_BATCH_SORTED) ++ ExternalStoreUtils.ddlOptions val EMPTY_STRING = "" val NONE = "NONE" @@ -336,7 +337,8 @@ object StoreUtils { .asInstanceOf[SnappyStoreHiveCatalog] .normalizeSchema(schema) val schemaFields = Utils.schemaFields(normalizedSchema) - val cols = v.split(",") map (_.trim) + val partitioningCols = v.split("SORTING").toSeq.map(a => a.trim).head + val cols = partitioningCols.split(",") map (_.trim) // always use case-insensitive analysis for partitioning columns // since table creation can use case-insensitive in creation val normalizedCols = cols.map(Utils.toUpperCase) @@ -355,7 +357,7 @@ object StoreUtils { */ } if (includeInPK) { - s"$PRIMARY_KEY ($v, $ROWID_COLUMN_NAME)" + s"$PRIMARY_KEY ($partitioningCols, $ROWID_COLUMN_NAME)" } else { s"$PRIMARY_KEY ($ROWID_COLUMN_NAME)" } @@ -381,7 +383,9 @@ object StoreUtils { throw Utils.analysisException("Column table cannot be " + "partitioned on PRIMARY KEY as no primary key") } - case _ => s"sparkhash COLUMN($v)" + case _ => + val partitioningParams = v.split("SORTING").toSeq.map(a => a.trim) + s"sparkhash COLUMN(${partitioningParams.head})" } } s"$GEM_PARTITION_BY $parClause " @@ -493,10 +497,17 @@ object StoreUtils { } def getPartitioningColumns( - parameters: mutable.Map[String, String]): Seq[String] = { - parameters.get(PARTITION_BY).map(v => { - v.split(",").toSeq.map(a => a.trim) - }).getOrElse(Nil) + parameters: mutable.Map[String, String]): (Seq[String], String) = { + val partitioningParams = parameters.get(PARTITION_BY).map(v => { + v.split("SORTING").toSeq.map(a => a.trim) + }) + val partitioningCols = if (partitioningParams.isDefined) { + partitioningParams.get.head.split(",").toSeq.map(a => a.trim) + } else Nil + val sortingParams = if (partitioningParams.isDefined && partitioningParams.get.size > 1) { + partitioningParams.get.tail.head + } else "" + (partitioningCols, sortingParams) } def getColumnUpdateDeleteOrdering(batchIdColumn: Attribute): SortOrder = { @@ -553,4 +564,10 @@ object StoreUtils { } result } + + def isColumnBatchSortedAscending(columnTableSorting: String): Boolean = + columnTableSorting.equalsIgnoreCase("ASC") || columnTableSorting.equalsIgnoreCase("Ascending") + + def isColumnBatchSortedDescending(columnTableSorting: String): Boolean = + columnTableSorting.equalsIgnoreCase("DESC") || columnTableSorting.equalsIgnoreCase("Descending") } diff --git a/core/src/main/scala/org/apache/spark/util/MultiThreadedBenchmark.scala b/core/src/main/scala/org/apache/spark/util/MultiThreadedBenchmark.scala new file mode 100644 index 0000000000..e8d40a7c65 --- /dev/null +++ b/core/src/main/scala/org/apache/spark/util/MultiThreadedBenchmark.scala @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.util + +import java.io.{OutputStream, PrintStream} +import java.util +import java.util.concurrent.{Executors, ThreadLocalRandom} + +import scala.collection.mutable +import scala.collection.mutable.ArrayBuffer +import scala.concurrent.duration._ + +import org.apache.commons.io.output.TeeOutputStream + +import org.apache.spark.internal.Logging +import org.apache.spark.util.Benchmark.Result + +/** + * Copy of BenchMark for specific purpose + * + * Utility class to benchmark components. An example of how to use this is: + * val benchmark = new Benchmark("My Benchmark", valuesPerIteration) + * benchmark.addCase("V1")() + * benchmark.addCase("V2")() + * benchmark.run + * This will output the average time to run each function and the rate of each function. + * + * The benchmark function takes one argument that is the iteration that's being run. + * + * @param name name of this benchmark. + * @param valuesPerIteration number of values used in the test case, used to compute rows/s. + * @param minNumIters the min number of iterations that will be run per case, not counting warm-up. + * @param warmupTime amount of time to spend running dummy case iterations for JIT warm-up. + * @param minTime further iterations will be run for each case until this time is used up. + * @param outputPerIteration if true, the timing for each run will be printed to stdout. + * @param output optional output stream to write benchmark results to + */ +private[spark] class MultiThreadedBenchmark( + name: String, + isMultithreaded: Boolean, + valuesPerIteration: Long, + minNumIters: Int = 2, + numThreads: Int = 1, + warmupTime: FiniteDuration = 2.seconds, + minTime: FiniteDuration = 2.seconds, + outputPerIteration: Boolean = false, + output: Option[OutputStream] = None) extends Logging { + + import MultiThreadedBenchmark._ + val benchmarks = mutable.ArrayBuffer.empty[MultiThreadedBenchmark.Case] + val out = if (output.isDefined) { + new PrintStream(new TeeOutputStream(System.out, output.get)) + } else { + System.out + } + + /** + * Adds a case to run when run() is called. The given function will be run for several + * iterations to collect timing statistics. + * + * @param name of the benchmark case + * @param numIters if non-zero, forces exactly this many iterations to be run + */ + def addCase( + name: String, + numIters: Int = 0, + prepare: () => Unit = () => { }, + cleanup: () => Unit = () => { })(f: (Int, Int) => Boolean): Unit = { + val timedF = (timer: Benchmark.Timer, threadId: Int) => { + timer.startTiming() + val ret = f(timer.iteration, threadId) + timer.stopTiming() + ret + } + benchmarks += MultiThreadedBenchmark.Case(name, timedF, numIters, prepare, cleanup) + } + + /** + * Adds a case with manual timing control. When the function is run, timing does not start + * until timer.startTiming() is called within the given function. The corresponding + * timer.stopTiming() method must be called before the function returns. + * + * @param name of the benchmark case + * @param numIters if non-zero, forces exactly this many iterations to be run + */ + def addTimerCase(name: String, numIters: Int = 0)(f: (Benchmark.Timer, Int) => Boolean): Unit = { + benchmarks += MultiThreadedBenchmark.Case(name, f, numIters) + } + + /** + * Runs the benchmark and outputs the results to stdout. This should be copied and added as + * a comment with the benchmark. Although the results vary from machine to machine, it should + * provide some baseline. + */ + def run(): Unit = { + require(benchmarks.nonEmpty) + // scalastyle:off + println("Running benchmark: " + name) + + val results = benchmarks.map { c => + println(" Running case: " + c.name) + try { + c.prepare() + if (isMultithreaded) measureMultiThreaded(valuesPerIteration, c.numIters)(c.fn) + else measure(valuesPerIteration, c.numIters)(c.fn) + } finally { + c.cleanup() + } + } + println + + val firstBest = results.head.bestMs + // The results are going to be processor specific so it is useful to include that. + out.println(Benchmark.getJVMOSInfo()) + out.println(Benchmark.getProcessorName()) + out.printf("%-40s %16s %12s %13s %10s\n", name + ":", "Best/Avg Time(ms)", "Rate(M/s)", + "Per Row(ns)", "Relative") + out.println("-" * 96) + results.zip(benchmarks).foreach { case (result, benchmark) => + out.printf("%-40s %16s %12s %13s %10s\n", + benchmark.name, + "%5.0f / %4.0f" format (result.bestMs, result.avgMs), + "%10.1f" format result.bestRate, + "%6.1f" format (1000 / result.bestRate), + "%3.1fX" format (firstBest / result.bestMs)) + } + out.println + // scalastyle:on + } + + /** + * Runs a single function `f` for iters, returning the average time the function took and + * the rate of the function. + */ + def measure(num: Long, overrideNumIters: Int)(f: (Benchmark.Timer, Int) => Boolean): Result = { + System.gc() // ensures garbage from previous cases don't impact this one + val warmupDeadline = warmupTime.fromNow + while (!warmupDeadline.isOverdue) { + f(new Benchmark.Timer(-1), 0) + } + val minIters = if (overrideNumIters != 0) overrideNumIters else minNumIters + val minDuration = if (overrideNumIters != 0) 0 else minTime.toNanos + val runTimes = ArrayBuffer[Long]() + var i = 0 + while (i < minIters || runTimes.sum < minDuration) { + var j = 1 + while (j < 101) { + val timer = new Benchmark.Timer(i) + val ret = f(timer, 0) + val runTime = timer.totalTime() + if (ret || j == 100) { + runTimes += runTime + if (outputPerIteration) { + // scalastyle:off + println(s"Iteration $i took ${runTime / 1000} microseconds") + // scalastyle:on + } + if (j == 100) { + setRandomValues(valuesPerIteration) + } + j = 101 + } else { + setRandomValues(valuesPerIteration) + if (outputPerIteration) { + // scalastyle:off + println(s"Iteration $i attempt $j failed") + // scalastyle:on + } + } + j += 1 + } + i += 1 + } + // scalastyle:off + println(s" Stopped after $i iterations, ${runTimes.sum / 1000000} ms") + // scalastyle:on + val best = runTimes.min + val avg = runTimes.sum / runTimes.size + Result(avg / 1000000.0, num / (avg / 1000.0), best / 1000000.0) + } + + /** + * Runs a single function `f` for minDuration time (though slight misnomer), + * returning total number of times the function took, that will be printed. + * Ignore returned Result. + */ + def measureMultiThreaded(num: Long, overrideNumIters: Int) + (f: (Benchmark.Timer, Int) => Boolean): Result = { + System.gc() // ensures garbage from previous cases don't impact this one + val warmupDeadline = warmupTime.fromNow + while (!warmupDeadline.isOverdue) { + f(new Benchmark.Timer(-1), 0) + } + + val numIters = if (overrideNumIters > 0) overrideNumIters else minNumIters + val timerList = new Array[Benchmark.Timer](numIters) + timerList.indices.foreach(i => { + timerList(i) = new Benchmark.Timer(i) + }) + + // numThreads threads will be executed for minTime + val numFuncExecuted = new Array[Int](numThreads) + val prematureExit = new Array[Boolean](numThreads) + numFuncExecuted.indices.foreach(numFuncExecuted(_) = 0) + val executorPool = Executors.newFixedThreadPool(numFuncExecuted.length) + val futures = new Array[util.concurrent.Future[_]](numFuncExecuted.length) + numFuncExecuted.indices.foreach(threadId => { + val runnable = new Runnable { + override def run(): Unit = { + while (true) { + try { + val i = (numFuncExecuted(threadId) + threadId) % numIters + f(timerList(i), threadId) + numFuncExecuted(threadId) += 1 + // scalastyle:off + // println(s"while-true $threadId $i ${numFuncExecuted(threadId)}") + // scalastyle:on + } catch { + case _: InterruptedException => + logError(s"$threadId got InterruptedException") + return + case t: Throwable => + prematureExit(threadId) = true + logError(s"$threadId" + t.getMessage, t) + return + } + } + } + } + futures(threadId) = executorPool.submit(runnable) + None + }) + Thread.sleep(minTime.toMillis) + futures.foreach(f => { + f.cancel(true) + }) + + // scalastyle:off + prematureExit.indices.foreach(i => if (prematureExit(i)) println(s"Thread $i failed")) + println(s" Stopped $minTime, Query ran ${numFuncExecuted.sum} times with $numThreads threads") + numFuncExecuted.indices.foreach(i => { + println(s" Individual threads-$i function count ${numFuncExecuted(i)}") + }) + // scalastyle:on + + prematureExit.foreach(b => assert(!b)) + val best = numFuncExecuted.min + val avg = numFuncExecuted.sum / numFuncExecuted.length + Result(avg, num / avg, best) + } +} + +private[spark] object MultiThreadedBenchmark { + case class Case( + name: String, + fn: (Benchmark.Timer, Int) => Boolean, + numIters: Int, + prepare: () => Unit = () => { }, + cleanup: () => Unit = () => { }) + + var firstRandomValue = getFirstRandomValue(10) + var secondRandomValue = getSecondRandomValue(10) + def setRandomValues(valuesPerIteration: Long) : Unit = { + firstRandomValue = getFirstRandomValue(valuesPerIteration) + secondRandomValue = getSecondRandomValue(valuesPerIteration) + } + def getFirstRandomValue(valuesPerIteration: Long) : Long = + ThreadLocalRandom.current().nextLong(valuesPerIteration) + def getSecondRandomValue(valuesPerIteration: Long) : Long = + ThreadLocalRandom.current().nextLong(valuesPerIteration) +} diff --git a/store b/store index 089ac72822..7817f78a3f 160000 --- a/store +++ b/store @@ -1 +1 @@ -Subproject commit 089ac72822e63520a5dee6cca98a1b02c0487894 +Subproject commit 7817f78a3fb974b098b63d0fbd54392e0b0e8b55