From 557b03f9a82cfa1d43667b07c8651f340c8323e3 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Fri, 28 Dec 2018 16:41:51 +0530 Subject: [PATCH 1/3] Added system procedure 'GET_TABLE_SIZE(tableName)' Use: 1. to get specified table size 2. further to be used to calculate memory of a table from external table --- .../ddl/catalog/GfxdSystemProcedures.java | 99 ++++++++++++++++- .../SnappyRegionStatsCollectorFunction.java | 103 +++++++++++++----- .../impl/sql/catalog/GfxdDataDictionary.java | 11 ++ 3 files changed, 181 insertions(+), 32 deletions(-) diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java index 93954afd4..7d0ba9469 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java @@ -27,10 +27,7 @@ import java.sql.SQLException; import java.sql.Types; import java.util.*; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; +import java.util.concurrent.*; import javax.annotation.Nonnull; import com.gemstone.gemfire.DataSerializer; @@ -41,6 +38,7 @@ import com.gemstone.gemfire.cache.TransactionException; import com.gemstone.gemfire.cache.control.RebalanceOperation; import com.gemstone.gemfire.cache.control.ResourceManager; +import com.gemstone.gemfire.cache.execute.FunctionException; import com.gemstone.gemfire.cache.execute.FunctionService; import com.gemstone.gemfire.distributed.DistributedMember; import com.gemstone.gemfire.distributed.internal.ServerLocation; @@ -70,6 +68,7 @@ import com.pivotal.gemfirexd.internal.engine.ddl.catalog.messages.GfxdSystemProcedureMessage; import com.pivotal.gemfirexd.internal.engine.ddl.resolver.GfxdPartitionByExpressionResolver; import com.pivotal.gemfirexd.internal.engine.ddl.wan.messages.AbstractGfxdReplayableMessage; +import com.pivotal.gemfirexd.internal.engine.diag.SnappyTableStatsVTI; import com.pivotal.gemfirexd.internal.engine.distributed.AckResultCollector; import com.pivotal.gemfirexd.internal.engine.distributed.ByteArrayDataOutput; import com.pivotal.gemfirexd.internal.engine.distributed.GfxdDistributionAdvisor; @@ -86,6 +85,7 @@ import com.pivotal.gemfirexd.internal.engine.store.CustomRowsResultSet; import com.pivotal.gemfirexd.internal.engine.store.GemFireContainer; import com.pivotal.gemfirexd.internal.engine.store.GemFireStore; +import com.pivotal.gemfirexd.internal.engine.ui.SnappyRegionStats; import com.pivotal.gemfirexd.internal.iapi.db.PropertyInfo; import com.pivotal.gemfirexd.internal.iapi.error.PublicAPI; import com.pivotal.gemfirexd.internal.iapi.error.StandardException; @@ -121,8 +121,12 @@ import com.pivotal.gemfirexd.internal.shared.common.sanity.SanityManager; import com.pivotal.gemfirexd.internal.snappy.LeadNodeSmartConnectorOpContext; import com.pivotal.gemfirexd.load.Import; +import com.pivotal.gemfirexd.internal.engine.ui.SnappyRegionStatsCollectorFunction; +import com.pivotal.gemfirexd.internal.engine.ui.SnappyRegionStatsCollectorResult; +//import com.pivotal.gemfirexd.internal.engine.distributed; import io.snappydata.thrift.ServerType; import io.snappydata.thrift.internal.ClientBlob; +import org.apache.commons.collections.bag.SynchronizedSortedBag; /** * GemFireXD built-in system procedures that will get executed on every @@ -690,7 +694,7 @@ public boolean getNext(DataValueDescriptor[] template) private static final ResultColumnDescriptor[] encryptColumnInfo = { EmbedResultSetMetaData.getResultColumnDescriptor("ENCRYPTED_PASSWORD", Types.VARCHAR, - false, Limits.DB2_VARCHAR_MAXWIDTH), + false, Limits.DB2_VARCHAR_MAXWIDTH) }; @@ -3242,4 +3246,87 @@ private static void rollBackAndThrowSQLException(Connection conn, } throw se; } -} + + public static void GET_TABLE_SIZE(String tableName, ResultSet[] tableSize) throws SQLException { + if (GemFireXDUtils.TraceSysProcedures) { + SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_SYS_PROCEDURES, + "executing GET_TABLE_SIZE "); + } + // SnappyTableStats + List result = new java.util.ArrayList<>(); + + Set dataServers = GfxdMessage.getAllDataStores(); + try { + + if (dataServers != null && dataServers.size() > 0) { + Map args = new HashMap(); + args.put("TABLE_NAME", tableName); + result = (ArrayList) FunctionService.onMembers(dataServers) + .withArgs(args) + .execute(SnappyRegionStatsCollectorFunction.ID).getResult(1, TimeUnit.SECONDS); + + System.out.println("AB: result set size " + result.size()); + List stats = ((SnappyRegionStatsCollectorResult) result.get(0)).getRegionStats(); + System.out.println("AB: SnappyRegionStats list size " + stats.size()); + + SnappyRegionStats statsResult = stats.get(0); + final long totalTableSize = statsResult.getTotalSize(); + final long inMemoryTableSize = statsResult.getSizeInMemory(); + Boolean resultFetched = true; + final CustomRowsResultSet.FetchDVDRows fetchRows = + new CustomRowsResultSet.FetchDVDRows() { + Boolean resultFetched = false; + + @Override + public boolean getNext(DataValueDescriptor[] template) + throws SQLException, StandardException { + if (!resultFetched) { + if (totalTableSize > 104857.6) { + float totalTableSizeInMB = (float) totalTableSize / 1048576; + template[0].setValue(tableName + " = " + totalTableSizeInMB + " MB"); + } else if (totalTableSize > 102.4) { + float totalTableSizeInKB = (float) totalTableSize / 1024; + template[0].setValue(tableName + " = " + totalTableSizeInKB + " KB"); + } else { + template[0].setValue(tableName + " = " + totalTableSize + " Bytes"); + } + + if (inMemoryTableSize > 104857.6) { + float inMemoryTableSizeInMB = (float) inMemoryTableSize / 1048576; + template[1].setValue(tableName + " = " + inMemoryTableSizeInMB + " MB"); + } else if (inMemoryTableSize > 102.4) { + float inMemoryTableSizeInKB = (float) inMemoryTableSize / 1024; + template[1].setValue(tableName + " = " + inMemoryTableSizeInKB + " KB"); + } else { + template[1].setValue(tableName + " = " + inMemoryTableSize + " Bytes"); + } + resultFetched = true; + return true; + } + return false; + } + }; + if (!result.isEmpty()) { + tableSize[0] = new CustomRowsResultSet(fetchRows, tableSizeInfo); + } else { + tableSize[0] = null; + } + } + } catch (InterruptedException e) { + e.printStackTrace(); // todo remove printStackTrace + } catch (StandardException se) { + se.printStackTrace(); + } + + } + + + private static final ResultColumnDescriptor[] tableSizeInfo = { + EmbedResultSetMetaData.getResultColumnDescriptor("TOTAL_TABLE_SIZE", Types.VARCHAR, + false, Limits.DB2_VARCHAR_MAXWIDTH), + EmbedResultSetMetaData.getResultColumnDescriptor("IN_MEMORY_TABLE_SIZE", Types.VARCHAR, + false, Limits.DB2_VARCHAR_MAXWIDTH) + }; + + +} \ No newline at end of file diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java index d7351369d..487561998 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java @@ -31,6 +31,9 @@ import com.gemstone.gemfire.management.internal.SystemManagementService; import com.pivotal.gemfirexd.internal.engine.Misc; import com.pivotal.gemfirexd.internal.engine.access.index.GfxdIndexManager; +import com.pivotal.gemfirexd.internal.engine.ddl.DDLConflatable; +import com.pivotal.gemfirexd.internal.engine.ddl.callbacks.CallbackProcedures; +import com.pivotal.gemfirexd.internal.engine.distributed.utils.GemFireXDUtils; import com.pivotal.gemfirexd.internal.engine.store.GemFireContainer; import com.pivotal.gemfirexd.internal.iapi.error.StandardException; import com.pivotal.gemfirexd.tools.sizer.GemFireXDInstrumentation; @@ -66,55 +69,75 @@ public void execute(FunctionContext context) { com.pivotal.gemfirexd.internal.snappy.CallbackFactoryProvider.getClusterCallbacks(). publishColumnTableStats(); SnappyRegionStatsCollectorResult result = new SnappyRegionStatsCollectorResult(); - Map cachBatchStats = new HashMap<>(); + Map cachedBatchStats = new HashMap<>(); ArrayList otherStats = new ArrayList<>(); - - try { - List containers = Misc.getMemStore().getAllContainers(); - final SystemManagementService managementService = - (SystemManagementService)ManagementService.getManagementService(Misc.getGemFireCache()); - - for (GemFireContainer container : containers) { - if (container.isApplicationTable()) { - LocalRegion r = container.getRegion(); - if (managementService != null && r != null ) { - RegionMXBean bean = managementService.getLocalRegionMBean(r.getFullPath()); - if (bean != null && !(r.getFullPath().startsWith( - "/" + Misc.SNAPPY_HIVE_METASTORE + '/'))) { - SnappyRegionStats dataCollector = collectDataFromBean(r, bean); - if (dataCollector.isColumnTable()) { - cachBatchStats.put(dataCollector.getTableName(), dataCollector); - } else { - otherStats.add(dataCollector); + (SystemManagementService) ManagementService.getManagementService(Misc.getGemFireCache()); + + if (context.getArguments() != null && context.getArguments() instanceof HashMap + && ((HashMap) context.getArguments()).containsKey("TABLE_NAME") + && !((HashMap) context.getArguments()).get("TABLE_NAME").isEmpty()) { + String tname = ((HashMap) context.getArguments()).get("TABLE_NAME"); + // Make it mandatory to pass the schema name as well. + String[] names = tname.split("\\."); + if (names.length == 2) { + + GemFireContainer container = CallbackProcedures.getContainerForTable(names[0], names[1]); + getSnappyRegionStats(otherStats, cachedBatchStats, container, managementService); + StoreCallbacks callback = CallbackFactoryProvider.getStoreCallbacks(); + String columnBatchTableName = callback.columnBatchTableName(otherStats.get(0).getTableName()); + System.out.println("AB: Looking for CB table " + columnBatchTableName); + try { + container = CallbackProcedures.getContainerForTable(names[0], + columnBatchTableName.substring(columnBatchTableName.indexOf(".") + 1)); + getSnappyRegionStats(otherStats, cachedBatchStats, container, managementService); + } catch (StandardException se) { + } + + } + } else { + List containers = Misc.getMemStore().getAllContainers(); + + for (GemFireContainer container : containers) { + if (container.isApplicationTable()) { + LocalRegion r = container.getRegion(); + if (managementService != null && r != null) { + RegionMXBean bean = managementService.getLocalRegionMBean(r.getFullPath()); + if (bean != null && !(r.getFullPath().startsWith( + "/" + Misc.SNAPPY_HIVE_METASTORE + '/'))) { + SnappyRegionStats dataCollector = collectDataFromBean(r, bean); + if (dataCollector.isColumnTable()) { + cachedBatchStats.put(dataCollector.getTableName(), dataCollector); + } else { + otherStats.add(dataCollector); + } } } - } /* if(!LocalRegion.isMetaTable(r.getFullPath())){ result.addAllIndexStat(getIndexStatForContainer(container)); } */ + } } } - if (Misc.reservoirRegionCreated) { for (SnappyRegionStats tableStats : otherStats) { String tableName = tableStats.getTableName(); StoreCallbacks callback = CallbackFactoryProvider.getStoreCallbacks(); String columnBatchTableName = callback.columnBatchTableName(tableName); - if (cachBatchStats.containsKey(columnBatchTableName)) { + if (cachedBatchStats.containsKey(columnBatchTableName)) { String reservoirRegionName = Misc.getReservoirRegionNameForSampleTable("APP", tableName); PartitionedRegion pr = Misc.getReservoirRegionForSampleTable(reservoirRegionName); if (managementService != null && pr != null) { RegionMXBean reservoirBean = managementService.getLocalRegionMBean(pr.getFullPath()); if (reservoirBean != null) { SnappyRegionStats rStats = collectDataFromBeanImpl(pr, reservoirBean, true); - SnappyRegionStats cStats = cachBatchStats.get(columnBatchTableName); - cachBatchStats.put(columnBatchTableName, cStats.getCombinedStats(rStats)); + SnappyRegionStats cStats = cachedBatchStats.get(columnBatchTableName); + cachedBatchStats.put(columnBatchTableName, cStats.getCombinedStats(rStats)); } } } @@ -122,22 +145,50 @@ public void execute(FunctionContext context) { } // Create one entry per Column Table by combining the results of row buffer and column table + for (SnappyRegionStats tableStats : otherStats) { StoreCallbacks callback = CallbackFactoryProvider.getStoreCallbacks(); String columnBatchTableName = callback.columnBatchTableName(tableStats.getTableName()); - if (cachBatchStats.containsKey(columnBatchTableName)) { - result.addRegionStat(tableStats.getCombinedStats(cachBatchStats.get(columnBatchTableName))); + if (cachedBatchStats.containsKey(columnBatchTableName)) { + System.out.println("AB: Returning combined stats for table " + columnBatchTableName); + result.addRegionStat(tableStats.getCombinedStats(cachedBatchStats.get(columnBatchTableName))); } else { + System.out.println("AB: Returning stats for table " + columnBatchTableName); result.addRegionStat(tableStats); } } } catch (CacheClosedException ignored) { + } catch (StandardException ignored) { + System.out.println("AB: " + ignored); } finally { context.getResultSender().lastResult(result); } } + private void getSnappyRegionStats(ArrayList otherStats, + Map cacheBatchStats, + GemFireContainer container, + SystemManagementService managementService) { + if (container.isApplicationTable()) { + LocalRegion r = container.getRegion(); + if (managementService != null && r != null) { + RegionMXBean bean = managementService.getLocalRegionMBean(r.getFullPath()); + if (bean != null && !(r.getFullPath().startsWith( + "/" + Misc.SNAPPY_HIVE_METASTORE + '/'))) { + SnappyRegionStats dataCollector = collectDataFromBean(r, bean); + if (dataCollector.isColumnTable()) { + System.out.println("AB: Adding stats for table " + dataCollector.getTableName()); + cacheBatchStats.put(dataCollector.getTableName(), dataCollector); + } else { + System.out.println("AB: Adding stats for table " + dataCollector.getTableName()); + otherStats.add(dataCollector); + } + } + } + } + } + private SnappyRegionStats collectDataFromBean(LocalRegion lr, RegionMXBean bean) { return collectDataFromBeanImpl(lr, bean, false); } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java index ab4479373..1794a51cb 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java @@ -2092,6 +2092,17 @@ private void createGfxdSystemProcedures(TransactionController tc, arg_names, arg_types, 0, 1, RoutineAliasInfo.READS_SQL_DATA, null, newlyCreatedRoutines, tc, GFXD_SYS_PROC_CLASSNAME, false); } + { + // GET_TABLE_SIZE + String[] arg_names = new String[]{"TABLE_NAME"}; + TypeDescriptor[] arg_types = new TypeDescriptor[]{ + DataTypeDescriptor.getCatalogType(Types.VARCHAR) + }; + super.createSystemProcedureOrFunction("GET_TABLE_SIZE", sysUUID, + arg_names, arg_types, 0, 1, RoutineAliasInfo.NO_SQL, null, + newlyCreatedRoutines, tc, GFXD_SYS_PROC_CLASSNAME, false); + } + } @SuppressWarnings({ "unchecked", "rawtypes" }) From 50b7d55afd78e362ed7002ec0c80e5fdabc00125 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Tue, 1 Jan 2019 15:55:18 +0530 Subject: [PATCH 2/3] Sampling external table into column table Modified system procedure 'GET_TABLE_SIZE(String tableName,Boolean isExternalTable ,Int SamplePercentage)' --- .../ddl/catalog/GfxdSystemProcedures.java | 31 ++++++++++++------- .../SnappyRegionStatsCollectorFunction.java | 6 ---- .../impl/sql/catalog/GfxdDataDictionary.java | 6 ++-- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java index 7d0ba9469..03c9c8357 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java @@ -3247,36 +3247,45 @@ private static void rollBackAndThrowSQLException(Connection conn, throw se; } - public static void GET_TABLE_SIZE(String tableName, ResultSet[] tableSize) throws SQLException { + public static void GET_TABLE_SIZE(String tableName, Boolean isExternalTable, int samplePercentage, ResultSet[] tableSize) throws SQLException { if (GemFireXDUtils.TraceSysProcedures) { SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_SYS_PROCEDURES, "executing GET_TABLE_SIZE "); } - // SnappyTableStats List result = new java.util.ArrayList<>(); - Set dataServers = GfxdMessage.getAllDataStores(); try { if (dataServers != null && dataServers.size() > 0) { Map args = new HashMap(); - args.put("TABLE_NAME", tableName); + if(isExternalTable) { + //create column table with same name + float sample = (float)samplePercentage/100; + String sampleTable=tableName+"_sample"; + Connection conn=getDefaultConn(); + conn.createStatement().execute("create table "+sampleTable+" using column as(select * from "+tableName+" where rand() < "+sample+" );"); + args.put("TABLE_NAME", sampleTable); + } + else { + args.put("TABLE_NAME", tableName); + samplePercentage=100; + } result = (ArrayList) FunctionService.onMembers(dataServers) .withArgs(args) .execute(SnappyRegionStatsCollectorFunction.ID).getResult(1, TimeUnit.SECONDS); - System.out.println("AB: result set size " + result.size()); List stats = ((SnappyRegionStatsCollectorResult) result.get(0)).getRegionStats(); - System.out.println("AB: SnappyRegionStats list size " + stats.size()); - SnappyRegionStats statsResult = stats.get(0); - final long totalTableSize = statsResult.getTotalSize(); - final long inMemoryTableSize = statsResult.getSizeInMemory(); + long sampleTableSize = statsResult.getTotalSize(); + long sampleInMemoryTableSize = statsResult.getSizeInMemory(); + + final long totalTableSize=(sampleTableSize*100)/samplePercentage; + final long inMemoryTableSize=(sampleInMemoryTableSize*100)/samplePercentage; + Boolean resultFetched = true; final CustomRowsResultSet.FetchDVDRows fetchRows = new CustomRowsResultSet.FetchDVDRows() { Boolean resultFetched = false; - @Override public boolean getNext(DataValueDescriptor[] template) throws SQLException, StandardException { @@ -3317,10 +3326,8 @@ public boolean getNext(DataValueDescriptor[] template) } catch (StandardException se) { se.printStackTrace(); } - } - private static final ResultColumnDescriptor[] tableSizeInfo = { EmbedResultSetMetaData.getResultColumnDescriptor("TOTAL_TABLE_SIZE", Types.VARCHAR, false, Limits.DB2_VARCHAR_MAXWIDTH), diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java index 487561998..e17740d28 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java @@ -87,7 +87,6 @@ public void execute(FunctionContext context) { getSnappyRegionStats(otherStats, cachedBatchStats, container, managementService); StoreCallbacks callback = CallbackFactoryProvider.getStoreCallbacks(); String columnBatchTableName = callback.columnBatchTableName(otherStats.get(0).getTableName()); - System.out.println("AB: Looking for CB table " + columnBatchTableName); try { container = CallbackProcedures.getContainerForTable(names[0], columnBatchTableName.substring(columnBatchTableName.indexOf(".") + 1)); @@ -150,16 +149,13 @@ public void execute(FunctionContext context) { StoreCallbacks callback = CallbackFactoryProvider.getStoreCallbacks(); String columnBatchTableName = callback.columnBatchTableName(tableStats.getTableName()); if (cachedBatchStats.containsKey(columnBatchTableName)) { - System.out.println("AB: Returning combined stats for table " + columnBatchTableName); result.addRegionStat(tableStats.getCombinedStats(cachedBatchStats.get(columnBatchTableName))); } else { - System.out.println("AB: Returning stats for table " + columnBatchTableName); result.addRegionStat(tableStats); } } } catch (CacheClosedException ignored) { } catch (StandardException ignored) { - System.out.println("AB: " + ignored); } finally { context.getResultSender().lastResult(result); } @@ -178,10 +174,8 @@ private void getSnappyRegionStats(ArrayList otherStats, "/" + Misc.SNAPPY_HIVE_METASTORE + '/'))) { SnappyRegionStats dataCollector = collectDataFromBean(r, bean); if (dataCollector.isColumnTable()) { - System.out.println("AB: Adding stats for table " + dataCollector.getTableName()); cacheBatchStats.put(dataCollector.getTableName(), dataCollector); } else { - System.out.println("AB: Adding stats for table " + dataCollector.getTableName()); otherStats.add(dataCollector); } } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java index 1794a51cb..81b95083e 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java @@ -2094,9 +2094,11 @@ private void createGfxdSystemProcedures(TransactionController tc, } { // GET_TABLE_SIZE - String[] arg_names = new String[]{"TABLE_NAME"}; + String[] arg_names = new String[]{"TABLE_NAME","IS_EXTERNAL_TABLE","SAMPLE_PERCENTAGE"}; TypeDescriptor[] arg_types = new TypeDescriptor[]{ - DataTypeDescriptor.getCatalogType(Types.VARCHAR) + DataTypeDescriptor.getCatalogType(Types.VARCHAR), + DataTypeDescriptor.getCatalogType(Types.BOOLEAN), + DataTypeDescriptor.getCatalogType(Types.INTEGER) }; super.createSystemProcedureOrFunction("GET_TABLE_SIZE", sysUUID, arg_names, arg_types, 0, 1, RoutineAliasInfo.NO_SQL, null, From 94ccef9a524de9857de74615b0ff8f45b8503465 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Wed, 2 Jan 2019 14:39:40 +0530 Subject: [PATCH 3/3] updated procedure with java docs --- .../ddl/catalog/GfxdSystemProcedures.java | 52 +++++++++++++------ .../SnappyRegionStatsCollectorFunction.java | 4 +- 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java index 03c9c8357..8bbf22996 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java @@ -3247,6 +3247,26 @@ private static void rollBackAndThrowSQLException(Connection conn, throw se; } + /** + * given a row/column table, method returns size of table with samplePercentage=100%, + * if given an external table and sample percentage, method creates a column table with + * sample percentage amount and extrapolates memory for loading whole(100% data) + * external table as column table. + * + * Cases(Exceptions) not handled: + * Case 1: given a row/column table and isExternalTable=TRUE then it doesn't + * show an error instead it creates another column table with sample amount. + * + * Case 2: given an external table with isExternalTable=FALSE or + * given any table which is not present then it gives error but need to show + * an error message as table not found. + * + * @param tableName table name with schema(fully qualified table name) + * @param isExternalTable true for external table else false + * @param samplePercentage sampling percentage (Integer :0 to 100) + * @param tableSize returns resultset + * @throws SQLException + */ public static void GET_TABLE_SIZE(String tableName, Boolean isExternalTable, int samplePercentage, ResultSet[] tableSize) throws SQLException { if (GemFireXDUtils.TraceSysProcedures) { SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_SYS_PROCEDURES, @@ -3258,17 +3278,16 @@ public static void GET_TABLE_SIZE(String tableName, Boolean isExternalTable, int if (dataServers != null && dataServers.size() > 0) { Map args = new HashMap(); - if(isExternalTable) { - //create column table with same name - float sample = (float)samplePercentage/100; - String sampleTable=tableName+"_sample"; - Connection conn=getDefaultConn(); - conn.createStatement().execute("create table "+sampleTable+" using column as(select * from "+tableName+" where rand() < "+sample+" );"); + if (isExternalTable) { + float sample = (float) samplePercentage / 100; + String sampleTable = tableName + "_sample"; + Connection conn = getDefaultConn(); + conn.createStatement().execute("drop table if exists " + sampleTable + " ;"); + conn.createStatement().execute("create table " + sampleTable + " using column as(select * from " + tableName + " where rand() < " + sample + " );"); args.put("TABLE_NAME", sampleTable); - } - else { + } else { args.put("TABLE_NAME", tableName); - samplePercentage=100; + samplePercentage = 100; } result = (ArrayList) FunctionService.onMembers(dataServers) .withArgs(args) @@ -3276,16 +3295,17 @@ public static void GET_TABLE_SIZE(String tableName, Boolean isExternalTable, int List stats = ((SnappyRegionStatsCollectorResult) result.get(0)).getRegionStats(); SnappyRegionStats statsResult = stats.get(0); - long sampleTableSize = statsResult.getTotalSize(); - long sampleInMemoryTableSize = statsResult.getSizeInMemory(); + long sampleTableSize = statsResult.getTotalSize(); + long sampleInMemoryTableSize = statsResult.getSizeInMemory(); - final long totalTableSize=(sampleTableSize*100)/samplePercentage; - final long inMemoryTableSize=(sampleInMemoryTableSize*100)/samplePercentage; + final long totalTableSize = (sampleTableSize * 100) / samplePercentage; + final long inMemoryTableSize = (sampleInMemoryTableSize * 100) / samplePercentage; Boolean resultFetched = true; final CustomRowsResultSet.FetchDVDRows fetchRows = new CustomRowsResultSet.FetchDVDRows() { Boolean resultFetched = false; + @Override public boolean getNext(DataValueDescriptor[] template) throws SQLException, StandardException { @@ -3322,16 +3342,16 @@ public boolean getNext(DataValueDescriptor[] template) } } } catch (InterruptedException e) { - e.printStackTrace(); // todo remove printStackTrace + e.printStackTrace(); } catch (StandardException se) { se.printStackTrace(); } } private static final ResultColumnDescriptor[] tableSizeInfo = { - EmbedResultSetMetaData.getResultColumnDescriptor("TOTAL_TABLE_SIZE", Types.VARCHAR, + EmbedResultSetMetaData.getResultColumnDescriptor("TOTAL TABLE SIZE", Types.VARCHAR, false, Limits.DB2_VARCHAR_MAXWIDTH), - EmbedResultSetMetaData.getResultColumnDescriptor("IN_MEMORY_TABLE_SIZE", Types.VARCHAR, + EmbedResultSetMetaData.getResultColumnDescriptor("IN MEMORY TABLE SIZE", Types.VARCHAR, false, Limits.DB2_VARCHAR_MAXWIDTH) }; diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java index e17740d28..c25ea8bfb 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java @@ -79,7 +79,6 @@ public void execute(FunctionContext context) { && ((HashMap) context.getArguments()).containsKey("TABLE_NAME") && !((HashMap) context.getArguments()).get("TABLE_NAME").isEmpty()) { String tname = ((HashMap) context.getArguments()).get("TABLE_NAME"); - // Make it mandatory to pass the schema name as well. String[] names = tname.split("\\."); if (names.length == 2) { @@ -94,11 +93,12 @@ public void execute(FunctionContext context) { } catch (StandardException se) { } - } + } // TODO else get default or current schemaname and proceed } else { List containers = Misc.getMemStore().getAllContainers(); for (GemFireContainer container : containers) { + //TODO replace block with method getSnappyRegionStats() if (container.isApplicationTable()) { LocalRegion r = container.getRegion(); if (managementService != null && r != null) {