diff --git a/.classpath b/.classpath index b078f0c2..870518be 100644 --- a/.classpath +++ b/.classpath @@ -23,5 +23,6 @@ + diff --git a/lib/RedshiftJDBC4-1.1.10.1010.jar b/lib/RedshiftJDBC4-1.1.10.1010.jar new file mode 100644 index 00000000..52b1cf81 Binary files /dev/null and b/lib/RedshiftJDBC4-1.1.10.1010.jar differ diff --git a/src/org/ohdsi/databases/DbType.java b/src/org/ohdsi/databases/DbType.java index bf6a15ae..1ec447f7 100644 --- a/src/org/ohdsi/databases/DbType.java +++ b/src/org/ohdsi/databases/DbType.java @@ -23,9 +23,10 @@ public class DbType { public static DbType ORACLE = new DbType("oracle"); public static DbType POSTGRESQL = new DbType("postgresql"); public static DbType MSACCESS = new DbType("msaccess"); + public static DbType REDSHIFT = new DbType("redshift"); private enum Type { - MYSQL, MSSQL, ORACLE, POSTGRESQL, MSACCESS + MYSQL, MSSQL, ORACLE, POSTGRESQL, MSACCESS, REDSHIFT }; private Type type; diff --git a/src/org/ohdsi/databases/RichConnection.java b/src/org/ohdsi/databases/RichConnection.java index b07ec676..3fcede20 100644 --- a/src/org/ohdsi/databases/RichConnection.java +++ b/src/org/ohdsi/databases/RichConnection.java @@ -56,10 +56,10 @@ public RichConnection(String server, String domain, String user, String password * @param sql */ public void execute(String sql) { + Statement statement = null; try { if (sql.length() == 0) return; - Statement statement = null; statement = connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); for (String subQuery : sql.split(";")) { @@ -78,10 +78,19 @@ public void execute(String sql) { statement.executeBatch(); if (verbose) outputQueryStats(statement, System.currentTimeMillis() - start); - statement.close(); } catch (SQLException e) { System.err.println(sql); e.printStackTrace(); + } finally { + if (statement != null) { + try { + statement.close(); + } + catch (SQLException e) { + // TODO Auto-generated catch block + System.err.println(e.getMessage()); + } + } } } @@ -121,7 +130,7 @@ public void use(String database) { return; if (dbType == DbType.ORACLE) execute("ALTER SESSION SET current_schema = " + database); - else if (dbType == DbType.POSTGRESQL) + else if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT) execute("SET search_path TO " + database); else if (dbType == DbType.MSACCESS) ; @@ -135,11 +144,11 @@ public List getTableNames(String database) { if (dbType == DbType.MYSQL) { query = "SHOW TABLES IN " + database; } else if (dbType == DbType.MSSQL) { - query = "SELECT name FROM " + database + ".sys.tables "; + query = "SELECT name FROM " + database + ".sys.tables ORDER BY name"; } else if (dbType == DbType.ORACLE) { query = "SELECT table_name FROM all_tables WHERE owner='" + database.toUpperCase() + "'"; - } else if (dbType == DbType.POSTGRESQL) { - query = "SELECT table_name FROM information_schema.tables WHERE table_schema = '" + database.toLowerCase() + "'"; + } else if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT) { + query = "SELECT table_name FROM information_schema.tables WHERE table_schema = '" + database.toLowerCase() + "' ORDER BY table_name"; } else if (dbType == DbType.MSACCESS) { query = "SELECT Name FROM sys.MSysObjects WHERE Type=1 AND Flags=0;"; } @@ -182,10 +191,24 @@ public ResultSet getMsAccessFieldNames(String table){ * @return */ public long getTableSize(String tableName) { + QueryResult qr = null; + Long returnVal = null; if (dbType == DbType.MSSQL || dbType == DbType.MSACCESS) - return Long.parseLong(query("SELECT COUNT(*) FROM [" + tableName + "];").iterator().next().getCells().get(0)); + qr = query("SELECT COUNT(*) FROM [" + tableName + "];"); + //return Long.parseLong(query("SELECT COUNT(*) FROM [" + tableName + "];").iterator().next().getCells().get(0)); else - return Long.parseLong(query("SELECT COUNT(*) FROM " + tableName + ";").iterator().next().getCells().get(0)); + qr = query("SELECT COUNT(*) FROM " + tableName + ";"); + // return Long.parseLong(query("SELECT COUNT(*) FROM " + tableName + ";").iterator().next().getCells().get(0)); + + // Obtain the value and close the connection + try { + returnVal = Long.parseLong(query("SELECT COUNT(*) FROM " + tableName + ";").iterator().next().getCells().get(0)); + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + if (qr != null) { qr.close(); } + } + return returnVal; } /** @@ -297,7 +320,7 @@ private void insert(String tableName, List rows) { if (value.length() == 0) value = null; // System.out.println(value); - if (dbType == DbType.POSTGRESQL) // PostgreSQL does not allow unspecified types + if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT) // PostgreSQL does not allow unspecified types statement.setObject(i + 1, value, Types.OTHER); else if (dbType == DbType.ORACLE) { if (isDate(value)) { @@ -416,6 +439,7 @@ private class DBRowIterator implements Iterator { private Set columnNames = new HashSet(); public DBRowIterator(String sql) { + Statement statement = null; try { sql.trim(); if (sql.endsWith(";")) @@ -427,7 +451,7 @@ public DBRowIterator(String sql) { System.out.println("Executing query: " + abbrSQL); } long start = System.currentTimeMillis(); - Statement statement = connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + statement = connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); resultSet = statement.executeQuery(sql.toString()); hasNext = resultSet.next(); if (verbose) diff --git a/src/org/ohdsi/utilities/StringUtilities.java b/src/org/ohdsi/utilities/StringUtilities.java index 8239020b..b727e88b 100644 --- a/src/org/ohdsi/utilities/StringUtilities.java +++ b/src/org/ohdsi/utilities/StringUtilities.java @@ -828,7 +828,12 @@ public static String wordWrap(String text, int lineLength) { if (text.substring(0, lineLength).contains("\n")) return text.substring(0, text.indexOf("\n")).trim() + "\n\n" + wordWrap(text.substring(text.indexOf("\n") + 1), lineLength); int place = Math.max(Math.max(text.lastIndexOf(" ", lineLength), text.lastIndexOf("\t", lineLength)), text.lastIndexOf("-", lineLength)); - return text.substring(0, place).trim() + "\n" + wordWrap(text.substring(place), lineLength); + if (place > 0) { + return text.substring(0, place).trim() + "\n" + wordWrap(text.substring(place), lineLength); + } + else { + return text; + } } public static boolean isDate(String string) { diff --git a/src/org/ohdsi/whiteRabbit/WhiteRabbitMain.java b/src/org/ohdsi/whiteRabbit/WhiteRabbitMain.java index 5d1d7bee..6f681d69 100644 --- a/src/org/ohdsi/whiteRabbit/WhiteRabbitMain.java +++ b/src/org/ohdsi/whiteRabbit/WhiteRabbitMain.java @@ -191,7 +191,7 @@ public void actionPerformed(ActionEvent e) { sourcePanel.setLayout(new GridLayout(0, 2)); sourcePanel.setBorder(BorderFactory.createTitledBorder("Source data location")); sourcePanel.add(new JLabel("Data type")); - sourceType = new JComboBox(new String[] { "Delimited text files", "MySQL", "Oracle", "SQL Server", "PostgreSQL", "MS Access" }); + sourceType = new JComboBox(new String[] { "Delimited text files", "MySQL", "Oracle", "SQL Server", "PostgreSQL", "MS Access", "Redshift" }); sourceType.setToolTipText("Select the type of source data available"); sourceType.addItemListener(new ItemListener() { @@ -350,8 +350,8 @@ public void stateChanged(ChangeEvent arg0) { scanOptionsPanel.add(Box.createHorizontalGlue()); scanOptionsPanel.add(new JLabel("Rows per table ")); - scanRowCount = new JComboBox(new String[] { "100,000", "1 million", "all" }); - scanRowCount.setSelectedIndex(1); + scanRowCount = new JComboBox(new String[] { "100,000", "500,000", "1 million", "all" }); + scanRowCount.setSelectedIndex(2); scanRowCount.setToolTipText("Maximum number of rows per table to be scanned for field values"); scanOptionsPanel.add(scanRowCount); @@ -692,6 +692,8 @@ else if (sourceType.getSelectedItem().toString().equals("Oracle")) dbSettings.dbType = DbType.ORACLE; else if (sourceType.getSelectedItem().toString().equals("PostgreSQL")) dbSettings.dbType = DbType.POSTGRESQL; + else if (sourceType.getSelectedItem().toString().equals("Redshift")) + dbSettings.dbType = DbType.REDSHIFT; else if (sourceType.getSelectedItem().toString().equals("SQL Server")) { dbSettings.dbType = DbType.MSSQL; if (sourceUserField.getText().length() != 0) { // Not using windows authentication @@ -815,6 +817,8 @@ private void scanRun() { int rowCount = 0; if (scanRowCount.getSelectedItem().toString().equals("100,000")) rowCount = 100000; + else if (scanRowCount.getSelectedItem().toString().equals("500,000")) + rowCount = 500000; else if (scanRowCount.getSelectedItem().toString().equals("1 million")) rowCount = 1000000; if (scanRowCount.getSelectedItem().toString().equals("all")) diff --git a/src/org/ohdsi/whiteRabbit/scan/SourceDataScan.java b/src/org/ohdsi/whiteRabbit/scan/SourceDataScan.java index 7daf1c99..a6d71ddb 100644 --- a/src/org/ohdsi/whiteRabbit/scan/SourceDataScan.java +++ b/src/org/ohdsi/whiteRabbit/scan/SourceDataScan.java @@ -253,19 +253,26 @@ private List processDatabaseTable(String table, RichConnection connec List fieldInfos = fetchTableStructure(connection, table); if (scanValues) { int actualCount = 0; - QueryResult queryResult = fetchRowsFromTable(connection, table, rowCount); - for (org.ohdsi.utilities.files.Row row : queryResult) { - for (int i = 0; i < fieldInfos.size(); i++) - fieldInfos.get(i).processValue(row.getCells().get(i)); - actualCount++; - if (sampleSize != -1 && actualCount >= sampleSize) { - System.out.println("Stopped after " + actualCount + " rows"); - break; + QueryResult queryResult = null; + try + { + queryResult = fetchRowsFromTable(connection, table, rowCount); + for (org.ohdsi.utilities.files.Row row : queryResult) { + for (int i = 0; i < fieldInfos.size(); i++) + fieldInfos.get(i).processValue(row.getCells().get(i)); + actualCount++; + if (sampleSize != -1 && actualCount >= sampleSize) { + System.out.println("Stopped after " + actualCount + " rows"); + break; + } } + for (FieldInfo fieldInfo : fieldInfos) + fieldInfo.trim(); + } catch (Exception e) { + System.out.println("Error: " + e.getMessage()); + } finally { + if (queryResult != null) { queryResult.close(); } } - queryResult.close(); // Not normally needed, but if we ended prematurely make sure its closed - for (FieldInfo fieldInfo : fieldInfos) - fieldInfo.trim(); } return fieldInfos; @@ -289,7 +296,7 @@ else if (dbType == DbType.ORACLE) { if (percentage < 100) query += " SAMPLE(" + percentage + ")"; } - } else if (dbType == DbType.POSTGRESQL) + } else if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT) query += " ORDER BY RANDOM() LIMIT " + sampleSize; else if (dbType == DbType.MSACCESS) query = "SELECT " + "TOP " + sampleSize + " * FROM [" + table + "]"; @@ -326,7 +333,7 @@ else if (dbType == DbType.MSSQL) { + "';"; } else if (dbType == DbType.MYSQL) query = "SELECT COLUMN_NAME,DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '" + database + "' AND TABLE_NAME = '" + table + "';"; - else if (dbType == DbType.POSTGRESQL) + else if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT) query = "SELECT COLUMN_NAME,DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '" + database.toLowerCase() + "' AND TABLE_NAME = '" + table.toLowerCase() + "' ORDER BY ordinal_position;";