Skip to content

Commit

Permalink
Adding support for REDSHIFT and added some additional checks to ensur…
Browse files Browse the repository at this point in the history 
…e database connections are properly closed.
  • Loading branch information
@anthonysena
anthonysena committed Mar 18, 2016
1 parent 1d40c29 commit 92c2d47
Show file tree
Hide file tree
Showing 7 changed files with 70 additions and 28 deletions.
1 change: 1 addition & 0 deletions .classpath
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,5 +23,6 @@
<classpathentry kind="lib" path="lib/jackcess-2.1.3.jar"/>
<classpathentry kind="lib" path="lib/commons-logging-1.2.jar"/>
<classpathentry kind="lib" path="lib/commons-lang-2.0.jar"/>
<classpathentry kind="lib" path="lib/RedshiftJDBC4-1.1.10.1010.jar"/>
<classpathentry kind="output" path="bin"/>
</classpath>
Binary file added lib/RedshiftJDBC4-1.1.10.1010.jar
View file
Binary file not shown.
3 changes: 2 additions & 1 deletion src/org/ohdsi/databases/DbType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,10 @@ public class DbType {
public static DbType ORACLE = new DbType("oracle");
public static DbType POSTGRESQL = new DbType("postgresql");
public static DbType MSACCESS = new DbType("msaccess");
public static DbType REDSHIFT = new DbType("redshift");

private enum Type {
MYSQL, MSSQL, ORACLE, POSTGRESQL, MSACCESS
MYSQL, MSSQL, ORACLE, POSTGRESQL, MSACCESS, REDSHIFT
};

private Type type;
Expand Down
44 changes: 34 additions & 10 deletions src/org/ohdsi/databases/RichConnection.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,10 @@ public RichConnection(String server, String domain, String user, String password
* @param sql
*/
public void execute(String sql) {
Statement statement = null;
try {
if (sql.length() == 0)
return;
Statement statement = null;

statement = connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
for (String subQuery : sql.split(";")) {
Expand All @@ -78,10 +78,19 @@ public void execute(String sql) {
statement.executeBatch();
if (verbose)
outputQueryStats(statement, System.currentTimeMillis() - start);
statement.close();
} catch (SQLException e) {
System.err.println(sql);
e.printStackTrace();
} finally {
if (statement != null) {
try {
statement.close();
}
catch (SQLException e) {
// TODO Auto-generated catch block
System.err.println(e.getMessage());
}
}
}
}

Expand Down Expand Up @@ -121,7 +130,7 @@ public void use(String database) {
return;
if (dbType == DbType.ORACLE)
execute("ALTER SESSION SET current_schema = " + database);
else if (dbType == DbType.POSTGRESQL)
else if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT)
execute("SET search_path TO " + database);
else if (dbType == DbType.MSACCESS)
;
Expand All @@ -135,11 +144,11 @@ public List<String> getTableNames(String database) {
if (dbType == DbType.MYSQL) {
query = "SHOW TABLES IN " + database;
} else if (dbType == DbType.MSSQL) {
query = "SELECT name FROM " + database + ".sys.tables ";
query = "SELECT name FROM " + database + ".sys.tables ORDER BY name";
} else if (dbType == DbType.ORACLE) {
query = "SELECT table_name FROM all_tables WHERE owner='" + database.toUpperCase() + "'";
} else if (dbType == DbType.POSTGRESQL) {
query = "SELECT table_name FROM information_schema.tables WHERE table_schema = '" + database.toLowerCase() + "'";
} else if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT) {
query = "SELECT table_name FROM information_schema.tables WHERE table_schema = '" + database.toLowerCase() + "' ORDER BY table_name";
} else if (dbType == DbType.MSACCESS) {
query = "SELECT Name FROM sys.MSysObjects WHERE Type=1 AND Flags=0;";
}
Expand Down Expand Up @@ -182,10 +191,24 @@ public ResultSet getMsAccessFieldNames(String table){
* @return
*/
public long getTableSize(String tableName) {
QueryResult qr = null;
Long returnVal = null;
if (dbType == DbType.MSSQL || dbType == DbType.MSACCESS)
return Long.parseLong(query("SELECT COUNT(*) FROM [" + tableName + "];").iterator().next().getCells().get(0));
qr = query("SELECT COUNT(*) FROM [" + tableName + "];");
//return Long.parseLong(query("SELECT COUNT(*) FROM [" + tableName + "];").iterator().next().getCells().get(0));
else
return Long.parseLong(query("SELECT COUNT(*) FROM " + tableName + ";").iterator().next().getCells().get(0));
qr = query("SELECT COUNT(*) FROM " + tableName + ";");
// return Long.parseLong(query("SELECT COUNT(*) FROM " + tableName + ";").iterator().next().getCells().get(0));

// Obtain the value and close the connection
try {
returnVal = Long.parseLong(query("SELECT COUNT(*) FROM " + tableName + ";").iterator().next().getCells().get(0));
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
if (qr != null) { qr.close(); }
}
return returnVal;
}

/**
Expand Down Expand Up @@ -297,7 +320,7 @@ private void insert(String tableName, List<Row> rows) {
if (value.length() == 0)
value = null;
// System.out.println(value);
if (dbType == DbType.POSTGRESQL) // PostgreSQL does not allow unspecified types
if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT) // PostgreSQL does not allow unspecified types
statement.setObject(i + 1, value, Types.OTHER);
else if (dbType == DbType.ORACLE) {
if (isDate(value)) {
Expand Down Expand Up @@ -416,6 +439,7 @@ private class DBRowIterator implements Iterator<Row> {
private Set<String> columnNames = new HashSet<String>();

public DBRowIterator(String sql) {
Statement statement = null;
try {
sql.trim();
if (sql.endsWith(";"))
Expand All @@ -427,7 +451,7 @@ public DBRowIterator(String sql) {
System.out.println("Executing query: " + abbrSQL);
}
long start = System.currentTimeMillis();
Statement statement = connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
statement = connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
resultSet = statement.executeQuery(sql.toString());
hasNext = resultSet.next();
if (verbose)
Expand Down
7 changes: 6 additions & 1 deletion src/org/ohdsi/utilities/StringUtilities.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -828,7 +828,12 @@ public static String wordWrap(String text, int lineLength) {
if (text.substring(0, lineLength).contains("\n"))
return text.substring(0, text.indexOf("\n")).trim() + "\n\n" + wordWrap(text.substring(text.indexOf("\n") + 1), lineLength);
int place = Math.max(Math.max(text.lastIndexOf(" ", lineLength), text.lastIndexOf("\t", lineLength)), text.lastIndexOf("-", lineLength));
return text.substring(0, place).trim() + "\n" + wordWrap(text.substring(place), lineLength);
if (place > 0) {
return text.substring(0, place).trim() + "\n" + wordWrap(text.substring(place), lineLength);
}
else {
return text;
}
}

public static boolean isDate(String string) {
Expand Down
10 changes: 7 additions & 3 deletions src/org/ohdsi/whiteRabbit/WhiteRabbitMain.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ public void actionPerformed(ActionEvent e) {
sourcePanel.setLayout(new GridLayout(0, 2));
sourcePanel.setBorder(BorderFactory.createTitledBorder("Source data location"));
sourcePanel.add(new JLabel("Data type"));
sourceType = new JComboBox(new String[] { "Delimited text files", "MySQL", "Oracle", "SQL Server", "PostgreSQL", "MS Access" });
sourceType = new JComboBox(new String[] { "Delimited text files", "MySQL", "Oracle", "SQL Server", "PostgreSQL", "MS Access", "Redshift" });
sourceType.setToolTipText("Select the type of source data available");
sourceType.addItemListener(new ItemListener() {

Expand Down Expand Up @@ -350,8 +350,8 @@ public void stateChanged(ChangeEvent arg0) {
scanOptionsPanel.add(Box.createHorizontalGlue());

scanOptionsPanel.add(new JLabel("Rows per table "));
scanRowCount = new JComboBox(new String[] { "100,000", "1 million", "all" });
scanRowCount.setSelectedIndex(1);
scanRowCount = new JComboBox(new String[] { "100,000", "500,000", "1 million", "all" });
scanRowCount.setSelectedIndex(2);
scanRowCount.setToolTipText("Maximum number of rows per table to be scanned for field values");
scanOptionsPanel.add(scanRowCount);

Expand Down Expand Up @@ -692,6 +692,8 @@ else if (sourceType.getSelectedItem().toString().equals("Oracle"))
dbSettings.dbType = DbType.ORACLE;
else if (sourceType.getSelectedItem().toString().equals("PostgreSQL"))
dbSettings.dbType = DbType.POSTGRESQL;
else if (sourceType.getSelectedItem().toString().equals("Redshift"))
dbSettings.dbType = DbType.REDSHIFT;
else if (sourceType.getSelectedItem().toString().equals("SQL Server")) {
dbSettings.dbType = DbType.MSSQL;
if (sourceUserField.getText().length() != 0) { // Not using windows authentication
Expand Down Expand Up @@ -815,6 +817,8 @@ private void scanRun() {
int rowCount = 0;
if (scanRowCount.getSelectedItem().toString().equals("100,000"))
rowCount = 100000;
else if (scanRowCount.getSelectedItem().toString().equals("500,000"))
rowCount = 500000;
else if (scanRowCount.getSelectedItem().toString().equals("1 million"))
rowCount = 1000000;
if (scanRowCount.getSelectedItem().toString().equals("all"))
Expand Down
33 changes: 20 additions & 13 deletions src/org/ohdsi/whiteRabbit/scan/SourceDataScan.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,19 +253,26 @@ private List<FieldInfo> processDatabaseTable(String table, RichConnection connec
List<FieldInfo> fieldInfos = fetchTableStructure(connection, table);
if (scanValues) {
int actualCount = 0;
QueryResult queryResult = fetchRowsFromTable(connection, table, rowCount);
for (org.ohdsi.utilities.files.Row row : queryResult) {
for (int i = 0; i < fieldInfos.size(); i++)
fieldInfos.get(i).processValue(row.getCells().get(i));
actualCount++;
if (sampleSize != -1 && actualCount >= sampleSize) {
System.out.println("Stopped after " + actualCount + " rows");
break;
QueryResult queryResult = null;
try
{
queryResult = fetchRowsFromTable(connection, table, rowCount);
for (org.ohdsi.utilities.files.Row row : queryResult) {
for (int i = 0; i < fieldInfos.size(); i++)
fieldInfos.get(i).processValue(row.getCells().get(i));
actualCount++;
if (sampleSize != -1 && actualCount >= sampleSize) {
System.out.println("Stopped after " + actualCount + " rows");
break;
}
}
for (FieldInfo fieldInfo : fieldInfos)
fieldInfo.trim();
} catch (Exception e) {
System.out.println("Error: " + e.getMessage());
} finally {
if (queryResult != null) { queryResult.close(); }
}
queryResult.close(); // Not normally needed, but if we ended prematurely make sure its closed
for (FieldInfo fieldInfo : fieldInfos)
fieldInfo.trim();
}

return fieldInfos;
Expand All @@ -289,7 +296,7 @@ else if (dbType == DbType.ORACLE) {
if (percentage < 100)
query += " SAMPLE(" + percentage + ")";
}
} else if (dbType == DbType.POSTGRESQL)
} else if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT)
query += " ORDER BY RANDOM() LIMIT " + sampleSize;
else if (dbType == DbType.MSACCESS)
query = "SELECT " + "TOP " + sampleSize + " * FROM [" + table + "]";
Expand Down Expand Up @@ -326,7 +333,7 @@ else if (dbType == DbType.MSSQL) {
+ "';";
} else if (dbType == DbType.MYSQL)
query = "SELECT COLUMN_NAME,DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '" + database + "' AND TABLE_NAME = '" + table + "';";
else if (dbType == DbType.POSTGRESQL)
else if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT)
query = "SELECT COLUMN_NAME,DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '" + database.toLowerCase() + "' AND TABLE_NAME = '"
+ table.toLowerCase() + "' ORDER BY ordinal_position;";

Expand Down

0 comments on commit 92c2d47

Please sign in to comment.