Skip to content

Commit

Permalink
Merge pull request #105 from OHDSI/redshift
Browse files Browse the repository at this point in the history 
Adding support for RedShift
  • Loading branch information
@schuemie
schuemie committed Mar 21, 2016
2 parents 1d40c29 + 92c2d47 commit bcf2bcb
Show file tree
Hide file tree
Showing 7 changed files with 70 additions and 28 deletions.
1 change: 1 addition & 0 deletions .classpath
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,5 +23,6 @@
<classpathentry kind="lib" path="lib/jackcess-2.1.3.jar"/>
<classpathentry kind="lib" path="lib/commons-logging-1.2.jar"/>
<classpathentry kind="lib" path="lib/commons-lang-2.0.jar"/>
<classpathentry kind="lib" path="lib/RedshiftJDBC4-1.1.10.1010.jar"/>
<classpathentry kind="output" path="bin"/>
</classpath>
Binary file added lib/RedshiftJDBC4-1.1.10.1010.jar
View file
Binary file not shown.
3 changes: 2 additions & 1 deletion src/org/ohdsi/databases/DbType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,10 @@ public class DbType {
public static DbType ORACLE = new DbType("oracle");
public static DbType POSTGRESQL = new DbType("postgresql");
public static DbType MSACCESS = new DbType("msaccess");
public static DbType REDSHIFT = new DbType("redshift");

private enum Type {
MYSQL, MSSQL, ORACLE, POSTGRESQL, MSACCESS
MYSQL, MSSQL, ORACLE, POSTGRESQL, MSACCESS, REDSHIFT
};

private Type type;
Expand Down
44 changes: 34 additions & 10 deletions src/org/ohdsi/databases/RichConnection.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,10 @@ public RichConnection(String server, String domain, String user, String password
* @param sql
*/
public void execute(String sql) {
Statement statement = null;
try {
if (sql.length() == 0)
return;
Statement statement = null;

statement = connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
for (String subQuery : sql.split(";")) {
Expand All @@ -78,10 +78,19 @@ public void execute(String sql) {
statement.executeBatch();
if (verbose)
outputQueryStats(statement, System.currentTimeMillis() - start);
statement.close();
} catch (SQLException e) {
System.err.println(sql);
e.printStackTrace();
} finally {
if (statement != null) {
try {
statement.close();
}
catch (SQLException e) {
// TODO Auto-generated catch block
System.err.println(e.getMessage());
}
}
}
}

Expand Down Expand Up @@ -121,7 +130,7 @@ public void use(String database) {
return;
if (dbType == DbType.ORACLE)
execute("ALTER SESSION SET current_schema = " + database);
else if (dbType == DbType.POSTGRESQL)
else if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT)
execute("SET search_path TO " + database);
else if (dbType == DbType.MSACCESS)
;
Expand All @@ -135,11 +144,11 @@ public List<String> getTableNames(String database) {
if (dbType == DbType.MYSQL) {
query = "SHOW TABLES IN " + database;
} else if (dbType == DbType.MSSQL) {
query = "SELECT name FROM " + database + ".sys.tables ";
query = "SELECT name FROM " + database + ".sys.tables ORDER BY name";
} else if (dbType == DbType.ORACLE) {
query = "SELECT table_name FROM all_tables WHERE owner='" + database.toUpperCase() + "'";
} else if (dbType == DbType.POSTGRESQL) {
query = "SELECT table_name FROM information_schema.tables WHERE table_schema = '" + database.toLowerCase() + "'";
} else if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT) {
query = "SELECT table_name FROM information_schema.tables WHERE table_schema = '" + database.toLowerCase() + "' ORDER BY table_name";
} else if (dbType == DbType.MSACCESS) {
query = "SELECT Name FROM sys.MSysObjects WHERE Type=1 AND Flags=0;";
}
Expand Down Expand Up @@ -182,10 +191,24 @@ public ResultSet getMsAccessFieldNames(String table){
* @return
*/
public long getTableSize(String tableName) {
QueryResult qr = null;
Long returnVal = null;
if (dbType == DbType.MSSQL || dbType == DbType.MSACCESS)
return Long.parseLong(query("SELECT COUNT(*) FROM [" + tableName + "];").iterator().next().getCells().get(0));
qr = query("SELECT COUNT(*) FROM [" + tableName + "];");
//return Long.parseLong(query("SELECT COUNT(*) FROM [" + tableName + "];").iterator().next().getCells().get(0));
else
return Long.parseLong(query("SELECT COUNT(*) FROM " + tableName + ";").iterator().next().getCells().get(0));
qr = query("SELECT COUNT(*) FROM " + tableName + ";");
// return Long.parseLong(query("SELECT COUNT(*) FROM " + tableName + ";").iterator().next().getCells().get(0));

// Obtain the value and close the connection
try {
returnVal = Long.parseLong(query("SELECT COUNT(*) FROM " + tableName + ";").iterator().next().getCells().get(0));
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
if (qr != null) { qr.close(); }
}
return returnVal;
}

/**
Expand Down Expand Up @@ -297,7 +320,7 @@ private void insert(String tableName, List<Row> rows) {
if (value.length() == 0)
value = null;
// System.out.println(value);
if (dbType == DbType.POSTGRESQL) // PostgreSQL does not allow unspecified types
if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT) // PostgreSQL does not allow unspecified types
statement.setObject(i + 1, value, Types.OTHER);
else if (dbType == DbType.ORACLE) {
if (isDate(value)) {
Expand Down Expand Up @@ -416,6 +439,7 @@ private class DBRowIterator implements Iterator<Row> {
private Set<String> columnNames = new HashSet<String>();

public DBRowIterator(String sql) {
Statement statement = null;
try {
sql.trim();
if (sql.endsWith(";"))
Expand All @@ -427,7 +451,7 @@ public DBRowIterator(String sql) {
System.out.println("Executing query: " + abbrSQL);
}
long start = System.currentTimeMillis();
Statement statement = connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
statement = connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
resultSet = statement.executeQuery(sql.toString());
hasNext = resultSet.next();
if (verbose)
Expand Down
7 changes: 6 additions & 1 deletion src/org/ohdsi/utilities/StringUtilities.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -828,7 +828,12 @@ public static String wordWrap(String text, int lineLength) {
if (text.substring(0, lineLength).contains("\n"))
return text.substring(0, text.indexOf("\n")).trim() + "\n\n" + wordWrap(text.substring(text.indexOf("\n") + 1), lineLength);
int place = Math.max(Math.max(text.lastIndexOf(" ", lineLength), text.lastIndexOf("\t", lineLength)), text.lastIndexOf("-", lineLength));
return text.substring(0, place).trim() + "\n" + wordWrap(text.substring(place), lineLength);
if (place > 0) {
return text.substring(0, place).trim() + "\n" + wordWrap(text.substring(place), lineLength);
}
else {
return text;
}
}

public static boolean isDate(String string) {
Expand Down
10 changes: 7 additions & 3 deletions src/org/ohdsi/whiteRabbit/WhiteRabbitMain.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ public void actionPerformed(ActionEvent e) {
sourcePanel.setLayout(new GridLayout(0, 2));
sourcePanel.setBorder(BorderFactory.createTitledBorder("Source data location"));
sourcePanel.add(new JLabel("Data type"));
sourceType = new JComboBox(new String[] { "Delimited text files", "MySQL", "Oracle", "SQL Server", "PostgreSQL", "MS Access" });
sourceType = new JComboBox(new String[] { "Delimited text files", "MySQL", "Oracle", "SQL Server", "PostgreSQL", "MS Access", "Redshift" });
sourceType.setToolTipText("Select the type of source data available");
sourceType.addItemListener(new ItemListener() {

Expand Down Expand Up @@ -350,8 +350,8 @@ public void stateChanged(ChangeEvent arg0) {
scanOptionsPanel.add(Box.createHorizontalGlue());

scanOptionsPanel.add(new JLabel("Rows per table "));
scanRowCount = new JComboBox(new String[] { "100,000", "1 million", "all" });
scanRowCount.setSelectedIndex(1);
scanRowCount = new JComboBox(new String[] { "100,000", "500,000", "1 million", "all" });
scanRowCount.setSelectedIndex(2);
scanRowCount.setToolTipText("Maximum number of rows per table to be scanned for field values");
scanOptionsPanel.add(scanRowCount);

Expand Down Expand Up @@ -692,6 +692,8 @@ else if (sourceType.getSelectedItem().toString().equals("Oracle"))
dbSettings.dbType = DbType.ORACLE;
else if (sourceType.getSelectedItem().toString().equals("PostgreSQL"))
dbSettings.dbType = DbType.POSTGRESQL;
else if (sourceType.getSelectedItem().toString().equals("Redshift"))
dbSettings.dbType = DbType.REDSHIFT;
else if (sourceType.getSelectedItem().toString().equals("SQL Server")) {
dbSettings.dbType = DbType.MSSQL;
if (sourceUserField.getText().length() != 0) { // Not using windows authentication
Expand Down Expand Up @@ -815,6 +817,8 @@ private void scanRun() {
int rowCount = 0;
if (scanRowCount.getSelectedItem().toString().equals("100,000"))
rowCount = 100000;
else if (scanRowCount.getSelectedItem().toString().equals("500,000"))
rowCount = 500000;
else if (scanRowCount.getSelectedItem().toString().equals("1 million"))
rowCount = 1000000;
if (scanRowCount.getSelectedItem().toString().equals("all"))
Expand Down
33 changes: 20 additions & 13 deletions src/org/ohdsi/whiteRabbit/scan/SourceDataScan.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,19 +253,26 @@ private List<FieldInfo> processDatabaseTable(String table, RichConnection connec
List<FieldInfo> fieldInfos = fetchTableStructure(connection, table);
if (scanValues) {
int actualCount = 0;
QueryResult queryResult = fetchRowsFromTable(connection, table, rowCount);
for (org.ohdsi.utilities.files.Row row : queryResult) {
for (int i = 0; i < fieldInfos.size(); i++)
fieldInfos.get(i).processValue(row.getCells().get(i));
actualCount++;
if (sampleSize != -1 && actualCount >= sampleSize) {
System.out.println("Stopped after " + actualCount + " rows");
break;
QueryResult queryResult = null;
try
{
queryResult = fetchRowsFromTable(connection, table, rowCount);
for (org.ohdsi.utilities.files.Row row : queryResult) {
for (int i = 0; i < fieldInfos.size(); i++)
fieldInfos.get(i).processValue(row.getCells().get(i));
actualCount++;
if (sampleSize != -1 && actualCount >= sampleSize) {
System.out.println("Stopped after " + actualCount + " rows");
break;
}
}
for (FieldInfo fieldInfo : fieldInfos)
fieldInfo.trim();
} catch (Exception e) {
System.out.println("Error: " + e.getMessage());
} finally {
if (queryResult != null) { queryResult.close(); }
}
queryResult.close(); // Not normally needed, but if we ended prematurely make sure its closed
for (FieldInfo fieldInfo : fieldInfos)
fieldInfo.trim();
}

return fieldInfos;
Expand All @@ -289,7 +296,7 @@ else if (dbType == DbType.ORACLE) {
if (percentage < 100)
query += " SAMPLE(" + percentage + ")";
}
} else if (dbType == DbType.POSTGRESQL)
} else if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT)
query += " ORDER BY RANDOM() LIMIT " + sampleSize;
else if (dbType == DbType.MSACCESS)
query = "SELECT " + "TOP " + sampleSize + " * FROM [" + table + "]";
Expand Down Expand Up @@ -326,7 +333,7 @@ else if (dbType == DbType.MSSQL) {
+ "';";
} else if (dbType == DbType.MYSQL)
query = "SELECT COLUMN_NAME,DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '" + database + "' AND TABLE_NAME = '" + table + "';";
else if (dbType == DbType.POSTGRESQL)
else if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT)
query = "SELECT COLUMN_NAME,DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '" + database.toLowerCase() + "' AND TABLE_NAME = '"
+ table.toLowerCase() + "' ORDER BY ordinal_position;";

Expand Down

0 comments on commit bcf2bcb

Please sign in to comment.