Merge pull request #131 from qkerby/riah-cdm5.1.0-csv

Add support for 5.1.0 CDM in Rabbit in a Hat

.classpath

@@ -1,28 +1,26 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
-	<classpathentry kind="src" path="src"/>
-	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
-	<classpathentry kind="lib" path="lib/ojdbc5.jar"/>
-	<classpathentry kind="lib" path="lib/ojdbc6.jar"/>
-	<classpathentry kind="lib" path="lib/sqljdbc4.jar"/>
-	<classpathentry kind="lib" path="lib/dom4j-1.6.1.jar"/>
-	<classpathentry kind="lib" path="lib/poi-3.9-20121203.jar"/>
-	<classpathentry kind="lib" path="lib/poi-excelant-3.9-20121203.jar"/>
-	<classpathentry kind="lib" path="lib/poi-ooxml-3.9-20121203.jar"/>
-	<classpathentry kind="lib" path="lib/poi-ooxml-schemas-3.9-20121203.jar"/>
-	<classpathentry kind="lib" path="lib/stax-api-1.0.1.jar"/>
-	<classpathentry kind="lib" path="lib/xmlbeans-2.3.0.jar"/>
-	<classpathentry kind="lib" path="lib/jtds-1.3.1-v20140512.jar"/>
-	<classpathentry kind="lib" path="lib/json-io-3.0.1.jar"/>
-	<classpathentry kind="lib" path="lib/commons-csv-1.1.jar"/>
-	<classpathentry kind="lib" path="lib/postgresql-9.4-1201.jdbc4.jar"/>
-	<classpathentry kind="lib" path="lib/postgresql-9.4-1201.jdbc41.jar"/>
-	<classpathentry kind="lib" path="lib/mysql-connector-java-5.1.37-bin.jar"/>
-	<classpathentry kind="lib" path="lib/ucanaccess-3.0.3.1.jar"/>
-	<classpathentry kind="lib" path="lib/hsqldb-2.2.5.jar"/>
-	<classpathentry kind="lib" path="lib/jackcess-2.1.3.jar"/>
-	<classpathentry kind="lib" path="lib/commons-logging-1.2.jar"/>
-	<classpathentry kind="lib" path="lib/commons-lang-2.0.jar"/>
-	<classpathentry kind="lib" path="lib/RedshiftJDBC4-1.1.10.1010.jar"/>
-	<classpathentry kind="output" path="bin"/>
-</classpath>
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="src" path="src"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+	<classpathentry kind="lib" path="lib/sqljdbc4.jar"/>
+	<classpathentry kind="lib" path="lib/dom4j-1.6.1.jar"/>
+	<classpathentry kind="lib" path="lib/poi-3.9-20121203.jar"/>
+	<classpathentry kind="lib" path="lib/poi-excelant-3.9-20121203.jar"/>
+	<classpathentry kind="lib" path="lib/poi-ooxml-3.9-20121203.jar"/>
+	<classpathentry kind="lib" path="lib/poi-ooxml-schemas-3.9-20121203.jar"/>
+	<classpathentry kind="lib" path="lib/stax-api-1.0.1.jar"/>
+	<classpathentry kind="lib" path="lib/xmlbeans-2.3.0.jar"/>
+	<classpathentry kind="lib" path="lib/json-io-3.0.1.jar"/>
+	<classpathentry kind="lib" path="lib/commons-csv-1.1.jar"/>
+	<classpathentry kind="lib" path="lib/postgresql-9.4-1201.jdbc4.jar"/>
+	<classpathentry kind="lib" path="lib/postgresql-9.4-1201.jdbc41.jar"/>
+	<classpathentry kind="lib" path="lib/mysql-connector-java-5.1.37-bin.jar"/>
+	<classpathentry kind="lib" path="lib/ucanaccess-3.0.3.1.jar"/>
+	<classpathentry kind="lib" path="lib/hsqldb-2.2.5.jar"/>
+	<classpathentry kind="lib" path="lib/jackcess-2.1.3.jar"/>
+	<classpathentry kind="lib" path="lib/commons-logging-1.2.jar"/>
+	<classpathentry kind="lib" path="lib/commons-lang-2.0.jar"/>
+	<classpathentry kind="lib" path="lib/RedshiftJDBC4-1.1.10.1010.jar"/>
+	<classpathentry kind="lib" path="lib/ojdbc7.jar"/>
+	<classpathentry kind="output" path="bin"/>
+</classpath>

README.md

@@ -10,6 +10,7 @@ Features
 - Can scan databases in SQL Server, Oracle, PostgreSQL, MySQL, MS Access, Amazon RedShift, and CSV files
 - The scan report contains information on tables, fields, and frequency distributions of values
 - Cutoff on the minimum frequency of values to protect patient privacy
+- WhiteRabbit can be run with a graphical user interface or from the command prompt
 - Interactive tool (Rabbit in a Hat) for designing the ETL using the scan report as basis
 - Rabbit in a Hat generates ETL specification document according to OMOP template
 
@@ -49,6 +50,8 @@ WhiteRabbit
 2. Unzip the download
 3. Double-click on WhiteRabbit.jar to start White Rabbit.
 
+(See the [Wiki](http://www.ohdsi.org/web/wiki/doku.php?id=documentation:software:whiterabbit#running_from_the_command_line) for details on how to run from the command prompt instead)
+
 Rabbit-In-A-Hat
 
 1. Using the files downloaded for WhiteRabbit, double-click on RabbitInAHat.jar to start Rabbit-In-A-Hat.
@@ -66,5 +69,7 @@ WhiteRabbit is licensed under Apache License 2.0
 Development
 ===========
 White Rabbit and Rabbit in a Hat are being developed in Eclipse. Contributions are welcome.
-###Development status
-Beta testing - We invite everyone to try the software, and report any issues they may find.
+
+### Development status
+
+Production. This program is being used by many people.

build.xml

@@ -22,15 +22,15 @@
 		<jar destfile="${build.dir}/WhiteRabbit.jar">
 			<manifest>
 				<attribute name="Main-Class" value="org.ohdsi.utilities.WhiteRabbitLauncher"/>
-				<attribute name="Class-Path" value=". WhiteRabbit_lib/ojdbc5.jar WhiteRabbit_lib/ojdbc6.jar WhiteRabbit_lib/sqljdbc4.jar WhiteRabbit_lib/mysql-connector-java-5.1.37-bin.jar WhiteRabbit_lib/dom4j-1.6.1.jar WhiteRabbit_lib/poi-3.9-20121203.jar WhiteRabbit_lib/poi-excelant-3.9-20121203.jar WhiteRabbit_lib/poi-ooxml-3.9-20121203.jar WhiteRabbit_lib/poi-ooxml-schemas-3.9-20121203.jar WhiteRabbit_lib/stax-api-1.0.1.jar WhiteRabbit_lib/xmlbeans-2.3.0.jar WhiteRabbit_lib/jtds-1.3.1-v20140512.jar WhiteRabbit_lib/postgresql-9.4-1201.jdbc4.jar WhiteRabbit_lib/postgresql-9.4-1201.jdbc41.jar WhiteRabbit_lib/json-io-3.0.1.jar WhiteRabbit_lib/commons-csv-1.1.jar WhiteRabbit_lib/commons-lang-2.0.jar WhiteRabbit_lib/commons-logging-1.2.jar WhiteRabbit_lib/hsqldb-2.2.5.jar WhiteRabbit_lib/jackcess-2.1.3.jar WhiteRabbit_lib/ucanaccess-3.0.3.1.jar WhiteRabbit_lib/RedshiftJDBC4-1.1.10.1010.jar"/>
+				<attribute name="Class-Path" value=". WhiteRabbit_lib/ojdbc7.jar WhiteRabbit_lib/sqljdbc4.jar WhiteRabbit_lib/mysql-connector-java-5.1.37-bin.jar WhiteRabbit_lib/dom4j-1.6.1.jar WhiteRabbit_lib/poi-3.9-20121203.jar WhiteRabbit_lib/poi-excelant-3.9-20121203.jar WhiteRabbit_lib/poi-ooxml-3.9-20121203.jar WhiteRabbit_lib/poi-ooxml-schemas-3.9-20121203.jar WhiteRabbit_lib/stax-api-1.0.1.jar WhiteRabbit_lib/xmlbeans-2.3.0.jar WhiteRabbit_lib/postgresql-9.4-1201.jdbc4.jar WhiteRabbit_lib/postgresql-9.4-1201.jdbc41.jar WhiteRabbit_lib/json-io-3.0.1.jar WhiteRabbit_lib/commons-csv-1.1.jar WhiteRabbit_lib/commons-lang-2.0.jar WhiteRabbit_lib/commons-logging-1.2.jar WhiteRabbit_lib/hsqldb-2.2.5.jar WhiteRabbit_lib/jackcess-2.1.3.jar WhiteRabbit_lib/ucanaccess-3.0.3.1.jar WhiteRabbit_lib/RedshiftJDBC4-1.1.10.1010.jar"/>
 			</manifest>
 			<fileset dir="${build.dir}"/>
 			<fileset dir="${src.dir}" includes="**/*.csv,**/*.png" />
 		</jar>
 		<jar destfile="${build.dir}/RabbitInAHat.jar">
 			<manifest>
 				<attribute name="Main-Class" value="org.ohdsi.utilities.RabbitInAHatLauncher"/>
-				<attribute name="Class-Path" value=". WhiteRabbit_lib/ojdbc5.jar WhiteRabbit_lib/ojdbc6.jar WhiteRabbit_lib/sqljdbc4.jar WhiteRabbit_lib/mysql-connector-java-5.1.37-bin.jar WhiteRabbit_lib/dom4j-1.6.1.jar WhiteRabbit_lib/poi-3.9-20121203.jar WhiteRabbit_lib/poi-excelant-3.9-20121203.jar WhiteRabbit_lib/poi-ooxml-3.9-20121203.jar WhiteRabbit_lib/poi-ooxml-schemas-3.9-20121203.jar WhiteRabbit_lib/stax-api-1.0.1.jar WhiteRabbit_lib/xmlbeans-2.3.0.jar WhiteRabbit_lib/jtds-1.3.1-v20140512.jar WhiteRabbit_lib/postgresql-9.4-1201.jdbc4.jar WhiteRabbit_lib/postgresql-9.4-1201.jdbc41.jar WhiteRabbit_lib/json-io-3.0.1.jar WhiteRabbit_lib/commons-csv-1.1.jar"/>
+				<attribute name="Class-Path" value=". WhiteRabbit_lib/dom4j-1.6.1.jar WhiteRabbit_lib/poi-3.9-20121203.jar WhiteRabbit_lib/poi-excelant-3.9-20121203.jar WhiteRabbit_lib/poi-ooxml-3.9-20121203.jar WhiteRabbit_lib/poi-ooxml-schemas-3.9-20121203.jar WhiteRabbit_lib/stax-api-1.0.1.jar WhiteRabbit_lib/xmlbeans-2.3.0.jar WhiteRabbit_lib/json-io-3.0.1.jar WhiteRabbit_lib/commons-csv-1.1.jar"/>
 			</manifest>
 			<fileset dir="${build.dir}"/>
 			<fileset dir="${src.dir}" includes="**/*.csv,**/*.png" />
@@ -40,8 +40,7 @@
 		</copy>
 		<delete dir="${build.dir}/WhiteRabbit_lib"/>
 		<mkdir dir="${build.dir}/WhiteRabbit_lib"/>
-		<copy file="lib/ojdbc5.jar" todir="${build.dir}/WhiteRabbit_lib"/>
-		<copy file="lib/ojdbc6.jar" todir="${build.dir}/WhiteRabbit_lib"/>
+		<copy file="lib/ojdbc7.jar" todir="${build.dir}/WhiteRabbit_lib"/>
 		<copy file="lib/sqljdbc4.jar" todir="${build.dir}/WhiteRabbit_lib"/>
 		<copy file="lib/mysql-connector-java-5.1.37-bin.jar" todir="${build.dir}/WhiteRabbit_lib"/>
 		<copy file="lib/dom4j-1.6.1.jar" todir="${build.dir}/WhiteRabbit_lib"/>
@@ -51,7 +50,6 @@
 		<copy file="lib/poi-ooxml-schemas-3.9-20121203.jar" todir="${build.dir}/WhiteRabbit_lib"/>
 		<copy file="lib/stax-api-1.0.1.jar" todir="${build.dir}/WhiteRabbit_lib"/>
 		<copy file="lib/xmlbeans-2.3.0.jar" todir="${build.dir}/WhiteRabbit_lib"/>
-		<copy file="lib/jtds-1.3.1-v20140512.jar" todir="${build.dir}/WhiteRabbit_lib"/>
 		<copy file="lib/postgresql-9.4-1201.jdbc4.jar" todir="${build.dir}/WhiteRabbit_lib"/>
 		<copy file="lib/postgresql-9.4-1201.jdbc41.jar" todir="${build.dir}/WhiteRabbit_lib"/>
 		<copy file="lib/json-io-3.0.1.jar" todir="${build.dir}/WhiteRabbit_lib"/>

iniFileExamples/WhiteRabbit.ini

@@ -0,0 +1,12 @@
+WORKING_FOLDER = /users/joe                   # Path to the folder where all output will be written 
+DATA_TYPE = PostgreSQL                        # "Delimited text files", "MySQL", "Oracle", "SQL Server", "PostgreSQL", "MS Access", or "Redshift"
+SERVER_LOCATION = 127.0.0.1/data_base_name    # Name or address of the server. For Postgres, add the database name 
+USER_NAME = joe                               # User name for the database 
+PASSWORD = supersecret                        # Password for the database 
+DATABASE_NAME = schema_name                   # Name of the data schema used 
+DELIMITER = ,                                 # The delimiter that separates values
+TABLES_TO_SCAN = *                            # Comma-delimited list of table names to scan. Use "*" (asterix) to include all tables in the database
+SCAN_FIELD_VALUES = yes                       # Include a frequency count of field values in the scan report? "yes" or "no"
+MIN_CELL_COUNT = 5                            # Minimum frequency for a field value to be included in the report 
+MAX_DISTINCT_VALUES = 1000                    # Maximum number of distinct values per field to be reported
+ROWS_PER_TABLE = 100000                       # Maximum number of rows per table to be scanned for field values

src/org/ohdsi/databases/DbType.java

@@ -20,21 +20,22 @@
 public class DbType {
 	public static DbType	MYSQL		= new DbType("mysql");
 	public static DbType	MSSQL		= new DbType("mssql");
+	public static DbType	PDW			= new DbType("pdw");
 	public static DbType	ORACLE		= new DbType("oracle");
 	public static DbType	POSTGRESQL	= new DbType("postgresql");
 	public static DbType	MSACCESS	= new DbType("msaccess");
-	public static DbType    REDSHIFT    = new DbType("redshift");
-	
+	public static DbType	REDSHIFT	= new DbType("redshift");
+
 	private enum Type {
-		MYSQL, MSSQL, ORACLE, POSTGRESQL, MSACCESS, REDSHIFT
+		MYSQL, MSSQL, PDW, ORACLE, POSTGRESQL, MSACCESS, REDSHIFT
 	};
-	
-	private Type	type;
-	
+
+	private Type type;
+
 	public DbType(String type) {
 		this.type = Type.valueOf(type.toUpperCase());
 	}
-	
+
 	public boolean equals(Object other) {
 		if (other instanceof DbType && ((DbType) other).type == type)
 			return true;

src/org/ohdsi/databases/RichConnection.java

@@ -143,7 +143,7 @@ public List<String> getTableNames(String database) {
 		String query = null;
 		if (dbType == DbType.MYSQL) {
 			query = "SHOW TABLES IN " + database;
-		} else if (dbType == DbType.MSSQL) {
+		} else if (dbType == DbType.MSSQL || dbType == DbType.PDW) {
 			query = "SELECT name FROM " + database + ".sys.tables ORDER BY name";
 		} else if (dbType == DbType.ORACLE) {
 			query = "SELECT table_name FROM all_tables WHERE owner='" + database.toUpperCase() + "'";
@@ -160,7 +160,7 @@ public List<String> getTableNames(String database) {
 
 	public List<String> getFieldNames(String table) {
 		List<String> names = new ArrayList<String>();
-		if (dbType == DbType.MSSQL) {
+		if (dbType == DbType.MSSQL || dbType == DbType.PDW) {
 			for (Row row : query("SELECT name FROM syscolumns WHERE id=OBJECT_ID('" + table + "')"))
 				names.add(row.get("name"));
 		} else if (dbType == DbType.MYSQL)
@@ -193,16 +193,14 @@ public ResultSet getMsAccessFieldNames(String table){
 	public long getTableSize(String tableName) {
 		QueryResult qr = null;
 		Long returnVal = null;
-		if (dbType == DbType.MSSQL || dbType == DbType.MSACCESS)
+		if (dbType == DbType.MSSQL|| dbType == DbType.PDW) 
+			qr = query("SELECT COUNT_BIG(*) FROM [" + tableName + "];");
+		else if (dbType == DbType.MSACCESS )
 			qr = query("SELECT COUNT(*) FROM [" + tableName + "];");
-			//return Long.parseLong(query("SELECT COUNT(*) FROM [" + tableName + "];").iterator().next().getCells().get(0));
 		else
 			qr = query("SELECT COUNT(*) FROM " + tableName + ";");
-		//	return Long.parseLong(query("SELECT COUNT(*) FROM " + tableName + ";").iterator().next().getCells().get(0));
-
-		// Obtain the value and close the connection
 		try {
-			returnVal = Long.parseLong(query("SELECT COUNT(*) FROM " + tableName + ";").iterator().next().getCells().get(0));
+			returnVal = Long.parseLong(qr.iterator().next().getCells().get(0));
 		} catch (Exception e) {
 			throw new RuntimeException(e);
 		} finally {
@@ -415,7 +413,7 @@ else if (maxLength > 255)
 					return columnNameToSqlName(name) + " text";
 				else
 					return columnNameToSqlName(name) + " varchar(255)";
-			} else if (dbType == DbType.MSSQL) {
+			} else if (dbType == DbType.MSSQL || dbType == DbType.PDW) {
 				if (isNumeric) {
 					if (maxLength < 10)
 						return columnNameToSqlName(name) + " int";