Fix for #411 (can't process custom models with UTF8 BOM in csv file)

rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Database.java

@@ -25,6 +25,7 @@
 
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVRecord;
+import org.apache.commons.io.input.BOMInputStream;
 import org.ohdsi.utilities.ScanFieldName;
 import org.ohdsi.utilities.ScanSheetName;
 import org.ohdsi.utilities.files.QuickAndDirtyXlsxReader;
@@ -78,15 +79,18 @@ public String getDbName() {
 		return dbName;
 	}
 
-	public static Database generateCDMModel(CDMVersion cdmVersion) {
+	public static Database generateCDMModel(CDMVersion cdmVersion) throws IOException {
 		return Database.generateModelFromCSV(Database.class.getResourceAsStream(cdmVersion.fileName), cdmVersion.fileName);
 	}
 
-	public static Database generateModelFromCSV(InputStream stream, String dbName) {
+	public static Database generateModelFromCSV(InputStream stream, String dbName) throws IOException {
 		Database database = new Database();
 
 		database.dbName = dbName.substring(0, dbName.lastIndexOf("."));
 
+		// wrap the stream with a BOM handling inputstream
+		stream = BOMInputStream.builder().setInputStream(stream).get();
+
 		Map<String, Table> nameToTable = new HashMap<>();
 		try {
 			ConceptsMap conceptIdHintsMap = new ConceptsMap(CONCEPT_ID_HINTS_FILE_NAME);

rabbit-core/src/test/java/org/ohdsi/rabbitInAHat/dataModel/TestDatabase.java

@@ -0,0 +1,30 @@
+package org.ohdsi.rabbitInAHat.dataModel;
+
+import org.apache.commons.io.input.BOMInputStream;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class TestDatabase {
+
+    @Test
+    void testGenerateModelFromCSV() throws IOException {
+        // confirm that issue #411 is fixed, can read custom models from (UTF-8) CSV files with and without BOM
+
+        // generate a model from a CSV file without BOM
+        String testFileWithoutBom = "tiny_riah_without_bom.csv";
+        InputStream inWithoutBom = TestDatabase.class.getResourceAsStream(testFileWithoutBom);
+        assertNotNull(inWithoutBom);
+        Database ignoredWithoutBom = Database.generateModelFromCSV(inWithoutBom, testFileWithoutBom);
+
+        // generate a model from a CSV file with BOM
+        String testFileWithBom = "tiny_riah_with_bom.csv";
+        InputStream inWithBom = TestDatabase.class.getResourceAsStream(testFileWithBom);
+        assertNotNull(inWithBom);
+        Database ignoredWithBom = Database.generateModelFromCSV(inWithBom, testFileWithBom);
+
+    }
+}

rabbit-core/src/test/resources/org/ohdsi/rabbitInAHat/dataModel/tiny_riah_with_bom.csv

@@ -0,0 +1,2 @@
+table,field,required,type,description,schema
+PERSON,person_id,Yes,Integer,PK; Patient ID,cdm

rabbit-core/src/test/resources/org/ohdsi/rabbitInAHat/dataModel/tiny_riah_without_bom.csv

@@ -0,0 +1,2 @@
+table,field,required,type,description,schema
+PERSON,person_id,Yes,Integer,PK; Patient ID,cdm

rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java

@@ -115,11 +115,11 @@ public class RabbitInAHatMain implements ResizeListener {
 	private JSplitPane				tableFieldSplitPane;
 	private JFileChooser			chooser;
 
-	public static void main(String[] args) {
+	public static void main(String[] args) throws IOException {
 		new RabbitInAHatMain(args);
 	}
 
-	public RabbitInAHatMain(String[] args) {
+	public RabbitInAHatMain(String[] args) throws IOException {
 
 		// Set look and feel to the system look and feel
 		try {
@@ -277,7 +277,13 @@ private JMenuBar createMenuBar() {
 				targetCDM.setSelected(true);
 			}
 			targetGroup.add(targetCDM);
-			targetDatabaseMenu.add(targetCDM).addActionListener(evt -> this.doSetTargetCDM(cdmOptions.get(optionName)));
+			targetDatabaseMenu.add(targetCDM).addActionListener(evt -> {
+                try {
+                    this.doSetTargetCDM(cdmOptions.get(optionName));
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+            });
 		}
 
 		targetCDM = new JRadioButtonMenuItem(ACTION_SET_TARGET_CUSTOM);
@@ -499,7 +505,7 @@ private void doSetTargetCustom(String filename) {
 
 	}
 
-	private void doSetTargetCDM(CDMVersion cdmVersion) {
+	private void doSetTargetCDM(CDMVersion cdmVersion) throws IOException {
 		ETL etl = new ETL(ObjectExchange.etl.getSourceDatabase(), Database.generateCDMModel(cdmVersion));
 		etl.copyETLMappings(ObjectExchange.etl);
 		tableMappingPanel.setMapping(etl.getTableToTableMapping());