From 4cd56b69c5ab9131c24d88fd7ad00b5bc08f1da1 Mon Sep 17 00:00:00 2001 From: schuemie Date: Fri, 29 Apr 2016 08:36:29 +0200 Subject: [PATCH] Added function to create a framework for rapidly writing unit tests for the ETL. --- .../ETLTestFrameWorkGenerator.java | 198 ++++++++++++++++++ .../ohdsi/rabbitInAHat/RabbitInAHatMain.java | 18 ++ .../dataModel/TableCellLongTextRenderer.java | 8 +- .../ohdsi/whiteRabbit/WhiteRabbitMain.java | 3 - 4 files changed, 219 insertions(+), 8 deletions(-) create mode 100644 src/org/ohdsi/rabbitInAHat/ETLTestFrameWorkGenerator.java diff --git a/src/org/ohdsi/rabbitInAHat/ETLTestFrameWorkGenerator.java b/src/org/ohdsi/rabbitInAHat/ETLTestFrameWorkGenerator.java new file mode 100644 index 00000000..690a5880 --- /dev/null +++ b/src/org/ohdsi/rabbitInAHat/ETLTestFrameWorkGenerator.java @@ -0,0 +1,198 @@ +/******************************************************************************* + * Copyright 2016 Observational Health Data Sciences and Informatics + * + * This file is part of WhiteRabbit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.ohdsi.rabbitInAHat; + +import java.util.ArrayList; +import java.util.List; + +import org.ohdsi.rabbitInAHat.dataModel.Database; +import org.ohdsi.rabbitInAHat.dataModel.ETL; +import org.ohdsi.rabbitInAHat.dataModel.Field; +import org.ohdsi.rabbitInAHat.dataModel.Table; +import org.ohdsi.utilities.StringUtilities; +import org.ohdsi.utilities.files.WriteTextFile; + +public class ETLTestFrameWorkGenerator { + public static void generate(ETL etl, String filename) { + List r = generateRScript(etl); + WriteTextFile out = new WriteTextFile(filename); + for (String line : r) + out.writeln(line); + out.close(); + } + + private static List generateRScript(ETL etl) { + List r = new ArrayList(); + createInitFunction(r, etl.getSourceDatabase()); + createDeclareTestFunction(r); + createAddFunctions(r, etl.getSourceDatabase()); + createExpectFunctions(r, false, etl.getTargetDatabase()); + createExpectFunctions(r, true, etl.getTargetDatabase()); + return r; + } + + private static void createDeclareTestFunction(List r) { + r.add("declareTest <- function(id, description) {"); + r.add(" assign(\"testId\", id, envir = globalenv()) "); + r.add(" assign(\"testDescription\", description, envir = globalenv()) "); + r.add("}"); + r.add(""); + } + + private static void createExpectFunctions(List r, boolean negation, Database database) { + for (Table table : database.getTables()) { + StringBuilder line = new StringBuilder(); + String rTableName = convertToRName(table.getName()); + String sqlTableName = convertToSqlName(table.getName()); + List argDefs = new ArrayList(); + List testDefs = new ArrayList(); + for (Field field : table.getFields()) { + String rFieldName = convertToRName(field.getName()); + String sqlFieldName = convertToSqlName(field.getName()); + argDefs.add(rFieldName); + testDefs.add(" if (!missing(" + rFieldName + ")) {"); + testDefs.add(" if (first) {"); + testDefs.add(" first <- FALSE"); + testDefs.add(" } else {"); + testDefs.add(" statement <- paste0(statement, \" AND\")"); + testDefs.add(" }"); + testDefs.add(" statement <- paste0(statement, \" " + sqlFieldName + " = '\", " + rFieldName + ",\"'\")"); + testDefs.add(" }"); + testDefs.add(""); + } + + if (negation) + line.append("expect_no_" + rTableName + " <- function("); + else + line.append("expect_" + rTableName + " <- function("); + + line.append(StringUtilities.join(argDefs, ", ")); + line.append(") {"); + r.add(line.toString()); + + line = new StringBuilder(); + line.append(" statement <- paste0(\"INSERT INTO test_results SELECT "); + line.append("\", get(\"testId\", envir = globalenv()), \" AS id, "); + line.append("'\", get(\"testDescription\", envir = globalenv()), \"' AS description, "); + line.append("'Expect " + table.getName() + "' AS test, "); + line.append("CASE WHEN(SELECT COUNT(*) FROM " + sqlTableName + " WHERE\")"); + r.add(line.toString()); + + r.add(" first <- TRUE"); + + r.addAll(testDefs); + + if (negation) + r.add(" statement <- paste0(statement, \") = 1 THEN 'FAIL' ELSE 'PASS' END AS status;\")"); + else + r.add(" statement <- paste0(statement, \") = 0 THEN 'FAIL' ELSE 'PASS' END AS status;\")"); + + r.add(" assign(\"testSql\", c(get(\"testSql\", envir = globalenv()), statement), envir = globalenv())"); + r.add(" invisible(statement)"); + r.add("}"); + r.add(""); + } + } + + private static String convertToSqlName(String name) { + if (name.contains(" ") || name.contains(".") || name.toLowerCase().equals("procedure")) + name = "[" + name + "]"; + return name; + } + + private static void createInitFunction(List r, Database database) { + r.add("initFramework <- function() {"); + r.add(" insertSql <- c()"); + for (Table table : database.getTables()) { + String sqlTableName = convertToSqlName(table.getName()); + r.add(" insertSql <- c(insertSql, \"TRUNCATE TABLE " + sqlTableName + ";\")"); + } + r.add(" assign(\"insertSql\", insertSql, envir = globalenv())"); + + r.add(" testSql <- c()"); + r.add(" testSql <- c(testSql, \"IF OBJECT_ID('test_results', 'U') IS NOT NULL\")"); + r.add(" testSql <- c(testSql, \" DROP TABLE test_results;\")"); + r.add(" testSql <- c(testSql, \"\")"); + r.add(" testSql <- c(testSql, \"CREATE TABLE test_results (id INT, description VARCHAR(512), test VARCHAR(256), status VARCHAR(5));\")"); + r.add(" testSql <- c(testSql, \"\")"); + + r.add(" assign(\"testSql\", testSql, envir = globalenv()) "); + r.add(" assign(\"testId\", 1, envir = globalenv()) "); + r.add(" assign(\"testDescription\", \"\", envir = globalenv()) "); + r.add("}"); + r.add(""); + r.add("initFramework()"); + r.add(""); + } + + private static void createAddFunctions(List r, Database database) { + for (Table table : database.getTables()) { + StringBuilder line = new StringBuilder(); + String rTableName = convertToRName(table.getName()); + String sqlTableName = convertToSqlName(table.getName()); + List argDefs = new ArrayList(); + List insertLines = new ArrayList(); + for (Field field : table.getFields()) { + String rFieldName = field.getName().replaceAll(" ", "_").replaceAll("-", "_"); + String sqlFieldName = convertToSqlName(field.getName()); + String defaultValue; + if (field.getValueCounts().length == 0) + defaultValue = ""; + else + defaultValue = field.getValueCounts()[0][0]; + if (defaultValue.equals("")) + argDefs.add(rFieldName + " = NULL"); + else + argDefs.add(rFieldName + " = \"" + defaultValue + "\""); + + insertLines.add(" if (!is.null(" + rFieldName + ")) {"); + insertLines.add(" insertFields <- c(insertFields, \"" + sqlFieldName + "\")"); + insertLines.add(" insertValues <- c(insertValues, " +rFieldName + ")"); + insertLines.add(" }"); + insertLines.add(""); + } + + line.append("add_" + rTableName + " <- function("); + line.append(StringUtilities.join(argDefs, ", ")); + line.append(") {"); + r.add(line.toString()); + + r.add(" insertFields <- c()"); + r.add(" insertValues <- c()"); + r.addAll(insertLines); + + line = new StringBuilder(); + line.append(" statement <- paste0(\"INSERT INTO " + sqlTableName + " (\", "); + line.append("paste(insertFields, collapse = \", \"), "); + line.append("\") VALUES ('\", "); + line.append("paste(insertValues, collapse = \"', '\"), "); + line.append("\"');\")"); + r.add(line.toString()); + + r.add(" assign(\"insertSql\", c(get(\"insertSql\", envir = globalenv()), statement), envir = globalenv())"); + r.add(" invisible(statement)"); + r.add("}"); + r.add(""); + } + } + + private static String convertToRName(String name) { + name = name.replaceAll(" ", "_").replaceAll("-", "_"); + return name; + } +} diff --git a/src/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java b/src/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java index 664897f5..b689a4e4 100644 --- a/src/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java +++ b/src/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java @@ -58,6 +58,7 @@ public class RabbitInAHatMain implements ResizeListener, ActionListener { public final static String ACTION_CMD_OPEN_ETL_SPECS = "Open ETL Specs"; public final static String ACTION_CMD_OPEN_SCAN_REPORT = "Open Scan Report"; public final static String ACTION_CMD_GENERATE_ETL_DOCUMENT = "Generate ETL Document"; + public final static String ACTION_CMD_GENERATE_TEST_FRAMEWORK = "Generate ETL Test Framework"; public final static String ACTION_CMD_DISCARD_COUNTS = "Discard Value Counts"; public final static String ACTION_CMD_FILTER = "Filter"; public final static String ACTION_CMD_MAKE_MAPPING = "Make Mappings"; @@ -73,6 +74,7 @@ public class RabbitInAHatMain implements ResizeListener, ActionListener { private final static FileFilter FILE_FILTER_GZ = new FileNameExtensionFilter("GZIP Files (*.gz)", "gz"); private final static FileFilter FILE_FILTER_DOCX = new FileNameExtensionFilter("Microsoft Word documents (*.docx)", "docx"); private final static FileFilter FILE_FILTER_CSV = new FileNameExtensionFilter("Text Files (*.csv)", "csv"); + private final static FileFilter FILE_FILTER_R = new FileNameExtensionFilter("R script (*.r)", "r"); private JFrame frame; private JScrollPane scrollPane1; private JScrollPane scrollPane2; @@ -210,6 +212,11 @@ private JMenuBar createMenuBar() { generateDocItem.setActionCommand(ACTION_CMD_GENERATE_ETL_DOCUMENT); fileMenu.add(generateDocItem); + JMenuItem generateTestFrameworkItem = new JMenuItem(ACTION_CMD_GENERATE_TEST_FRAMEWORK); + generateTestFrameworkItem.addActionListener(this); + generateTestFrameworkItem.setActionCommand(ACTION_CMD_GENERATE_TEST_FRAMEWORK); + fileMenu.add(generateTestFrameworkItem); + JMenu editMenu = new JMenu("Edit"); menuBar.add(editMenu); @@ -360,6 +367,9 @@ public void actionPerformed(ActionEvent event) { case ACTION_CMD_GENERATE_ETL_DOCUMENT: doGenerateEtlDoc(chooseSavePath(FILE_FILTER_DOCX)); break; + case ACTION_CMD_GENERATE_TEST_FRAMEWORK: + doGenerateTestFramework(chooseSavePath(FILE_FILTER_R)); + break; case ACTION_CMD_DISCARD_COUNTS: doDiscardCounts(); break; @@ -394,6 +404,14 @@ public void actionPerformed(ActionEvent event) { } } + private void doGenerateTestFramework(String filename) { + if (filename != null) { + frame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); + ETLTestFrameWorkGenerator.generate(ObjectExchange.etl, filename); + frame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); + } + } + private void doOpenWiki() { try { Desktop desktop = Desktop.getDesktop(); diff --git a/src/org/ohdsi/rabbitInAHat/dataModel/TableCellLongTextRenderer.java b/src/org/ohdsi/rabbitInAHat/dataModel/TableCellLongTextRenderer.java index 8761891b..3c81bf2a 100644 --- a/src/org/ohdsi/rabbitInAHat/dataModel/TableCellLongTextRenderer.java +++ b/src/org/ohdsi/rabbitInAHat/dataModel/TableCellLongTextRenderer.java @@ -1,14 +1,12 @@ package org.ohdsi.rabbitInAHat.dataModel; -import java.awt.Color; import java.awt.Component; import java.awt.Insets; -import javax.swing.JTable; -import javax.swing.JTextArea; -import javax.swing.UIManager; +import javax.swing.JTable; +import javax.swing.JTextArea; import javax.swing.table.DefaultTableCellRenderer; -import javax.swing.table.TableCellRenderer; +import javax.swing.table.TableCellRenderer; /** diff --git a/src/org/ohdsi/whiteRabbit/WhiteRabbitMain.java b/src/org/ohdsi/whiteRabbit/WhiteRabbitMain.java index 6f681d69..e3fff8bd 100644 --- a/src/org/ohdsi/whiteRabbit/WhiteRabbitMain.java +++ b/src/org/ohdsi/whiteRabbit/WhiteRabbitMain.java @@ -31,7 +31,6 @@ import java.awt.event.ActionListener; import java.awt.event.ItemEvent; import java.awt.event.ItemListener; -import java.awt.event.KeyEvent; import java.awt.event.WindowAdapter; import java.awt.event.WindowEvent; import java.io.File; @@ -65,7 +64,6 @@ import javax.swing.JTabbedPane; import javax.swing.JTextArea; import javax.swing.JTextField; -import javax.swing.KeyStroke; import javax.swing.border.TitledBorder; import javax.swing.event.ChangeEvent; import javax.swing.event.ChangeListener; @@ -73,7 +71,6 @@ import org.ohdsi.databases.DbType; import org.ohdsi.databases.RichConnection; -import org.ohdsi.rabbitInAHat.dataModel.Database.CDMVersion; import org.ohdsi.utilities.DirectoryUtilities; import org.ohdsi.utilities.StringUtilities; import org.ohdsi.whiteRabbit.fakeDataGenerator.FakeDataGenerator;