From eb248cf610460de7ad1e15c0b66d796a2f7c6792 Mon Sep 17 00:00:00 2001 From: "Williams, Jenny Marie Weisenberg" Date: Fri, 17 Mar 2023 13:12:20 -0400 Subject: [PATCH] Add Logger and use it to report progress loading ingestion package (#19) Prepend server response with INFO/WARNING/ERROR. Provide Java methods to parse level/message from a log line. --- .../com/ge/research/semtk/utility/Logger.java | 66 +++++++++++++++++++ .../semtk/load/config/LoadDataConfig.java | 34 ++++++---- .../semtk/load/config/LoadOwlConfig.java | 10 +-- .../semtk/load/config/ManifestConfig.java | 46 ++++++++----- .../semtk/load/config/YamlConfig.java | 10 +-- .../client/NodeGroupStoreRestClient.java | 25 ++++--- .../nodegroupStore/StoreDataCsvReader.java | 7 +- .../config/test/LoadDataConfigTest_IT.java | 15 ++--- .../config/test/LoadOwlConfigTest_IT.java | 6 +- .../config/test/ManifestConfigTest_IT.java | 8 +-- .../ManifestConfigTest_LoadTurnstile_IT.java | 2 +- .../test/NodeGroupStoreTest_IT.java | 4 +- .../semtk/utility/test/LoggerTest.java | 28 ++++++++ .../utility/test/UtilityClientTest_IT.java | 27 ++++---- .../standaloneExecutables/StoreNodeGroup.java | 3 +- .../utility/UtilityServiceRestController.java | 11 ++-- 16 files changed, 204 insertions(+), 98 deletions(-) create mode 100644 connectionUtils/src/main/java/com/ge/research/semtk/utility/Logger.java create mode 100644 sparqlGraphLibrary/src/test/java/com/ge/research/semtk/utility/test/LoggerTest.java diff --git a/connectionUtils/src/main/java/com/ge/research/semtk/utility/Logger.java b/connectionUtils/src/main/java/com/ge/research/semtk/utility/Logger.java new file mode 100644 index 000000000..1fb00d736 --- /dev/null +++ b/connectionUtils/src/main/java/com/ge/research/semtk/utility/Logger.java @@ -0,0 +1,66 @@ +package com.ge.research.semtk.utility; + +import java.io.PrintWriter; + +public class Logger { + + public enum Levels {INFO, WARNING, ERROR}; + private PrintWriter printWriter; + + /** + * Constructor + */ + public Logger(PrintWriter printWriter) { + this.printWriter = printWriter; + } + + /** + * Log an info message + */ + public void info(String s) { + log(s, Levels.INFO); + } + + /** + * Log a warning message + */ + public void warning(String s) { + log(s, Levels.WARNING); + } + + /** + * Log an error message + */ + public void error(String s) { + log(s, Levels.ERROR); + } + + /** + * Get the logging level from a logged line + * @param line (e.g. "INFO: performed an action") + * @return the level (e.g. Levels.INFO) + */ + public static Levels getLevel(String line) throws Exception { + String levelStr = line.substring(0, line.indexOf(":")); + if(levelStr.equals(Levels.INFO.name())) { return Levels.INFO; } + if(levelStr.equals(Levels.WARNING.name())) { return Levels.WARNING; } + if(levelStr.equals(Levels.ERROR.name())) { return Levels.ERROR; } + throw new Exception("Unrecognized level: " + levelStr); + } + + /** + * Get the message from a logged line + * @param line (e.g. "INFO: performed an action") + * @return the message, e.g. "performed an action" + */ + public static String getMessage(String line) { + return line.substring(line.indexOf(": ") + 2); + } + + // log the message + private void log(String s, Levels level) { + printWriter.println(level.name() + ": " + s); + printWriter.flush(); + } + +} diff --git a/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/load/config/LoadDataConfig.java b/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/load/config/LoadDataConfig.java index 88511a1f4..0087eb757 100644 --- a/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/load/config/LoadDataConfig.java +++ b/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/load/config/LoadDataConfig.java @@ -17,7 +17,6 @@ package com.ge.research.semtk.load.config; import java.io.File; -import java.io.PrintWriter; import java.nio.file.Files; import java.util.Arrays; import java.util.LinkedList; @@ -29,6 +28,7 @@ import com.ge.research.semtk.sparqlX.SparqlEndpointInterface; import com.ge.research.semtk.sparqlX.client.SparqlQueryClient; import com.ge.research.semtk.utility.LocalLogger; +import com.ge.research.semtk.utility.Logger; import com.ge.research.semtk.utility.Utility; /** @@ -124,9 +124,9 @@ public void addDatagraph(String datagraph) { * @param serverTypeString triple store type * @param clear if true, clears before loading * @param ingestClient Ingestor rest client - * @param progressWriter writer for reporting progress + * @param logger logger for reporting progress (may be null) */ - public void load(String modelGraph, LinkedList dataGraphs, String server, String serverType, boolean clear, IngestorRestClient ingestClient, NodeGroupExecutionClient ngeClient, SparqlQueryClient queryClient, PrintWriter progressWriter) throws Exception { + public void load(String modelGraph, LinkedList dataGraphs, String server, String serverType, boolean clear, IngestorRestClient ingestClient, NodeGroupExecutionClient ngeClient, SparqlQueryClient queryClient, Logger logger) throws Exception { try { // determine which model/data graphs to use @@ -154,11 +154,11 @@ public void load(String modelGraph, LinkedList dataGraphs, String server // execute each step for(IngestionStep step : this.getSteps()) { if(step instanceof CsvByClassIngestionStep) { - ((CsvByClassIngestionStep)step).run(conn, ingestClient, ngeClient, progressWriter); + ((CsvByClassIngestionStep)step).run(conn, ingestClient, ngeClient, logger); }else if(step instanceof CsvByNodegroupIngestionStep) { - ((CsvByNodegroupIngestionStep)step).run(conn, ngeClient, progressWriter); + ((CsvByNodegroupIngestionStep)step).run(conn, ngeClient, logger); }else if(step instanceof OwlIngestionStep) { - ((OwlIngestionStep)step).run(conn, progressWriter); + ((OwlIngestionStep)step).run(conn, logger); }else { throw new Exception("Unrecognized ingestion step"); // should not get here } @@ -206,12 +206,16 @@ public CsvByClassIngestionStep(String clazz, String baseDir, String csvPath) { public String getClazz() { return clazz; } - public void run(SparqlConnection conn, IngestorRestClient ingestClient, NodeGroupExecutionClient ngeClient, PrintWriter progressWriter) throws Exception { - writeProgress("Load CSV " + getDisplayableFilePath() + " as class " + clazz, progressWriter); + public void run(SparqlConnection conn, IngestorRestClient ingestClient, NodeGroupExecutionClient ngeClient, Logger logger) throws Exception { + if(logger != null) { + logger.info("Load CSV " + getDisplayableFilePath() + " as class " + clazz); + } String jobId = ingestClient.execFromCsvUsingClassTemplate(clazz, null, Files.readString(getFile().toPath()), conn, false, null); if(ingestClient.getWarnings() != null) { for (String warning : ingestClient.getWarnings()) { - writeProgress("Load CSV warning: " + warning, progressWriter); + if(logger != null) { + logger.warning(warning); + } } } ngeClient.waitForCompletion(jobId); @@ -231,8 +235,10 @@ public CsvByNodegroupIngestionStep(String nodegroupId, String baseDir, String cs public String getNodegroupId() { return nodegroupId; } - public void run(SparqlConnection conn, NodeGroupExecutionClient ngeClient, PrintWriter progressWriter) throws Exception { - writeProgress("Load CSV " + getDisplayableFilePath() + " using nodegroup \"" + nodegroupId + "\"", progressWriter); + public void run(SparqlConnection conn, NodeGroupExecutionClient ngeClient, Logger logger) throws Exception { + if(logger != null) { + logger.info("Load CSV " + getDisplayableFilePath() + " using nodegroup \"" + nodegroupId + "\""); + } ngeClient.dispatchIngestFromCsvStringsByIdSync(this.nodegroupId, Files.readString(getFile().toPath()), conn); } } @@ -242,12 +248,14 @@ public static class OwlIngestionStep extends FileIngestionStep{ public OwlIngestionStep(String baseDir, String owlPath) { super(baseDir, owlPath); } - public void run(SparqlConnection conn, PrintWriter progressWriter) throws Exception { + public void run(SparqlConnection conn, Logger logger) throws Exception { if(conn.getDataInterfaceCount() != 1) { throw new Exception("Error: cannot load OWL because 0 or multiple data interfaces are specified"); } SparqlEndpointInterface sei = conn.getDataInterface(0); - writeProgress("Load OWL " + getDisplayableFilePath() + " to " + sei.getGraph(), progressWriter); + if(logger != null) { + logger.info("Load OWL " + getDisplayableFilePath() + " to " + sei.getGraph()); + } sei.uploadOwl(Files.readAllBytes(getFile().toPath())); // OK to use SEI (instead of client) because uploading data (not model) } } diff --git a/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/load/config/LoadOwlConfig.java b/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/load/config/LoadOwlConfig.java index f86495af6..68a30cafe 100644 --- a/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/load/config/LoadOwlConfig.java +++ b/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/load/config/LoadOwlConfig.java @@ -17,7 +17,6 @@ package com.ge.research.semtk.load.config; import java.io.File; -import java.io.PrintWriter; import java.util.LinkedList; import java.util.List; @@ -25,6 +24,7 @@ import com.ge.research.semtk.sparqlX.SparqlEndpointInterface; import com.ge.research.semtk.sparqlX.client.SparqlQueryClient; import com.ge.research.semtk.utility.LocalLogger; +import com.ge.research.semtk.utility.Logger; import com.ge.research.semtk.utility.Utility; /** @@ -79,9 +79,9 @@ public void addFile(String file) { * @param modelGraph a model graph (optional - overrides if present) * @param server triple store location * @param serverTypeString triple store type - * @param progressWriter writer for reporting progress + * @param logger logger for reporting progress (may be null) */ - public void load(String modelGraph, String server, String serverType, SparqlQueryClient queryClient, PrintWriter progressWriter) throws Exception { + public void load(String modelGraph, String server, String serverType, SparqlQueryClient queryClient, Logger logger) throws Exception { try { // use modelGraph from method parameter if present. Else use from config YAML if present. Else use default. modelGraph = (modelGraph != null) ? modelGraph : (this.getModelgraph() != null ? this.getModelgraph() : this.defaultModelGraph ); @@ -90,7 +90,9 @@ public void load(String modelGraph, String server, String serverType, SparqlQuer // upload each OWL file to model graph for(String fileStr : this.getFiles()) { File file = new File(this.baseDir, fileStr); - writeProgress("Load OWL " + new File(this.baseDir).getName() + File.separator + file.getName() + " to " + modelGraph, progressWriter); + if(logger != null) { + logger.info("Load OWL " + new File(this.baseDir).getName() + File.separator + file.getName() + " to " + modelGraph); + } SparqlEndpointInterface sei = SparqlEndpointInterface.getInstance(serverType, server, modelGraph, username, password); queryClient.setSei(sei); queryClient.uploadOwl(file); diff --git a/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/load/config/ManifestConfig.java b/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/load/config/ManifestConfig.java index 1696a7d42..aa390a2b2 100644 --- a/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/load/config/ManifestConfig.java +++ b/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/load/config/ManifestConfig.java @@ -17,7 +17,6 @@ package com.ge.research.semtk.load.config; import java.io.File; -import java.io.PrintWriter; import java.util.LinkedList; import org.apache.commons.math3.util.Pair; @@ -29,6 +28,7 @@ import com.ge.research.semtk.sparqlX.SparqlConnection; import com.ge.research.semtk.sparqlX.SparqlEndpointInterface; import com.ge.research.semtk.sparqlX.client.SparqlQueryClient; +import com.ge.research.semtk.utility.Logger; import com.ge.research.semtk.utility.Utility; /** @@ -212,17 +212,21 @@ public SparqlConnection getFootprintConnection(String server, String serverTypeS * @param clear if true, clears the footprint graphs (before loading) * @param topLevel true if this is a top-level manifest, false for recursively calling sub-manifests * @param ingestClient ingestionRestClient - * @param progressWriter writer for reporting progress + * @param logger logger for reporting progress (may be null) */ - public void load(String server, String serverTypeString, boolean clear, boolean topLevel, IngestorRestClient ingestClient, NodeGroupExecutionClient ngeClient, NodeGroupStoreRestClient ngStoreClient, SparqlQueryClient queryClient, PrintWriter progressWriter) throws Exception { + public void load(String server, String serverTypeString, boolean clear, boolean topLevel, IngestorRestClient ingestClient, NodeGroupExecutionClient ngeClient, NodeGroupStoreRestClient ngStoreClient, SparqlQueryClient queryClient, Logger logger) throws Exception { - writeProgress("Loading manifest for '" + getName() + "'...", progressWriter); + if(logger != null) { + logger.info("Loading manifest for '" + getName() + "'..."); + } // clear graphs first if(clear) { // clear each model and data graph in the footprint for(String g : getGraphsFootprint()) { - writeProgress("Clear graph " + g, progressWriter); + if(logger != null) { + logger.info("Clear graph " + g); + } queryClient.setSei(SparqlEndpointInterface.getInstance(serverTypeString, server, g, username, password)); queryClient.clearAll(); } @@ -237,32 +241,36 @@ public void load(String server, String serverTypeString, boolean clear, boolean // load via an owl ingestion YAML File stepFile = new File(baseDir, (String)step.getValue()); LoadOwlConfig config = new LoadOwlConfig(stepFile, this.defaultModelGraph); - config.load(null, server, serverTypeString, queryClient, progressWriter); + config.load(null, server, serverTypeString, queryClient, logger); }else if(type == StepType.DATA) { // load content using CSV ingestion YAML File stepFile = new File(baseDir, (String)step.getValue()); LoadDataConfig config = new LoadDataConfig(stepFile, this.defaultModelGraph, this.defaultDataGraph); - config.load(null, null, server, serverTypeString, false, ingestClient, ngeClient, queryClient, progressWriter); + config.load(null, null, server, serverTypeString, false, ingestClient, ngeClient, queryClient, logger); }else if(type == StepType.NODEGROUPS) { // load nodegroups/reports from a directory - writeProgress("Store nodegroups", progressWriter); + if(logger != null) { + logger.info("Store nodegroups"); + } File nodegroupsDirectory = new File(baseDir, (String)step.getValue()); File csvFile = new File(nodegroupsDirectory, "store_data.csv"); - ngStoreClient.loadStoreDataCsv(csvFile.getAbsolutePath(), null, progressWriter); + ngStoreClient.loadStoreDataCsv(csvFile.getAbsolutePath(), null, logger); }else if(type == StepType.MANIFEST) { // load content using sub-manifest File stepFile = new File(baseDir, (String)step.getValue()); ManifestConfig subManifest = new ManifestConfig(stepFile, defaultModelGraph, defaultDataGraph); - subManifest.load(server, serverTypeString, false, false, ingestClient, ngeClient, ngStoreClient, queryClient, progressWriter); + subManifest.load(server, serverTypeString, false, false, ingestClient, ngeClient, ngStoreClient, queryClient, logger); }else if(type == StepType.COPYGRAPH) { // perform the copy String fromGraph = (String)((Pair)step.getValue()).getFirst(); String toGraph = (String)((Pair)step.getValue()).getSecond(); - writeProgress("Copy " + fromGraph + " to " + toGraph, progressWriter); + if(logger != null) { + logger.info("Copy " + fromGraph + " to " + toGraph); + } ngeClient.copyGraph(server, serverTypeString, fromGraph, server, serverTypeString, toGraph); }else { @@ -277,21 +285,29 @@ public void load(String server, String serverTypeString, boolean clear, boolean String copyToGraph = this.getCopyToGraph(); if(copyToGraph != null) { if(clear) { - writeProgress("Clear graph " + copyToGraph, progressWriter); + if(logger != null) { + logger.info("Clear graph " + copyToGraph); + } queryClient.setSei(SparqlEndpointInterface.getInstance(serverTypeString, server, copyToGraph, username, password)); queryClient.clearAll(); } for(String graph : this.getGraphsFootprint()) { // copy each model/data footprint graph to the given graph - writeProgress("Copy graph " + graph + " to " + copyToGraph, progressWriter); + if(logger != null) { + logger.info("Copy graph " + graph + " to " + copyToGraph); + } String msg = ngeClient.copyGraph(server, serverTypeString, graph, server, serverTypeString, copyToGraph); - writeProgress(msg, progressWriter); + if(logger != null) { + logger.info(msg); + } } } // entity resolution String entityResolutionGraph = this.getEntityResolutionGraph(); if(entityResolutionGraph != null) { - writeProgress("Perform entity resolution in " + entityResolutionGraph, progressWriter); + if(logger != null) { + logger.info("Perform entity resolution in " + entityResolutionGraph); + } try { SparqlConnection conn = new SparqlConnection("Entity Resolution", serverTypeString, server, entityResolutionGraph); ngeClient.combineEntitiesInConn(conn); diff --git a/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/load/config/YamlConfig.java b/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/load/config/YamlConfig.java index 962210346..d057621eb 100644 --- a/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/load/config/YamlConfig.java +++ b/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/load/config/YamlConfig.java @@ -1,7 +1,6 @@ package com.ge.research.semtk.load.config; import java.io.File; -import java.io.PrintWriter; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; @@ -19,6 +18,7 @@ public abstract class YamlConfig { protected JsonNode configNode; // this file as a JsonNode protected String username = "YamlConfigUser"; // no user or password functionality yet. protected String password = "YamlConfigPassword"; + /** * Constructor */ @@ -83,12 +83,4 @@ protected static String getStringOrFirstArrayEntry(JsonNode node) throws Excepti return null; } - /** - * Convenience method to write/flush - */ - protected static void writeProgress(String s, PrintWriter progressWriter) { - progressWriter.println(s); - progressWriter.flush(); - } - } diff --git a/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/nodeGroupStore/client/NodeGroupStoreRestClient.java b/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/nodeGroupStore/client/NodeGroupStoreRestClient.java index a31f52de3..928f3486b 100644 --- a/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/nodeGroupStore/client/NodeGroupStoreRestClient.java +++ b/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/nodeGroupStore/client/NodeGroupStoreRestClient.java @@ -18,12 +18,7 @@ package com.ge.research.semtk.nodeGroupStore.client; import java.io.File; -import java.io.FileReader; -import java.io.PrintWriter; import java.nio.file.Paths; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Hashtable; import org.json.simple.JSONObject; @@ -33,12 +28,11 @@ import com.ge.research.semtk.resultSet.TableResultSet; import com.ge.research.semtk.services.client.RestClient; import com.ge.research.semtk.services.nodegroupStore.NgStore; -import com.ge.research.semtk.services.nodegroupStore.NgStore.StoredItemTypes; import com.ge.research.semtk.services.nodegroupStore.StoreDataCsvReader; import com.ge.research.semtk.sparqlX.SparqlConnection; import com.ge.research.semtk.utility.LocalLogger; +import com.ge.research.semtk.utility.Logger; import com.ge.research.semtk.utility.Utility; -import com.opencsv.CSVReader; public class NodeGroupStoreRestClient extends RestClient { @@ -46,13 +40,11 @@ public class NodeGroupStoreRestClient extends RestClient { @Override public void buildParametersJSON() throws Exception { // TODO Auto-generated method stub - } @Override public void handleEmptyResponse() throws Exception { // TODO Auto-generated method stub - } public NodeGroupStoreRestClient (NodeGroupStoreConfig config) { @@ -65,6 +57,7 @@ public NodeGroupStoreRestClient (NodeGroupStoreConfig config) { * @return * @throws Exception */ + @SuppressWarnings("unchecked") public TableResultSet executeGetNodeGroupById(String id) throws Exception{ TableResultSet retval = new TableResultSet(); @@ -137,6 +130,7 @@ public TableResultSet executeGetNodeGroupMetadata() throws Exception { * @return * @throws Exception */ + @SuppressWarnings("unchecked") public TableResultSet executeGetStoredItemsMetadata(NgStore.StoredItemTypes itemType) throws Exception { TableResultSet retval = new TableResultSet(); @@ -157,6 +151,7 @@ public TableResultSet executeGetStoredItemsMetadata(NgStore.StoredItemTypes item return retval; } + @SuppressWarnings("unchecked") public TableResultSet executeGetNodeGroupRuntimeConstraints(String nodeGroupId) throws Exception { TableResultSet retval = new TableResultSet(); @@ -243,6 +238,7 @@ public SimpleResultSet deleteStoredNodeGroup(String nodeGroupID) throws DoesNotE * @throws DoesNotExistException - nodegroup doesn't exist * @throws Exception - other error in the REST call */ + @SuppressWarnings("unchecked") public SimpleResultSet deleteStoredItem(String nodeGroupID, NgStore.StoredItemTypes itemType) throws DoesNotExistException, Exception{ SimpleResultSet retval = null; @@ -275,16 +271,15 @@ public SimpleResultSet deleteStoredItem(String nodeGroupID, NgStore.StoredItemTy * Process a store_data.csv file * @param csvFileName * @param sparqlConnOverrideFile - override all the nodegroup connections - * @param statusWriter - get status messages + * @param logger - write status messages * @throws Exception */ - public void loadStoreDataCsv(String csvFileName, String sparqlConnOverrideFile, PrintWriter statusWriter) throws Exception { + public void loadStoreDataCsv(String csvFileName, String sparqlConnOverrideFile, Logger logger) throws Exception { - StoreDataCsvReader br = new StoreDataCsvReader(csvFileName, statusWriter); + StoreDataCsvReader br = new StoreDataCsvReader(csvFileName); while (br.readNext() != null) { - String ngId = br.getId(); String ngComments = br.getComments(); String ngOwner = br.getCreator(); @@ -300,7 +295,9 @@ public void loadStoreDataCsv(String csvFileName, String sparqlConnOverrideFile, // add this.storeItem(ngId, ngComments, ngOwner, ngFilePath, itemType, sparqlConnOverrideFile, true); - if (statusWriter != null) { statusWriter.println("Stored: " + ngId); statusWriter.flush(); } + if (logger != null) { + logger.info("Stored: " + ngId); + } } } diff --git a/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/services/nodegroupStore/StoreDataCsvReader.java b/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/services/nodegroupStore/StoreDataCsvReader.java index c337679b6..f95bb89ee 100644 --- a/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/services/nodegroupStore/StoreDataCsvReader.java +++ b/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/services/nodegroupStore/StoreDataCsvReader.java @@ -2,8 +2,6 @@ import java.io.FileReader; import java.io.IOException; -import java.io.PrintWriter; -import java.util.Arrays; import java.util.Hashtable; import com.ge.research.semtk.services.nodegroupStore.NgStore.StoredItemTypes; @@ -13,19 +11,18 @@ public class StoreDataCsvReader extends StoreDataCsv { String filename = null; CSVReader br = null; - PrintWriter writer = null; Hashtable colHash = null; int colCount = 0; int lineNumber; String [] parsedLine = null; - public StoreDataCsvReader(String csvFileName, PrintWriter writer) throws Exception { + public StoreDataCsvReader(String csvFileName) throws Exception { this.br = new CSVReader(new FileReader(csvFileName)); this.filename = csvFileName; this.lineNumber = 0; this.checkHeaders(); } - + private void checkHeaders() throws Exception { String[] parsedLine = br.readNext(); // header line diff --git a/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/load/config/test/LoadDataConfigTest_IT.java b/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/load/config/test/LoadDataConfigTest_IT.java index 337624887..48f277ab1 100644 --- a/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/load/config/test/LoadDataConfigTest_IT.java +++ b/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/load/config/test/LoadDataConfigTest_IT.java @@ -24,7 +24,6 @@ import org.junit.Test; import java.io.File; -import java.io.PrintWriter; import java.nio.file.Paths; import java.util.Arrays; import java.util.LinkedList; @@ -113,15 +112,15 @@ public void testLoad() throws Exception{ // Case 1: if load() data graph parameter, then confirm loads there clearGraphs(); - loadOwlConfig.load(modelSei.getGraph(), TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), IntegrationTestUtility.getSparqlQueryAuthClient(), new PrintWriter(System.out)); // loads OWL - loadDataConfig.load(modelSei.getGraph(), new LinkedList(Arrays.asList(dataSei.getGraph())), TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), false, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), new PrintWriter(System.out)); + loadOwlConfig.load(modelSei.getGraph(), TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), IntegrationTestUtility.getSparqlQueryAuthClient(), null); // loads OWL + loadDataConfig.load(modelSei.getGraph(), new LinkedList(Arrays.asList(dataSei.getGraph())), TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), false, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), null); assertEquals(dataSei.getNumTriples(), NUM_EXPECTED_TRIPLES); assertEquals(dataFallbackSei.getNumTriples(), 0); // Case 2: if no load() data graph parameter, then confirm loads to YAML data graph if present "http://junit/rack001/data" clearGraphs(); - loadOwlConfig.load(null, TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), IntegrationTestUtility.getSparqlQueryAuthClient(), new PrintWriter(System.out)); // loads OWL to fallback - loadDataConfig.load(null, null, TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), false, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), new PrintWriter(System.out)); + loadOwlConfig.load(null, TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), IntegrationTestUtility.getSparqlQueryAuthClient(), null); // loads OWL to fallback + loadDataConfig.load(null, null, TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), false, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), null); SparqlEndpointInterface dataSeiFromYaml = TestGraph.getSei(loadDataConfig.getDatagraphs().get(0)); // this is what's in the YAML assertEquals(dataSei.getNumTriples(), 0); assertEquals(dataSeiFromYaml.getNumTriples(), NUM_EXPECTED_TRIPLES); @@ -129,9 +128,9 @@ public void testLoad() throws Exception{ // Case 3: if no load() data graph parameter, and no YAML data graph, then confirm loads to fallback clearGraphs(); - loadOwlConfig.load(null, TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), IntegrationTestUtility.getSparqlQueryAuthClient(), new PrintWriter(System.out)); // loads OWL to fallback + loadOwlConfig.load(null, TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), IntegrationTestUtility.getSparqlQueryAuthClient(), null); // loads OWL to fallback loadDataConfig.setDataGraphs(null); // no datagraph in YAML - loadDataConfig.load(null, null, TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), false, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), new PrintWriter(System.out)); + loadDataConfig.load(null, null, TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), false, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), null); assertEquals(dataSei.getNumTriples(), 0); assertEquals(dataFallbackSei.getNumTriples(), NUM_EXPECTED_TRIPLES); @@ -160,7 +159,7 @@ public void testLoad_ByNodegroup() throws Exception{ // load data by nodegroup File tempDir = TestGraph.unzipAndUniquifyJunitGraphs(this, "/config/LoadDataConfig-byNodegroup.zip"); LoadDataConfig loadDataConfig = new LoadDataConfig(Paths.get(tempDir.getAbsolutePath(), "import.yaml").toFile(), TestGraph.getSei().getGraph(), TestGraph.getSei().getGraph()); - loadDataConfig.load(TestGraph.getSei().getGraph(), new LinkedList(Arrays.asList(TestGraph.getSei().getGraph())), TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), false, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), new PrintWriter(System.out)); + loadDataConfig.load(TestGraph.getSei().getGraph(), new LinkedList(Arrays.asList(TestGraph.getSei().getGraph())), TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), false, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), null); assertEquals(TestGraph.getSei().getNumTriples(), 334); } finally{ diff --git a/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/load/config/test/LoadOwlConfigTest_IT.java b/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/load/config/test/LoadOwlConfigTest_IT.java index 665b5e03c..d69bc1988 100644 --- a/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/load/config/test/LoadOwlConfigTest_IT.java +++ b/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/load/config/test/LoadOwlConfigTest_IT.java @@ -91,7 +91,7 @@ public void testLoad() throws Exception{ // Case 1: if load() model graph parameter, then confirm loads there clearGraphs(); - config.load(modelSei.getGraph(), TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), IntegrationTestUtility.getSparqlQueryAuthClient(), new PrintWriter(System.out)); + config.load(modelSei.getGraph(), TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), IntegrationTestUtility.getSparqlQueryAuthClient(), null); assertEquals(modelSei.getNumTriples(), NUM_EXPECTED_TRIPLES); assertEquals(modelFallbackSei.getNumTriples(), 0); @@ -100,7 +100,7 @@ public void testLoad() throws Exception{ config.setModelgraph(TestGraph.uniquifyJunitGraphs("http://junit/rack001/model")); // add model graph to YAML config SparqlEndpointInterface modelSeiFromYaml = TestGraph.getSei(config.getModelgraph()); IntegrationTestUtility.clearGraph(modelSeiFromYaml); - config.load(null, TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), IntegrationTestUtility.getSparqlQueryAuthClient(), new PrintWriter(System.out)); + config.load(null, TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), IntegrationTestUtility.getSparqlQueryAuthClient(), null); assertEquals(modelSei.getNumTriples(), 0); assertEquals(modelFallbackSei.getNumTriples(), 0); assertEquals(modelSeiFromYaml.getNumTriples(), NUM_EXPECTED_TRIPLES); @@ -108,7 +108,7 @@ public void testLoad() throws Exception{ // Case 3: if no load() model graph parameter, and no YAML data graph, then confirm loads to fallback clearGraphs(); config.setModelgraph(null); // remove model graph from YAML config - config.load(null, TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), IntegrationTestUtility.getSparqlQueryAuthClient(), new PrintWriter(System.out)); + config.load(null, TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), IntegrationTestUtility.getSparqlQueryAuthClient(), null); assertEquals(modelSei.getNumTriples(), 0); assertEquals(modelFallbackSei.getNumTriples(), NUM_EXPECTED_TRIPLES); diff --git a/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/load/config/test/ManifestConfigTest_IT.java b/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/load/config/test/ManifestConfigTest_IT.java index cbad16871..dd9597415 100644 --- a/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/load/config/test/ManifestConfigTest_IT.java +++ b/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/load/config/test/ManifestConfigTest_IT.java @@ -67,7 +67,7 @@ public void testLoadManifest() throws Exception{ reset(); manifest.setCopyToGraph(null); manifest.setEntityResolutionGraph(null); - manifest.load(TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), true, true, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getNodeGroupStoreRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), new PrintWriter(System.out)); + manifest.load(TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), true, true, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getNodeGroupStoreRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), null); assertEquals("Number of triples loaded to model graph", NUM_EXPECTED_TRIPLES_MODEL, modelFallbackSei.getNumTriples()); assertEquals("Number of triples loaded to data graph", NUM_EXPECTED_TRIPLES_DATA, dataSeiFromYaml.getNumTriples()); assertEquals("Number of triples loaded to default graph", 0, defaultGraphSei.getNumTriples()); @@ -76,7 +76,7 @@ public void testLoadManifest() throws Exception{ // test copy-graph step reset(); manifest.addStep(new Step(StepType.COPYGRAPH, new Pair(TestGraph.uniquifyJunitGraphs("http://junit/rack001/data"), TestGraph.uniquifyJunitGraphs("http://junit/rack001/data/copy")))); - manifest.load(TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), true, true, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getNodeGroupStoreRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), new PrintWriter(System.out)); + manifest.load(TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), true, true, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getNodeGroupStoreRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), null); assertEquals("Number of triples loaded to data graph", NUM_EXPECTED_TRIPLES_DATA, dataSeiFromYaml.getNumTriples()); assertEquals("Number of triples loaded to data graph copy", NUM_EXPECTED_TRIPLES_DATA, dataSeiFromYamlCopy.getNumTriples()); @@ -105,7 +105,7 @@ public void testLoadManifest_UsesDefaultGraph() throws Exception{ reset(); manifest.setCopyToGraph(SparqlEndpointInterface.SEMTK_DEFAULT_GRAPH_NAME); manifest.setEntityResolutionGraph(null); - manifest.load(TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), false, true, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getNodeGroupStoreRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), new PrintWriter(System.out)); + manifest.load(TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), false, true, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getNodeGroupStoreRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), null); assertEquals("Number of triples loaded to model graph", NUM_EXPECTED_TRIPLES_MODEL, modelFallbackSei.getNumTriples()); assertEquals("Number of triples loaded to data graph", NUM_EXPECTED_TRIPLES_DATA, dataSeiFromYaml.getNumTriples()); assertEquals("Number of triples loaded to default graph", NUM_EXPECTED_TRIPLES_MODEL + NUM_EXPECTED_TRIPLES_DATA, defaultGraphSei.getNumTriples()); @@ -114,7 +114,7 @@ public void testLoadManifest_UsesDefaultGraph() throws Exception{ reset(); manifest.setCopyToGraph(SparqlEndpointInterface.SEMTK_DEFAULT_GRAPH_NAME); manifest.setEntityResolutionGraph(SparqlEndpointInterface.SEMTK_DEFAULT_GRAPH_NAME); - manifest.load(TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), false, true, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getNodeGroupStoreRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), new PrintWriter(System.out)); + manifest.load(TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), false, true, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getNodeGroupStoreRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), null); assertEquals("Number of triples loaded to model graph", NUM_EXPECTED_TRIPLES_MODEL, modelFallbackSei.getNumTriples()); assertEquals("Number of triples loaded to data graph", NUM_EXPECTED_TRIPLES_DATA, dataSeiFromYaml.getNumTriples()); assertEquals("Number of triples loaded to default graph", NUM_EXPECTED_TRIPLES_MODEL + NUM_EXPECTED_TRIPLES_DATA + NUM_NET_CHANGE_ENTITY_RESOLUTION_TRIPLES, defaultGraphSei.getNumTriples()); diff --git a/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/load/config/test/ManifestConfigTest_LoadTurnstile_IT.java b/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/load/config/test/ManifestConfigTest_LoadTurnstile_IT.java index 5029fb0a8..722b52ac1 100644 --- a/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/load/config/test/ManifestConfigTest_LoadTurnstile_IT.java +++ b/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/load/config/test/ManifestConfigTest_LoadTurnstile_IT.java @@ -63,7 +63,7 @@ public void test() throws Exception{ reset(); manifest.setCopyToGraph(null); manifest.setEntityResolutionGraph(null); - manifest.load(TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), true, true, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getNodeGroupStoreRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), new PrintWriter(System.out)); + manifest.load(TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), true, true, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getNodeGroupStoreRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), null); assertEquals("Number of triples loaded to model fallback graph", 1986, modelFallbackSei.getNumTriples()); assertEquals("Number of triples loaded to data graph 1", 338, dataSeiFromYaml1.getNumTriples()); assertEquals("Number of triples loaded to data graph 2", 104, dataSeiFromYaml2.getNumTriples()); // full Turnstile example has more data loaded here diff --git a/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/nodegroupstore/test/NodeGroupStoreTest_IT.java b/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/nodegroupstore/test/NodeGroupStoreTest_IT.java index e8cec7e97..52e1be21d 100644 --- a/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/nodegroupstore/test/NodeGroupStoreTest_IT.java +++ b/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/nodegroupstore/test/NodeGroupStoreTest_IT.java @@ -250,8 +250,7 @@ public void testLoadStoreDataCsv() throws Exception { IntegrationTestUtility.cleanupNodegroupStore("junit"); try { File tempDir = TestGraph.unzipAndUniquifyJunitGraphs(this, "/store_data.zip"); - - nodeGroupStoreClient.loadStoreDataCsv(Paths.get(tempDir.getAbsolutePath(), "store_data.csv").toString(), null, new PrintWriter(System.out)); + nodeGroupStoreClient.loadStoreDataCsv(Paths.get(tempDir.getAbsolutePath(), "store_data.csv").toString(), null, null); assertEquals(4, IntegrationTestUtility.countItemsInStoreByCreator("junit")); } finally { try { @@ -260,5 +259,4 @@ public void testLoadStoreDataCsv() throws Exception { } } - } diff --git a/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/utility/test/LoggerTest.java b/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/utility/test/LoggerTest.java new file mode 100644 index 000000000..5d41522ea --- /dev/null +++ b/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/utility/test/LoggerTest.java @@ -0,0 +1,28 @@ +package com.ge.research.semtk.utility.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import org.junit.Test; + +import com.ge.research.semtk.utility.Logger; +import com.ge.research.semtk.utility.Logger.Levels; + +public class LoggerTest { + + @Test + public void test() throws Exception { + assertEquals(Logger.getMessage("INFO: loaded data"), "loaded data"); + assertEquals(Logger.getLevel("INFO: loaded data"), Levels.INFO); + assertEquals(Logger.getLevel("WARNING: missing column"), Levels.WARNING); + assertEquals(Logger.getLevel("ERROR: load failed"), Levels.ERROR); + + try { + Logger.getLevel("MESSAGE: loaded data"); + fail(); // should not get here + }catch(Exception e) { + assert(e.getMessage().contains("Unrecognized level")); + } + } + +} diff --git a/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/utility/test/UtilityClientTest_IT.java b/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/utility/test/UtilityClientTest_IT.java index df80dcdac..b9d5acd7c 100644 --- a/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/utility/test/UtilityClientTest_IT.java +++ b/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/utility/test/UtilityClientTest_IT.java @@ -73,17 +73,18 @@ public void testLoadIngestionPackage() throws Exception { String response = Utility.readToString(reader); // check the response stream - assertTrue("Bad response:\n" + response, response.contains("Loading manifest for 'Entity Resolution'...")); - assertTrue(response.contains("Loading manifest for 'RACK ontology'...")); - assertTrue(response.matches("(.*)Clear graph http://junit/(.*)/rack001/model(.*)")); - assertTrue(response.matches("(.*)Clear graph http://junit/(.*)/rack001/data(.*)")); - assertTrue(response.matches("(.*)Load OWL OwlModels(.*)AGENTS.owl(.*)")); - assertTrue(response.contains("Store nodegroups")); - assertTrue(response.contains("Stored: JUNIT query Files of a Given Format")); - assertTrue(response.matches("(.*)Load CSV Package-1(.*)PROV_S_ACTIVITY1.csv as class http://arcos.rack/PROV-S#ACTIVITY(.*)")); - assertTrue(response.matches("(.*)Copy graph http://junit/(.*)/auto/rack001/data to uri://DefaultGraph(.*)")); - assertTrue(response.contains("Perform entity resolution")); - assertTrue(response.contains("Load complete")); + assertTrue("Bad response:\n" + response, response.contains("INFO: Loading manifest for 'Entity Resolution'...")); + assertTrue(response.contains("INFO: Loading manifest for 'RACK ontology'...")); + assertTrue(response.matches("(.*)INFO: Clear graph http://junit/(.*)/rack001/model(.*)")); + assertTrue(response.matches("(.*)INFO: Clear graph http://junit/(.*)/rack001/data(.*)")); + assertTrue(response.matches("(.*)INFO: Load OWL OwlModels(.*)AGENTS.owl(.*)")); + assertTrue(response.contains("INFO: Store nodegroups")); + assertTrue(response.contains("INFO: Stored: JUNIT query Files of a Given Format")); + assertTrue(response.matches("(.*)INFO: Load CSV Package-1(.*)PROV_S_ACTIVITY1.csv as class http://arcos.rack/PROV-S#ACTIVITY(.*)")); + assertTrue(response.contains("WARNING: Input is missing these columns:")); + assertTrue(response.matches("(.*)INFO: Copy graph http://junit/(.*)/auto/rack001/data to uri://DefaultGraph(.*)")); + assertTrue(response.contains("INFO: Perform entity resolution")); + assertTrue(response.contains("INFO: Load complete")); // check the counts assertEquals("Number of triples loaded to model graph", ManifestConfigTest_IT.NUM_EXPECTED_TRIPLES_MODEL, modelFallbackSei.getNumTriples()); @@ -105,11 +106,11 @@ public void testLoadIngestionPackage_errorConditions() throws Exception { // not a zip file response = Utility.readToString(client.execLoadIngestionPackage(Utility.getResourceAsTempFile(this,"/animalQuery.json"), TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), false, modelFallbackSei.getGraph(), dataFallbackSei.getGraph())); - assertTrue(response.contains("Error: This endpoint only accepts ingestion packages in zip file format")); + assertTrue(response.contains("ERROR: This endpoint only accepts ingestion packages in zip file format")); // contains no top-level manifest.yaml response = Utility.readToString(client.execLoadIngestionPackage(Utility.getResourceAsTempFile(this,"/config/IngestionPackageNoManifest.zip"), TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), false, modelFallbackSei.getGraph(), dataFallbackSei.getGraph())); - assertTrue(response.contains("Error: Cannot find a top-level manifest")); + assertTrue(response.contains("ERROR: Cannot find a top-level manifest")); } // clear graphs and nodegroup store diff --git a/standaloneExecutables/src/main/java/com/ge/research/semtk/standaloneExecutables/StoreNodeGroup.java b/standaloneExecutables/src/main/java/com/ge/research/semtk/standaloneExecutables/StoreNodeGroup.java index 63c0ac260..d65cbb2cd 100644 --- a/standaloneExecutables/src/main/java/com/ge/research/semtk/standaloneExecutables/StoreNodeGroup.java +++ b/standaloneExecutables/src/main/java/com/ge/research/semtk/standaloneExecutables/StoreNodeGroup.java @@ -19,6 +19,7 @@ package com.ge.research.semtk.standaloneExecutables; import com.ge.research.semtk.utility.LocalLogger; +import com.ge.research.semtk.utility.Logger; import com.ge.research.semtk.nodeGroupStore.client.NodeGroupStoreConfig; import com.ge.research.semtk.nodeGroupStore.client.NodeGroupStoreRestClient; @@ -49,7 +50,7 @@ public static void main(String[] args) throws Exception{ NodeGroupStoreRestClient client = new NodeGroupStoreRestClient(config); try { PrintWriter out = new PrintWriter(System.out); - client.loadStoreDataCsv(csvFile, sparqlConnOverrideFile, out); + client.loadStoreDataCsv(csvFile, sparqlConnOverrideFile, new Logger(out)); out.close(); } catch(Exception e){ LocalLogger.printStackTrace(e); diff --git a/utilityService/src/main/java/com/ge/research/semtk/services/utility/UtilityServiceRestController.java b/utilityService/src/main/java/com/ge/research/semtk/services/utility/UtilityServiceRestController.java index 05558224f..da855d152 100755 --- a/utilityService/src/main/java/com/ge/research/semtk/services/utility/UtilityServiceRestController.java +++ b/utilityService/src/main/java/com/ge/research/semtk/services/utility/UtilityServiceRestController.java @@ -66,6 +66,7 @@ import com.ge.research.semtk.springutillib.properties.NodegroupStoreServiceProperties; import com.ge.research.semtk.springutillib.properties.QueryServiceProperties; import com.ge.research.semtk.utility.LocalLogger; +import com.ge.research.semtk.utility.Logger; import com.ge.research.semtk.utility.Utility; import io.swagger.v3.oas.annotations.Operation; @@ -387,12 +388,14 @@ public void loadIngestionPackage( @RequestParam("serverAndPort") String serverAn final String ENDPOINT_NAME = "loadIngestionPackage"; LocalLogger.logToStdOut(SERVICE_NAME + " " + ENDPOINT_NAME); PrintWriter responseWriter = null; + Logger responseLogger = null; // set up try { resp.addHeader("content-type", "text/plain; charset=utf-8"); resp.flushBuffer(); responseWriter = resp.getWriter(); + responseLogger = new Logger(responseWriter); }catch(Exception e) { LocalLogger.printStackTrace(e); } @@ -418,15 +421,13 @@ public void loadIngestionPackage( @RequestParam("serverAndPort") String serverAn throw new Exception("Cannot find a top-level manifest in " + ingestionPackageZipFile.getOriginalFilename()); } ManifestConfig manifest = new ManifestConfig(manifestFile, defaultModelGraph, defaultDataGraph); - manifest.load(serverAndPort, serverType, clear, true, ingest_prop.getClient(), ngexec_prop.getClient(), ngstore_prop.getClient(), getSparqlQueryClient(), responseWriter); + manifest.load(serverAndPort, serverType, clear, true, ingest_prop.getClient(), ngexec_prop.getClient(), ngstore_prop.getClient(), getSparqlQueryClient(), responseLogger); - responseWriter.println("Load complete"); - responseWriter.flush(); + responseLogger.info("Load complete"); LocalLogger.logToStdOut(SERVICE_NAME + " " + ENDPOINT_NAME + " completed"); } catch (Exception e){ - responseWriter.println("Error: " + e.getMessage()); - responseWriter.flush(); + responseLogger.error(e.getMessage()); LocalLogger.printStackTrace(e); }finally { try {