diff --git a/connectionUtils/pom.xml b/connectionUtils/pom.xml index dca48a26d..1d50e80f4 100644 --- a/connectionUtils/pom.xml +++ b/connectionUtils/pom.xml @@ -103,6 +103,13 @@ jena-arq 4.9.0 + + + + org.apache.jena + jena-rdfconnection + 4.9.0 + diff --git a/connectionUtils/src/main/java/com/ge/research/semtk/sparqlX/SparqlConnection.java b/connectionUtils/src/main/java/com/ge/research/semtk/sparqlX/SparqlConnection.java index f13887e62..d139c0731 100644 --- a/connectionUtils/src/main/java/com/ge/research/semtk/sparqlX/SparqlConnection.java +++ b/connectionUtils/src/main/java/com/ge/research/semtk/sparqlX/SparqlConnection.java @@ -1,5 +1,5 @@ /** - ** Copyright 2016-2018 General Electric Company + ** Copyright 2016-2023 General Electric Company ** ** ** Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,6 +21,9 @@ import java.util.ArrayList; import java.util.Arrays; +import org.apache.jena.graph.Graph; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdfconnection.RDFConnection; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; @@ -598,4 +601,35 @@ public String getUniqueKey() { ret.append(this.enableOwlImports ? "owlImports;" : "noImports;"); return ret.toString(); } + + + /** + * Create a Jena Graph from this SparqlConnection + */ + public Graph getJenaGraph() { + ArrayList fetched = new ArrayList(); + + Model model = null; + RDFConnection rdfConn = null; + try { + for(SparqlEndpointInterface sei : this.getAllInterfaces()) { + rdfConn = RDFConnection.connect(sei.getServerAndPort()); + String graphName = sei.getGraph(); + if(fetched.contains(sei.getServerAndPort() + graphName)) { + continue; // skip if already fetched it (e.g. if model graph is the same as data graph) + } + Model m = rdfConn.fetch(graphName); + fetched.add(sei.getServerAndPort() + graphName); + if(model == null) { + model = m; + }else { + model.add(m); + } + rdfConn.close(); + } + return model.getGraph(); + }finally { + rdfConn.close(); + } + } } diff --git a/connectionUtils/src/main/java/com/ge/research/semtk/utility/Utility.java b/connectionUtils/src/main/java/com/ge/research/semtk/utility/Utility.java index 0d8df9980..51cccca5d 100644 --- a/connectionUtils/src/main/java/com/ge/research/semtk/utility/Utility.java +++ b/connectionUtils/src/main/java/com/ge/research/semtk/utility/Utility.java @@ -82,6 +82,12 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.text.StrSubstitutor; +import org.apache.jena.graph.Graph; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFDataMgr; +import org.apache.jena.riot.RDFParser; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; @@ -347,6 +353,17 @@ public static boolean arraysSameMinusOrder(String[] arr1, String[] arr2) { return Arrays.equals(arr1Clone, arr2Clone); } + /** + * Determine if two JSONObjects are equivalent. + * Ignores key order (but not JSONArray list order) + */ + public static boolean equals(JSONObject o1, JSONObject o2) throws Exception { + ObjectMapper mapper = new ObjectMapper(); + JsonNode json1 = mapper.readTree(o1.toString()); + JsonNode json2 = mapper.readTree(o2.toString()); + return json1.equals(json2); + } + public static String readFile(String path) throws IOException { return FileUtils.readFileToString(new File(path)); } @@ -1190,5 +1207,36 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) } }); } + + /** + * Create a Jena Graph from Turtle data + * @param ttlInputStream Turtle data in an input stream + * @return the Jena graph + * @throws Exception + */ + public static Graph getJenaGraphFromTurtle(InputStream ttlInputStream) throws Exception { + try { + return RDFParser.source(ttlInputStream).lang(Lang.TTL).toGraph(); + }catch(Exception e) { + throw new Exception("Error creating graph from Turtle: " + e.getMessage(), e); + } + } + /** + * Get a Turtle string for a given Jena Graph + * @param graph the graph + * @return the Turtle string + */ + public static String getTurtleFromJenaGraph(Graph graph) throws IOException { + OutputStream outputStream = null; + try { + Model model = ModelFactory.createModelForGraph(graph); + outputStream = new ByteArrayOutputStream(); + RDFDataMgr.write(outputStream, model, Lang.TTL); + }finally { + outputStream.close(); + } + return outputStream.toString(); + } + } diff --git a/sparqlGraphLibrary/pom.xml b/sparqlGraphLibrary/pom.xml index a2587f863..17093455b 100644 --- a/sparqlGraphLibrary/pom.xml +++ b/sparqlGraphLibrary/pom.xml @@ -105,6 +105,13 @@ jena-arq 4.9.0 + + + + org.apache.jena + jena-shacl + 4.9.0 + diff --git a/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/belmont/ValueConstraint.java b/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/belmont/ValueConstraint.java index ea64a8017..79893d0bb 100644 --- a/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/belmont/ValueConstraint.java +++ b/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/belmont/ValueConstraint.java @@ -25,8 +25,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.jena.sparql.lang.SPARQLParserFactory; - import com.ge.research.semtk.sparqlX.SparqlEndpointInterface; import com.ge.research.semtk.sparqlX.XSDSupportedType; import com.ge.research.semtk.utility.Utility; diff --git a/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/ontologyTools/PredicateStats.java b/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/ontologyTools/PredicateStats.java index 8b4d18266..8de87fd4a 100644 --- a/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/ontologyTools/PredicateStats.java +++ b/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/ontologyTools/PredicateStats.java @@ -3,7 +3,6 @@ import java.util.HashSet; import java.util.Hashtable; -import org.apache.jena.atlas.json.JSON; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; diff --git a/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/plotting/PlotlyPlotSpec.java b/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/plotting/PlotlyPlotSpec.java index 4d4e89710..a65759b3f 100644 --- a/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/plotting/PlotlyPlotSpec.java +++ b/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/plotting/PlotlyPlotSpec.java @@ -22,7 +22,6 @@ import java.util.stream.Collectors; import org.apache.commons.lang.math.NumberUtils; -import org.apache.jena.atlas.json.JSON; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; diff --git a/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/services/client/UtilityClient.java b/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/services/client/UtilityClient.java index 81f18c844..a0fc1c680 100644 --- a/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/services/client/UtilityClient.java +++ b/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/services/client/UtilityClient.java @@ -21,7 +21,11 @@ import java.io.InputStreamReader; import java.net.ConnectException; +import org.apache.jena.shacl.validation.Severity; + import com.ge.research.semtk.connutil.EndpointNotFoundException; +import com.ge.research.semtk.resultSet.SimpleResultSet; +import com.ge.research.semtk.sparqlX.SparqlConnection; /** * Client for UtilityService @@ -39,6 +43,7 @@ private void cleanUp() { conf.setServiceEndpoint(null); this.parametersJSON.clear(); this.fileParameter = null; + this.fileParameterName = "file"; } /** @@ -76,4 +81,37 @@ public BufferedReader execLoadIngestionPackage(File ingestionPackageFile, String } } + + /** + * Executes a call to run SHACL + * @param shaclTtlFile the SHACL file (in ttl format) + * @param conn the connection + * @param severity return problems with this severity level or higher (Info, Warning, Violation) + * @return jobId + * @throws Exception + */ + @SuppressWarnings("unchecked") + public String execGetShaclResults(File shaclTtlFile, SparqlConnection conn, Severity severity) throws Exception { + + if(!shaclTtlFile.exists()) { + throw new Exception("File does not exist: " + shaclTtlFile.getAbsolutePath()); + } + + this.parametersJSON.clear(); + this.fileParameter = shaclTtlFile; + this.fileParameterName = "shaclTtlFile"; + this.parametersJSON.put("conn", conn.toJson().toJSONString()); + this.parametersJSON.put("severity", severity.level().getLocalName()); // e.g. Info, Warning, Violation + conf.setServiceEndpoint("utility/getShaclResults"); + this.conf.setMethod(RestClientConfig.Methods.POST); + + try { + SimpleResultSet res = this.executeWithSimpleResultReturn(); + res.throwExceptionIfUnsuccessful(); + return res.getJobId(); + } finally { + this.cleanUp(); + } + } + } diff --git a/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/validate/ShaclRunner.java b/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/validate/ShaclRunner.java new file mode 100644 index 000000000..9db9879f1 --- /dev/null +++ b/sparqlGraphLibrary/src/main/java/com/ge/research/semtk/validate/ShaclRunner.java @@ -0,0 +1,437 @@ +/** + ** Copyright 2023 General Electric Company + ** + ** + ** Licensed under the Apache License, Version 2.0 (the "License"); + ** you may not use this file except in compliance with the License. + ** You may obtain a copy of the License at + ** + ** http://www.apache.org/licenses/LICENSE-2.0 + ** + ** Unless required by applicable law or agreed to in writing, software + ** distributed under the License is distributed on an "AS IS" BASIS, + ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ** See the License for the specific language governing permissions and + ** limitations under the License. + */ +package com.ge.research.semtk.validate; + +import java.util.HashMap; +import java.util.Collection; +import java.util.Iterator; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.io.InputStream; + +import org.apache.jena.graph.Graph; +import org.apache.jena.graph.Node; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.apache.jena.shacl.ShaclValidator; +import org.apache.jena.shacl.Shapes; +import org.apache.jena.shacl.ValidationReport; +import org.apache.jena.shacl.engine.Target; +import org.apache.jena.shacl.parser.Shape; +import org.apache.jena.shacl.validation.ReportEntry; +import org.apache.jena.shacl.validation.Severity; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; + +import com.ge.research.semtk.edc.JobTracker; +import com.ge.research.semtk.ontologyTools.OntologyName; +import com.ge.research.semtk.sparqlX.SparqlConnection; +import com.ge.research.semtk.utility.LocalLogger; +import com.ge.research.semtk.utility.Utility; + +/** + * Validate RDF data using SHACL + */ +public class ShaclRunner { + + private Graph dataGraph; + private Shapes shapes; + private ValidationReport report; + private String shapesAndReportTurtle; // turtle string with raw shapes and raw report + private HashMap>> shapeTargetFocusHash = new HashMap>>(); // hash to avoid redundant processing + + private JobTracker tracker = null; + private String jobId = null; + private int startPercent = 0; + private int endPercent = 100; + + public static final String JSON_KEY_ENTRIES = "reportEntries"; + + private static final String JSON_KEY_SOURCESHAPE = "sourceShape"; // the URI of the source shape (e.g. http://DeliveryBasketExample#Shape_Fruit) + private static final String JSON_KEY_TARGETTYPE = "targetType"; // the target type. One of: targetNode, targetClass, targetSubjectsOf, targetObjectsOf + private static final String JSON_KEY_TARGETOBJECT = "targetObject"; // the target object. Can be URI or literal (e.g. http://DeliveryBasketExample#FruitBasket, http://DeliveryBasketExample#holds, "pear55") + private static final String JSON_KEY_PATH = "path"; // the relevant path (e.g. , ^, /, more) + private static final String JSON_KEY_CONSTRAINT = "constraint"; // the constraint (e.g. maxCount[3]) + public static final String JSON_KEY_SEVERITY = "severity"; // Violation, Info, or Warning (e.g. from http://www.w3.org/ns/shacl#Violation) + private static final String JSON_KEY_FOCUSNODE = "focusNode"; // the offending item, could be URI or literal (e.g. http://DeliveryBasketExample#basket100) + private static final String JSON_KEY_VALUE = "value"; // sometimes populated (e.g. for failed DataType constraint) + private static final String JSON_KEY_MESSAGE = "message"; // a message (auto-generated unless defined in SHACL shape) e.g. "maxCount[1]: Invalid cardinality: expected max 1: Got count = 2". No other way to get actual, so best practice is to let it auto-generate + private static final String JSON_KEY_MESSAGETRANSFORMED = "messageTransformed"; // a message transformed to be more readable than the original + + /** + * Constructor + * @param shaclTtlInputStream SHACL shapes in a TTL input stream + * @param conn connection containing model/data + */ + public ShaclRunner(InputStream shaclTtlInputStream, SparqlConnection conn) throws Exception{ + this(shaclTtlInputStream, conn, null, null, 0, 100); + } + + /** + * Constructor + * @param shaclTtlInputStream SHACL shapes in a TTL input stream + * @param conn connection containing model/data + * @param tracker the job tracker + * @param jobId the job id for setting job status + * @param percentStart the start percent for setting job status + * @param percentEnd the end percent for setting job status + */ + public ShaclRunner(InputStream shaclTtlInputStream, SparqlConnection conn, JobTracker tracker, String jobId, int percentStart, int percentEnd) throws Exception{ + + this.tracker = tracker; + this.jobId = jobId; + startPercent = percentStart; + endPercent = percentEnd; + + if(this.tracker != null) { this.tracker.setJobPercentComplete(this.jobId, startPercent, "Parsing SHACL shapes"); } + shapes = parseShapes(shaclTtlInputStream); + if(this.tracker != null) { this.tracker.setJobPercentComplete(this.jobId, (int)(startPercent + (endPercent - startPercent)*0.3), "Creating Jena graph"); } + dataGraph = conn.getJenaGraph(); + if(this.tracker != null) { this.tracker.setJobPercentComplete(this.jobId, (int)(startPercent + (endPercent - startPercent)*0.6), "Running SHACL"); } + run(); + if(this.tracker != null) { this.tracker.setJobPercentComplete(this.jobId, endPercent); } + } + + /** + * Constructor + * @param shaclTtlInputStream SHACL shapes in a TTL input stream + * @param dataTtlInputStream data/model in a TTL input stream + */ + public ShaclRunner(InputStream shaclTtlInputStream, InputStream dataTtlInputStream) throws Exception{ + shapes = parseShapes(shaclTtlInputStream); + dataGraph = Utility.getJenaGraphFromTurtle(dataTtlInputStream); + run(); + } + + // run SHACL shapes on a Jena Graph object, get report object and raw ttl + private void run() throws Exception { + + try { + report = ShaclValidator.get().validate(shapes, dataGraph); + }catch(Exception e) { + throw new Exception("Error creating SHACL validation report: " + e.getMessage(), e); + } + + // create TTL file with shapes and report + Model shapesAndReport = ModelFactory.createModelForGraph(shapes.getGraph()); + shapesAndReport.add(ModelFactory.createModelForGraph(report.getGraph())); + shapesAndReportTurtle = Utility.getTurtleFromJenaGraph(shapesAndReport.getGraph()); + } + + /** + * Return true if the data validates successfully + */ + public boolean conforms() { + return report.conforms(); + } + + /** + * Get the raw shapes and raw report information in Turtle format. + * This content is produced by the Jena shapes parser and validator, with no post-processing. + */ + public String getResultsRawTurtle() { + return shapesAndReportTurtle; + } + + /** + * Get the SHACL results. + * This content includes post-processing to gather key information for each result. + */ + public JSONObject getResults() throws Exception { + return getResults(Severity.Info); + } + + /** + * Get the SHACL results (for the given severity level and above) + * This content includes post-processing to gather key information for each result. + */ + @SuppressWarnings("unchecked") + public JSONObject getResults(Severity severityLevel) throws Exception { + + JSONArray reportEntriesJsonArr = new JSONArray(); + for(ReportEntry entry : report.getEntries()) { + + // get the focus node, source shape and its target + Node focusNode = entry.focusNode(); + Shape sourceShape = getShape(entry.source().toString()); + Target sourceShapeTarget = getMatchingTarget(sourceShape, focusNode); // a source shape may have multiple targets - determine which one produced our focus node + + // assemble the JSON content + String sourceShapeUri = sourceShape.getShapeNode().getURI(); // see notes in JSON_KEY_* above + String sourceShapeTargetType = sourceShapeTarget.getTargetType().toString(); // see notes in JSON_KEY_* above + String sourceShapeTargetObject = sourceShapeTarget.getObject().toString(); // see notes in JSON_KEY_* above + String path = (entry.resultPath() != null) ? entry.resultPath().toString() : null; // see notes in JSON_KEY_* above. Path will exist iff the constraint is on a PropertyShape + String constraint = getConstraintString(entry); // see notes in JSON_KEY_* above + String severity = getLocalName(entry.severity().level().toString()); // see notes in JSON_KEY_* above + String focusNodeStr = focusNode.isURI() ? focusNode.getURI() : focusNode.toString(); // see notes in JSON_KEY_* above + String value = entry.value() != null ? entry.value().toString() : ""; // see notes in JSON_KEY_* above + String message = entry.message(); // see notes in JSON_KEY_* above + String messageTransformed = transformMessage(message); // see notes in JSON_KEY_* above + + if(compare(entry.severity(), severityLevel) >= 0) { + JSONObject entryJson = new JSONObject(); + entryJson.put(JSON_KEY_SOURCESHAPE, sourceShapeUri); + entryJson.put(JSON_KEY_TARGETTYPE, sourceShapeTargetType); + entryJson.put(JSON_KEY_TARGETOBJECT, sourceShapeTargetObject); + entryJson.put(JSON_KEY_PATH, (path != null) ? path : ""); + entryJson.put(JSON_KEY_CONSTRAINT, constraint); + entryJson.put(JSON_KEY_SEVERITY, severity); + entryJson.put(JSON_KEY_FOCUSNODE, focusNodeStr); + entryJson.put(JSON_KEY_VALUE, value); + entryJson.put(JSON_KEY_MESSAGE, message); + entryJson.put(JSON_KEY_MESSAGETRANSFORMED, messageTransformed); + reportEntriesJsonArr.add(entryJson); + } + } + + JSONObject resultsJson = new JSONObject(); + resultsJson.put(JSON_KEY_ENTRIES, reportEntriesJsonArr); + return resultsJson; + } + + // a source shape may have multiple targets - determine which one produced our focus node + private Target getMatchingTarget(Shape sourceShape, Node focusNode) throws Exception { + + // if hash doesn't already have this sourceShape, then add it (hashing yields major speedup for big data graphs) + if(shapeTargetFocusHash.get(sourceShape) == null) { + HashMap> targetFocusHash = new HashMap>(); // maps target to its focus nodes + Iterator targetsIter = sourceShape.getTargets().iterator(); + while(targetsIter.hasNext()) { + Target target = targetsIter.next(); + targetFocusHash.put(target, target.getFocusNodes(dataGraph)); + } + shapeTargetFocusHash.put(sourceShape, targetFocusHash); + } + + // get target from the hash + HashMap> map = shapeTargetFocusHash.get(sourceShape); + for(Target target : map.keySet()) { + if(map.get(target).contains(focusNode)) { + return target; + } + } + throw new Exception("Unexpected error: didn't find target"); + } + + // get a constraint string (e.g. "DatatypeConstraint[xsd:int]") from an entry + private String getConstraintString(ReportEntry entry) { + String s = entry.constraint().toString(); // normally something helpful like "DatatypeConstraint[xsd:int]", but sometimes like this "org.apache.jena.shacl.engine.constraint.ReportConstraint@16c8b7bd" + int index = s.indexOf("org.apache.jena.shacl.engine.constraint.ReportConstraint@"); + if(index == -1) { + return s; + }else { + return "org.apache.jena.shacl.engine.constraint.ReportConstraint@XXXXXXXX"; // substituting XXXXXXXX to enable unit testing - original content likely not needed + } + } + + // parse shapes + private Shapes parseShapes(InputStream shaclTtlInputStream) throws Exception { + try { + return Shapes.parse(Utility.getJenaGraphFromTurtle(shaclTtlInputStream)); + }catch(Exception e) { + throw new Exception("Error parsing SHACL shapes: " + e.getMessage(), e); + } + } + + /** + * Find a shape with the given string from the results + * @param shapeStr the shape string, e.g. 904d05c172ca9ceece3fe110063b826f + * @return the shape + * @throws Exception + */ + private Shape getShape(String shapeStr) throws Exception { + Iterator shapesIter = shapes.iterator(); + while(shapesIter.hasNext()) { + Shape s = shapesIter.next(); + // check if the top-level shape node has the given string + if(s.getShapeNode().toString().equals(shapeStr)) { + return s; + } + // check if the shape's property shapes have the given string + for(Shape ps : s.getPropertyShapes()) { + if(ps.getShapeNode().toString().equals(shapeStr)) { + return s; + } + } + } + throw new Exception("Could not find shape " + shapeStr); + } + + /** + * Make a SHACL message more human friendly. + * This method transforms auto-generated messages corresponding to various constraint types (https://www.w3.org/TR/shacl/#core-components) + * This is a best-effort approach because of the many different message formats possible (including custom messages). If no match found, then log it and return an empty string. + * + * @param message the message to transform + * @return a transformed message, or empty if could not transform it + * + * TODO add more constraint types + */ + private String transformMessage(String message) { + + try { + + Pattern pattern = null; // note: need the () subpattern groupings for exec() to work + Matcher matcher = null; + + // ClassConstraint[]: Expected class : for + pattern = Pattern.compile("ClassConstraint\\[(.+)\\]: Expected class :(.+) for (.+)"); + matcher = pattern.matcher(message); + if (matcher.matches()) { + return "Expect " + matcher.group(3) + " to be an instance of " + matcher.group(2); + } + + // DatatypeConstraint[xsd:bool]: Expected xsd:bool : Actual xsd:string : Node "id0" + pattern = Pattern.compile("DatatypeConstraint(.+): Expected (.+) : Actual (.+) : Node (.+)"); + matcher = pattern.matcher(message); + if (matcher.matches()) { + return "Expect datatype " + matcher.group(2) + " (got " + matcher.group(4) + ", type " + matcher.group(3) + ")"; + } + + // maxCount[3]: Invalid cardinality: expected max 3: Got count = 4 + pattern = Pattern.compile("maxCount\\[(\\d+)\\]: Invalid cardinality: expected max (\\d+): Got count = (\\d+)"); + matcher = pattern.matcher(message); + if (matcher.matches()) { + return "Expect count <= " + matcher.group(2) + " (got " + matcher.group(3) + ")"; + } + // minCount[1]: Invalid cardinality: expected min 1: Got count = 0 + pattern = Pattern.compile("minCount\\[(\\d+)\\]: Invalid cardinality: expected min (\\d+): Got count = (\\d+)"); + matcher = pattern.matcher(message); + if (matcher.matches()) { + return "Expect count >= " + matcher.group(2) + " (got " + matcher.group(3) + ")"; + } + + // Data value "0.5"^^xsd:double is not greater than or equal to 1 + // covers minInclusive, minExclusive, maxInclusive, maxExclusive + pattern = Pattern.compile("Data value \\\"(.+)\\\"\\^\\^xsd:(.+) is not (.+)"); + matcher = pattern.matcher(message); + if (matcher.matches()) { + return "Expect a " + matcher.group(2) + " " + matcher.group(3) + " (got " + matcher.group(1) + ")"; + } + + // MinLengthConstraint[5]: String too short: id0 + pattern = Pattern.compile("MinLengthConstraint\\[(\\d+)\\]: String too short: (.+)"); + matcher = pattern.matcher(message); + if (matcher.matches()) { + return "Expect string length >= " + matcher.group(1) + " (\"" + matcher.group(2) + "\")"; + } + // MaxLengthConstraint[5]: String too long: id0 + pattern = Pattern.compile("MaxLengthConstraint\\[(\\d+)\\]: String too long: (.+)"); + matcher = pattern.matcher(message); + if (matcher.matches()) { + return "Expect string length <= " + matcher.group(1) + " (\"" + matcher.group(2) + "\")"; + } + + // Pattern[(.+)peach(.+)]: Does not match: 'http://DeliveryBasketExample#basket100' + pattern = Pattern.compile("Pattern\\[(.+)\\]: Does not match: '(.+)'"); + matcher = pattern.matcher(message); + if (matcher.matches()) { + return "Does not match pattern " + matcher.group(1) + ": \"" + matcher.group(2) + "\""; + } + + // Equals[]: not equal: value node "Rebecca Recipient" is not in ["Carey careof"] + pattern = Pattern.compile("Equals\\[(.+)\\]: not equal(.+)"); + matcher = pattern.matcher(message); + if (matcher.matches()) { + return "Expect to equal " + getLocalName(matcher.group(1)) + " (but does not)"; + } + + // LessThan[]: value node "2023-01-01"^^xsd:date is not less than "1999-01-01"^^xsd:date + // LessThanOrEquals[]: value node "2023-01-01"^^xsd:date is not less than or equal to "1999-01-01"^^xsd:date + pattern = Pattern.compile("LessThan(.*)\\[(.+)\\]: value node \"(.+)\"\\^\\^xsd:(.+) is not (.+) \"(.+)\"\\^\\^xsd:(.+)"); + matcher = pattern.matcher(message); + if (matcher.matches()) { + return "Expect to be " + matcher.group(5) + " " + getLocalName(matcher.group(2)) + " (but is not)"; + } + + // Not[NodeShape[30e4dc59675ddbcdb75c01a2287258dc]] at focusNode http://DeliveryBasketExample#addressTwoZips + // Not[PropertyShape[e6e0b7919760c0c84e28e6a95804080a -> ]] at focusNode + pattern = Pattern.compile("Not\\[(.+)"); + matcher = pattern.matcher(message); + if (matcher.matches()) { + return "Expect to not conform to a shape, but it does"; + } + + // Xone has 2 conforming shapes at focusNode + pattern = Pattern.compile("Xone has (.+) conforming shapes at focusNode (.+)"); // confirmed works on 0 and 2 + matcher = pattern.matcher(message); + if (matcher.matches()) { + return "Expect to conform to exactly one shape (but conforms to " + matcher.group(1) + ")"; + } + + // Node[] at focusNode + pattern = Pattern.compile("Node\\[(.+)\\] at focusNode (.+)"); + matcher = pattern.matcher(message); + if (matcher.matches()) { + return "Expect to conform to shape " + getLocalName(matcher.group(1)); + } + + // QualifiedValueShape[2,_,false]: Min = 2 but got 1 validations + pattern = Pattern.compile("QualifiedValueShape\\[(.+)\\]: Min = (.+) but got (.+) validations"); + matcher = pattern.matcher(message); + if (matcher.matches()) { + return "Expect at least " + matcher.group(2) + " items that conform to a shape, but got " + matcher.group(3); + } + // QualifiedValueShape[_,2,false]: Max = 2 but got 3 validations + pattern = Pattern.compile("QualifiedValueShape\\[(.+)\\]: Max = (.+) but got (.+) validations"); + matcher = pattern.matcher(message); + if (matcher.matches()) { + return "Expect at most " + matcher.group(2) + " items that conform to a shape, but got " + matcher.group(3); + } + + // Closed[http://DeliveryBasketExample#includes] Property = rdf:type : Object = + // Closed[http://DeliveryBasketExample#includes] Property = : Object = "10"^^xsd:double + pattern = Pattern.compile("Closed(.+) Property = (.+) : Object = (.+)"); + matcher = pattern.matcher(message); + if (matcher.matches()) { + return "Expect to only have properties " + matcher.group(1) + " (but has " + matcher.group(2) + ")"; + } + + }catch(Exception e) { + LocalLogger.logToStdErr("Error transforming SHACL message '" + message + "': " + e); + LocalLogger.printStackTrace(e); + } + + LocalLogger.logToStdOut("No SHACL message transform applies: " + message); + return ""; + } + + // Get a URI's local name + // e.g. => FruitBasket + // Removes angled brackets if present (omitting angled brackets from regex because may not be present, e.g. for blank nodes) + private String getLocalName(String uri) { + if(uri.startsWith("<") && uri.endsWith(">")) { + uri = uri.substring(0, uri.length() - 1); + } + return (new OntologyName(uri)).getLocalName(); // if # exists, returns substring following it + } + + /** + * Compare SHACL severity levels + */ + public static int compare(Severity severity1, Severity severity2) throws Exception { + if(severity1.equals(severity2)) { return 0; } + if(severity1.equals(Severity.Violation)) { return 1; } + if(severity1.equals(Severity.Info)) { return -1; } + if(severity1.equals(Severity.Warning)) { + if(severity2.equals(Severity.Violation)) { return -1; } + else { return 1; } + } + throw new Exception("Unexpected error comparing severities"); + } + +} diff --git a/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/belmont/test/ValidationAssistantTest_IT.java b/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/belmont/test/ValidationAssistantTest_IT.java index 497bdf640..2a55f4960 100644 --- a/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/belmont/test/ValidationAssistantTest_IT.java +++ b/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/belmont/test/ValidationAssistantTest_IT.java @@ -19,7 +19,6 @@ import java.util.ArrayList; -import org.apache.jena.atlas.json.JSON; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.junit.BeforeClass; diff --git a/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/sparqlX/test/SparqlConnectionTest_IT.java b/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/sparqlX/test/SparqlConnectionTest_IT.java new file mode 100644 index 000000000..23b55435c --- /dev/null +++ b/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/sparqlX/test/SparqlConnectionTest_IT.java @@ -0,0 +1,36 @@ +/** + ** Copyright 2023 General Electric Company + ** + ** + ** Licensed under the Apache License, Version 2.0 (the "License"); + ** you may not use this file except in compliance with the License. + ** You may obtain a copy of the License at + ** + ** http://www.apache.org/licenses/LICENSE-2.0 + ** + ** Unless required by applicable law or agreed to in writing, software + ** distributed under the License is distributed on an "AS IS" BASIS, + ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ** See the License for the specific language governing permissions and + ** limitations under the License. + */ +package com.ge.research.semtk.sparqlX.test; + +import static org.junit.Assert.assertEquals; + +import org.apache.jena.graph.Graph; +import org.junit.Test; + +import com.ge.research.semtk.test.TestGraph; + +public class SparqlConnectionTest_IT { + + @Test + public void testGetJenaGraph() throws Exception { + TestGraph.clearGraph(); + TestGraph.uploadTurtleResource(this, "musicTestDataset_2017.q2.ttl"); + Graph g = TestGraph.getSparqlConn().getJenaGraph(); + assertEquals(g.size(), 215); + } + +} diff --git a/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/utility/test/UtilityClientTest_IT.java b/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/utility/test/UtilityClientTest_IT.java index 697e70342..f75166a35 100644 --- a/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/utility/test/UtilityClientTest_IT.java +++ b/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/utility/test/UtilityClientTest_IT.java @@ -20,7 +20,11 @@ import static org.junit.Assume.*; import java.io.BufferedReader; +import java.io.File; +import org.apache.jena.shacl.validation.Severity; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; import org.junit.BeforeClass; import org.junit.Test; @@ -31,6 +35,8 @@ import com.ge.research.semtk.test.IntegrationTestUtility; import com.ge.research.semtk.test.TestGraph; import com.ge.research.semtk.utility.Utility; +import com.ge.research.semtk.validate.ShaclRunner; +import com.ge.research.semtk.validate.test.ShaclRunnerTest_IT; public class UtilityClientTest_IT { @@ -114,6 +120,41 @@ public void testLoadIngestionPackage_errorConditions() throws Exception { assertTrue(response.contains("ERROR: Cannot find a top-level manifest")); } + + @Test + public void testGetShaclResults() throws Exception { + TestGraph.clearGraph(); + TestGraph.uploadOwlResource(this, "DeliveryBasketExample.owl"); + + File ttlFile = Utility.getResourceAsTempFile(this, "DeliveryBasketExample-shacl.ttl"); + String jobId; + JSONObject resultsJson; + JSONObject expectedJson = Utility.getResourceAsJson(this, "DeliveryBasketExample-shacl-results.json"); + + // info level + jobId = client.execGetShaclResults(ttlFile, TestGraph.getSparqlConn(), Severity.Info); + IntegrationTestUtility.getStatusClient(jobId).waitForCompletion(jobId); + resultsJson = IntegrationTestUtility.getResultsClient().execGetBlobResult(jobId); + assertEquals(((JSONArray)resultsJson.get(ShaclRunner.JSON_KEY_ENTRIES)).size(), ((JSONArray)expectedJson.get(ShaclRunner.JSON_KEY_ENTRIES)).size()); + Utility.equals(resultsJson, expectedJson); + + // violation level + jobId = client.execGetShaclResults(ttlFile, TestGraph.getSparqlConn(), Severity.Violation); + IntegrationTestUtility.getStatusClient(jobId).waitForCompletion(jobId); + resultsJson = IntegrationTestUtility.getResultsClient().execGetBlobResult(jobId); + expectedJson = ShaclRunnerTest_IT.removeEntriesBelowSeverityLevel(expectedJson, Severity.Violation); + assertEquals(((JSONArray)resultsJson.get(ShaclRunner.JSON_KEY_ENTRIES)).size(), ((JSONArray)expectedJson.get(ShaclRunner.JSON_KEY_ENTRIES)).size()); + Utility.equals(resultsJson, expectedJson); + + try { + client.execGetShaclResults(new File("src/nonexistent.ttl"), TestGraph.getSparqlConn(), Severity.Info); + fail(); // should not get here + }catch(Exception e) { + assert(e.getMessage().contains("File does not exist")); + } + TestGraph.clearGraph(); + } + // clear graphs and nodegroup store private void reset() throws Exception { IntegrationTestUtility.clearGraph(modelFallbackSei); diff --git a/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/utility/test/UtilityTest.java b/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/utility/test/UtilityTest.java index 655af0480..6e5a68a0b 100644 --- a/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/utility/test/UtilityTest.java +++ b/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/utility/test/UtilityTest.java @@ -30,8 +30,10 @@ import java.util.HashSet; import java.util.Random; +import org.json.simple.JSONObject; import org.apache.commons.io.FileUtils; import org.junit.Test; +import org.apache.jena.graph.Graph; import com.ge.research.semtk.utility.Utility; @@ -275,4 +277,34 @@ public void testGetFromRdf() throws Exception { } + @Test + public void testJenaFunctions() throws Exception{ + + // get Jena graph from TTL file + File file = Utility.getResourceAsTempFile(this, "musicTestDataset_2017.q2.ttl"); + Graph g = Utility.getJenaGraphFromTurtle(new FileInputStream(file)); + assertEquals(g.size(), 215); + + // get TTL file from Jena graph + String ttl = Utility.getTurtleFromJenaGraph(g); + assertTrue(ttl.contains("ns1:Kansas rdf:type ns2:Band;")); + assertEquals(ttl.length(), 10034); + } + + @Test + // TODO add a list + // TODO prove ignores order + public void testIsJsonEqual() throws Exception { + JSONObject o1 = new JSONObject(); + o1.put("fruit", "apple"); + o1.put("vegetable", "asparagus"); + o1.put("protein", "beef"); + + JSONObject o2 = new JSONObject(); + o2.put("vegetable", "asparagus"); + o2.put("protein", "beef"); + o2.put("fruit", "apple"); + + assertTrue(Utility.equals(o1, o2)); + } } diff --git a/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/validate/test/ShaclRunnerTest_IT.java b/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/validate/test/ShaclRunnerTest_IT.java new file mode 100644 index 000000000..06bc61a08 --- /dev/null +++ b/sparqlGraphLibrary/src/test/java/com/ge/research/semtk/validate/test/ShaclRunnerTest_IT.java @@ -0,0 +1,176 @@ +/** + ** Copyright 2023 General Electric Company + ** + ** + ** Licensed under the Apache License, Version 2.0 (the "License"); + ** you may not use this file except in compliance with the License. + ** You may obtain a copy of the License at + ** + ** http://www.apache.org/licenses/LICENSE-2.0 + ** + ** Unless required by applicable law or agreed to in writing, software + ** distributed under the License is distributed on an "AS IS" BASIS, + ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ** See the License for the specific language governing permissions and + ** limitations under the License. + */ +package com.ge.research.semtk.validate.test; + +import java.io.File; +import java.io.FileInputStream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.apache.jena.shacl.validation.Severity; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.junit.Test; + +import com.ge.research.semtk.resultSet.Table; +import com.ge.research.semtk.sparqlX.SparqlConnection; +import com.ge.research.semtk.test.TestGraph; +import com.ge.research.semtk.utility.Utility; +import com.ge.research.semtk.validate.ShaclRunner; + +public class ShaclRunnerTest_IT { + + // test the processed results JSON + @Test + public void testResults() throws Exception{ + + ShaclRunner runner = getShaclRunnerForDeliveryBasketExample(); + assertFalse("data conforms when it should not", runner.conforms()); + + // compare results to expected results + final String EXPECTED_RESULTS_FILE = "DeliveryBasketExample-shacl-results.json"; + JSONObject resultsJson; + JSONObject expectedJson; + + // test at Info level + resultsJson = runner.getResults(Severity.Info); + expectedJson = Utility.getResourceAsJson(this, EXPECTED_RESULTS_FILE); + assertEquals(((JSONArray)resultsJson.get(ShaclRunner.JSON_KEY_ENTRIES)).size(), ((JSONArray)expectedJson.get(ShaclRunner.JSON_KEY_ENTRIES)).size()); + assertTrue(Utility.equals(resultsJson, expectedJson)); + + // test at Warning level + resultsJson = runner.getResults(Severity.Warning); + expectedJson = removeEntriesBelowSeverityLevel(Utility.getResourceAsJson(this, EXPECTED_RESULTS_FILE), Severity.Warning); + assertEquals(((JSONArray)resultsJson.get(ShaclRunner.JSON_KEY_ENTRIES)).size(), ((JSONArray)expectedJson.get(ShaclRunner.JSON_KEY_ENTRIES)).size()); + assertTrue(Utility.equals(resultsJson, expectedJson)); + + // test at Violation level + resultsJson = runner.getResults(Severity.Violation); + expectedJson = removeEntriesBelowSeverityLevel(Utility.getResourceAsJson(this, EXPECTED_RESULTS_FILE), Severity.Violation); + assertEquals(((JSONArray)resultsJson.get(ShaclRunner.JSON_KEY_ENTRIES)).size(), ((JSONArray)expectedJson.get(ShaclRunner.JSON_KEY_ENTRIES)).size()); + assertTrue(Utility.equals(resultsJson, expectedJson)); + } + + // test the raw turtle output + @Test + public void testResultsRawTurtle() throws Exception { + + ShaclRunner runner = getShaclRunnerForDeliveryBasketExample(); + String resultsTurtle = runner.getResultsRawTurtle(); // turtle containing raw shape and validation result info + + // load the turtle and query via SPARQL + TestGraph.clearGraph(); + TestGraph.uploadTurtleString(resultsTurtle); + + // note: query below may not be complete/robust enough for purposes beyond the row count used below - but it's a good start. + // note: would have preferred to compare against Turtle from a file, but not possible because blank node URIs change + String query = "select * from <" + TestGraph.getDataset() + "> where {" + + "?result a ." + + + // these parameters must have exactly 1 value + "?result ?focusNode ." + // e.g. http://DeliveryBasketExample#basket100 + "?result ?resultSeverity ." + // e.g. http://www.w3.org/ns/shacl#Violation + "?result ?sourceConstraintComponent ." + // e.g. http://www.w3.org/ns/shacl#MaxCountConstraintComponent + + // these parameters are optional + "OPTIONAL { ?result ?resultPath } ." + + "OPTIONAL { ?result ?resultMessage } ." + // not more than 1 per language tag. May be auto-generated if not specified in shape. + + // link the NodeShape + "OPTIONAL { " + + "?result ?prop ." + + "?nodeShape a ." + + "?nodeShape ?prop ." + + "OPTIONAL { ?nodeShape ?targetClass } ." + + "OPTIONAL { ?nodeShape ?targetSubjectsOf } ." + + "OPTIONAL { ?nodeShape ?targetObjectsOf } ." + + " } . " + + + "} " + + "ORDER BY ?nodeShape"; + + Table res = TestGraph.execQueryToTable(query); + int expectedCount = ((JSONArray)Utility.getResourceAsJson(this, "DeliveryBasketExample-shacl-results.json").get(ShaclRunner.JSON_KEY_ENTRIES)).size(); + assertEquals(res.getNumRows(), expectedCount); + } + + // test loading data from TTL file + @Test + public void testLoadDataFromTTL() throws Exception { + File shaclFile = Utility.getResourceAsTempFile(this, "musicTestDataset-shacl.ttl"); + ShaclRunner runner = new ShaclRunner(new FileInputStream(shaclFile), new FileInputStream(new File("src/test/resources/musicTestDataset_2017.q2.ttl"))); + JSONObject resultsJson = runner.getResults(); + assertEquals(((JSONArray)resultsJson.get(ShaclRunner.JSON_KEY_ENTRIES)).size(), 3); + assertTrue(resultsJson.toJSONString().contains("Expect a int less than 300 (got 318)")); + assertTrue(resultsJson.toJSONString().contains("Expect a int less than 300 (got 323)")); + assertTrue(resultsJson.toJSONString().contains("Expect a int less than 300 (got 334)")); + } + + @Test + public void testCompareSeverity() throws Exception { + assertEquals(ShaclRunner.compare(Severity.Violation, Severity.Info), 1); + assertEquals(ShaclRunner.compare(Severity.Warning, Severity.Info), 1); + assertEquals(ShaclRunner.compare(Severity.Violation, Severity.Warning), 1); + assertEquals(ShaclRunner.compare(Severity.Violation, Severity.Violation), 0); + assertEquals(ShaclRunner.compare(Severity.Warning, Severity.Warning), 0); + assertEquals(ShaclRunner.compare(Severity.Info, Severity.Info), 0); + assertEquals(ShaclRunner.compare(Severity.Info, Severity.Violation), -1); + assertEquals(ShaclRunner.compare(Severity.Warning, Severity.Violation), -1); + assertEquals(ShaclRunner.compare(Severity.Info, Severity.Warning), -1); + } + + + // helper function for test setup + private ShaclRunner getShaclRunnerForDeliveryBasketExample() throws Exception { + + // create a SPARQL connection, populate it + TestGraph.clearGraph(); + TestGraph.uploadOwlResource(this, "DeliveryBasketExample.owl"); + SparqlConnection sparqlConn = TestGraph.getSparqlConn(); + + // perform SHACL validation + File shaclFile = Utility.getResourceAsTempFile(this, "DeliveryBasketExample-shacl.ttl"); + return new ShaclRunner(new FileInputStream(shaclFile), sparqlConn); + } + + // helper function to filter expected results to a given severity level + @SuppressWarnings("unchecked") + public static JSONObject removeEntriesBelowSeverityLevel(JSONObject resultsJson, Severity severityLevel) throws Exception { + + JSONArray entriesToKeep = new JSONArray(); + + // iterate through entries, keeping the ones at the appropriate level + JSONArray entries = (JSONArray) resultsJson.get(ShaclRunner.JSON_KEY_ENTRIES); + for(int i = 0; i < entries.size(); i++) { + JSONObject entry = (JSONObject) entries.get(i); + String severityStr = (String) entry.get(ShaclRunner.JSON_KEY_SEVERITY); + if(severityStr.equals("Violation")) { + entriesToKeep.add(entry); + }else if(severityStr.equals("Warning") && ShaclRunner.compare(Severity.Warning, severityLevel) >= 0) { + entriesToKeep.add(entry); + }else if(severityStr.equals("Info") && ShaclRunner.compare(Severity.Info, severityLevel) >= 0) { + entriesToKeep.add(entry); + } + } + + resultsJson.put(ShaclRunner.JSON_KEY_ENTRIES, entriesToKeep); + return resultsJson; + } + +} diff --git a/sparqlGraphLibrary/src/test/resources/DeliveryBasketExample-shacl-results.json b/sparqlGraphLibrary/src/test/resources/DeliveryBasketExample-shacl-results.json new file mode 100644 index 000000000..a6869acac --- /dev/null +++ b/sparqlGraphLibrary/src/test/resources/DeliveryBasketExample-shacl-results.json @@ -0,0 +1,352 @@ +{ + "reportEntries": [ + { + "severity": "Violation", + "path": "", + "targetObject": "http:\/\/DeliveryBasketExample#FruitBasket", + "sourceShape": "http:\/\/DeliveryBasketExample#FruitBasketConforms", + "targetType": "targetClass", + "constraint": "LessThanOrEquals[]", + "focusNode": "http:\/\/DeliveryBasketExample#basket3", + "message": "LessThanOrEquals[]: value node \"2023-01-01\"^^xsd:date is not less than or equal to \"1999-01-01\"^^xsd:date", + "value": "\"2023-01-01\"^^http:\/\/www.w3.org\/2001\/XMLSchema#date", + "messageTransformed": "Expect to be less than or equal to expirationDate (but is not)" + }, + { + "severity": "Violation", + "path": "", + "targetObject": "http:\/\/DeliveryBasketExample#FruitBasket", + "sourceShape": "http:\/\/DeliveryBasketExample#FruitBasketConforms", + "targetType": "targetClass", + "constraint": "LessThan[]", + "focusNode": "http:\/\/DeliveryBasketExample#basket3", + "message": "LessThan[]: value node \"2023-01-01\"^^xsd:date is not less than \"1999-01-01\"^^xsd:date", + "value": "\"2023-01-01\"^^http:\/\/www.w3.org\/2001\/XMLSchema#date", + "messageTransformed": "Expect to be less than expirationDate (but is not)" + }, + { + "severity": "Violation", + "path": "", + "targetObject": "http:\/\/DeliveryBasketExample#FruitBasket", + "sourceShape": "http:\/\/DeliveryBasketExample#FruitBasketConforms", + "targetType": "targetClass", + "constraint": "minCount[1]", + "focusNode": "http:\/\/DeliveryBasketExample#basket3", + "message": "minCount[1]: Invalid cardinality: expected min 1: Got count = 0", + "value": "", + "messageTransformed": "Expect count >= 1 (got 0)" + }, + { + "severity": "Violation", + "path": "", + "targetObject": "http:\/\/DeliveryBasketExample#FruitBasket", + "sourceShape": "http:\/\/DeliveryBasketExample#FruitBasketConforms", + "targetType": "targetClass", + "constraint": "maxCount[3]", + "focusNode": "http:\/\/DeliveryBasketExample#basket100", + "message": "maxCount[3]: Invalid cardinality: expected max 3: Got count = 4", + "value": "", + "messageTransformed": "Expect count <= 3 (got 4)" + }, + { + "severity": "Violation", + "path": "", + "targetObject": "http:\/\/DeliveryBasketExample#Delivery", + "sourceShape": "http:\/\/DeliveryBasketExample#DeliveryHasRecipientAndLongEnoughIdentifiers", + "targetType": "targetClass", + "constraint": "Node[]", + "focusNode": "http:\/\/DeliveryBasketExample#deliveryWithoutRecipient", + "message": "Node[] at focusNode ", + "value": "http:\/\/DeliveryBasketExample#addressWithoutRecipient", + "messageTransformed": "Expect to conform to shape HasAtLeastOneRecipient" + }, + { + "severity": "Info", + "path": "()+\/", + "targetObject": "http:\/\/DeliveryBasketExample#Delivery", + "sourceShape": "http:\/\/DeliveryBasketExample#DeliveryHasRecipientAndLongEnoughIdentifiers", + "targetType": "targetClass", + "constraint": "MinLengthConstraint[10]", + "focusNode": "http:\/\/DeliveryBasketExample#delivery1", + "message": "MinLengthConstraint[10]: String too short: basket1", + "value": "\"basket1\"", + "messageTransformed": "Expect string length >= 10 (\"basket1\")" + }, + { + "severity": "Info", + "path": "()+\/", + "targetObject": "http:\/\/DeliveryBasketExample#Delivery", + "sourceShape": "http:\/\/DeliveryBasketExample#DeliveryHasRecipientAndLongEnoughIdentifiers", + "targetType": "targetClass", + "constraint": "MinLengthConstraint[10]", + "focusNode": "http:\/\/DeliveryBasketExample#delivery1", + "message": "MinLengthConstraint[10]: String too short: fruit1", + "value": "\"fruit1\"", + "messageTransformed": "Expect string length >= 10 (\"fruit1\")" + }, + { + "severity": "Violation", + "path": "", + "targetObject": "http:\/\/DeliveryBasketExample#Address", + "sourceShape": "http:\/\/DeliveryBasketExample#AddressRecipientIsNotRex", + "targetType": "targetClass", + "constraint": "SPARQL[PREFIX dbex: SELECT ?this (dbex:recipientName AS ?path) ?value WHERE { ?this dbex:recipientName ?value FILTER contains(?value, \"Rex\") }]", + "focusNode": "http:\/\/DeliveryBasketExample#addressTwoZips", + "message": "Recipient name cannot contain 'Rex', but it does", + "value": "\"Rex Recipient\"", + "messageTransformed": "" + }, + { + "severity": "Violation", + "path": "^", + "targetObject": "http:\/\/DeliveryBasketExample#Fruit", + "sourceShape": "http:\/\/DeliveryBasketExample#FruitIsInABasket", + "targetType": "targetClass", + "constraint": "minCount[1]", + "focusNode": "http:\/\/DeliveryBasketExample#peach1", + "message": "minCount[1]: Invalid cardinality: expected min 1: Got count = 0", + "value": "", + "messageTransformed": "Expect count >= 1 (got 0)" + }, + { + "severity": "Violation", + "path": "^", + "targetObject": "http:\/\/DeliveryBasketExample#Fruit", + "sourceShape": "http:\/\/DeliveryBasketExample#FruitIsInABasket", + "targetType": "targetClass", + "constraint": "minCount[1]", + "focusNode": "http:\/\/DeliveryBasketExample#fruit6", + "message": "minCount[1]: Invalid cardinality: expected min 1: Got count = 0", + "value": "", + "messageTransformed": "Expect count >= 1 (got 0)" + }, + { + "severity": "Violation", + "path": "^", + "targetObject": "http:\/\/DeliveryBasketExample#Fruit", + "sourceShape": "http:\/\/DeliveryBasketExample#FruitIsInABasket", + "targetType": "targetClass", + "constraint": "maxCount[1]", + "focusNode": "http:\/\/DeliveryBasketExample#fruit5", + "message": "maxCount[1]: Invalid cardinality: expected max 1: Got count = 2", + "value": "", + "messageTransformed": "Expect count <= 1 (got 2)" + }, + { + "severity": "Violation", + "path": "", + "targetObject": "http:\/\/DeliveryBasketExample#basket100", + "sourceShape": "http:\/\/DeliveryBasketExample#Basket100HasNoExtraProperties", + "targetType": "targetNode", + "constraint": "Closed[http:\/\/DeliveryBasketExample#includes][http:\/\/www.w3.org\/1999\/02\/22-rdf-syntax-ns#type]", + "focusNode": "http:\/\/DeliveryBasketExample#basket100", + "message": "Closed[http:\/\/DeliveryBasketExample#includes][http:\/\/www.w3.org\/1999\/02\/22-rdf-syntax-ns#type] Property = : Object = \"10\"^^xsd:double", + "value": "\"10\"^^http:\/\/www.w3.org\/2001\/XMLSchema#double", + "messageTransformed": "Expect to only have properties [http:\/\/DeliveryBasketExample#includes][http:\/\/www.w3.org\/1999\/02\/22-rdf-syntax-ns#type] (but has )" + }, + { + "severity": "Warning", + "path": "", + "targetObject": "http:\/\/DeliveryBasketExample#basket4", + "sourceShape": "http:\/\/DeliveryBasketExample#FruitBasketIncludesAtLeastTwoPeaches", + "targetType": "targetNode", + "constraint": "org.apache.jena.shacl.engine.constraint.ReportConstraint@XXXXXXXX", + "focusNode": "http:\/\/DeliveryBasketExample#basket4", + "message": "Must include at least 2 peaches", + "value": "", + "messageTransformed": "" + }, + { + "severity": "Info", + "path": "", + "targetObject": "http:\/\/DeliveryBasketExample#basket100", + "sourceShape": "http:\/\/DeliveryBasketExample#URIContainsPeach", + "targetType": "targetNode", + "constraint": "Pattern[(.+)peach(.+)]", + "focusNode": "http:\/\/DeliveryBasketExample#basket100", + "message": "Pattern[(.+)peach(.+)]: Does not match: 'http:\/\/DeliveryBasketExample#basket100'", + "value": "http:\/\/DeliveryBasketExample#basket100", + "messageTransformed": "Does not match pattern (.+)peach(.+): \"http:\/\/DeliveryBasketExample#basket100\"" + }, + { + "severity": "Warning", + "path": "", + "targetObject": "http:\/\/DeliveryBasketExample#basket100", + "sourceShape": "http:\/\/DeliveryBasketExample#FruitBasketIncludesAtMostTwoPeaches", + "targetType": "targetNode", + "constraint": "org.apache.jena.shacl.engine.constraint.ReportConstraint@XXXXXXXX", + "focusNode": "http:\/\/DeliveryBasketExample#basket100", + "message": "Must include at most 2 peaches", + "value": "", + "messageTransformed": "" + }, + { + "severity": "Warning", + "path": "|", + "targetObject": "http:\/\/DeliveryBasketExample#Address", + "sourceShape": "http:\/\/DeliveryBasketExample#AddressPersonsHaveFirstLastNameOnly", + "targetType": "targetClass", + "constraint": "Pattern[^[A-Z][a-z]* [A-Z][a-z]*$]", + "focusNode": "http:\/\/DeliveryBasketExample#address1", + "message": "Pattern[^[A-Z][a-z]* [A-Z][a-z]*$]: Does not match: 'Carey careof'", + "value": "\"Carey careof\"", + "messageTransformed": "Does not match pattern ^[A-Z][a-z]* [A-Z][a-z]*$: \"Carey careof\"" + }, + { + "severity": "Violation", + "path": "", + "targetObject": "http:\/\/DeliveryBasketExample#Address", + "sourceShape": "http:\/\/DeliveryBasketExample#AddressConforms", + "targetType": "targetClass", + "constraint": "Xone", + "focusNode": "http:\/\/DeliveryBasketExample#addressTwoZips", + "message": "Xone has 2 conforming shapes at focusNode ", + "value": "http:\/\/DeliveryBasketExample#addressTwoZips", + "messageTransformed": "Expect to conform to exactly one shape (but conforms to 2)" + }, + { + "severity": "Warning", + "path": "", + "targetObject": "http:\/\/DeliveryBasketExample#Address", + "sourceShape": "http:\/\/DeliveryBasketExample#AddressConforms", + "targetType": "targetClass", + "constraint": "Equals[]", + "focusNode": "http:\/\/DeliveryBasketExample#address1", + "message": "Equals[]: not equal: value node \"Rebecca Recipient\" is not in [\"Carey careof\"]", + "value": "\"Rebecca Recipient\"", + "messageTransformed": "Expect to equal careOfName (but does not)" + }, + { + "severity": "Warning", + "path": "", + "targetObject": "http:\/\/DeliveryBasketExample#Address", + "sourceShape": "http:\/\/DeliveryBasketExample#AddressConforms", + "targetType": "targetClass", + "constraint": "Equals[]", + "focusNode": "http:\/\/DeliveryBasketExample#address1", + "message": "Equals[]: not equal: value \"Carey careof\" is not in [\"Rebecca Recipient\"]", + "value": "\"Carey careof\"", + "messageTransformed": "Expect to equal careOfName (but does not)" + }, + { + "severity": "Violation", + "path": "", + "targetObject": "http:\/\/DeliveryBasketExample#Address", + "sourceShape": "http:\/\/DeliveryBasketExample#AddressConforms", + "targetType": "targetClass", + "constraint": "Node[]", + "focusNode": "http:\/\/DeliveryBasketExample#addressWithoutRecipient", + "message": "Node[] at focusNode ", + "value": "http:\/\/DeliveryBasketExample#addressWithoutRecipient", + "messageTransformed": "Expect to conform to shape HasAtLeastOneRecipient" + }, + { + "severity": "Warning", + "path": "", + "targetObject": "http:\/\/DeliveryBasketExample#includes", + "sourceShape": "http:\/\/DeliveryBasketExample#SomethingThatIncludesHasCapacity", + "targetType": "targetSubjectsOf", + "constraint": "minInclusive[1]", + "focusNode": "http:\/\/DeliveryBasketExample#basketTooSmall", + "message": "Data value \"0.5\"^^xsd:double is not greater than or equal to 1", + "value": "\"0.5\"^^http:\/\/www.w3.org\/2001\/XMLSchema#double", + "messageTransformed": "Expect a double greater than or equal to 1 (got 0.5)" + }, + { + "severity": "Violation", + "path": "", + "targetObject": "http:\/\/DeliveryBasketExample#identifier", + "sourceShape": "http:\/\/DeliveryBasketExample#IdentifierIsUniqueAndMinLength5", + "targetType": "targetObjectsOf", + "constraint": "MinLengthConstraint[5]", + "focusNode": "\"id0\"", + "message": "MinLengthConstraint[5]: String too short: id0", + "value": "\"id0\"", + "messageTransformed": "Expect string length >= 5 (\"id0\")" + }, + { + "severity": "Violation", + "path": "^", + "targetObject": "http:\/\/DeliveryBasketExample#identifier", + "sourceShape": "http:\/\/DeliveryBasketExample#IdentifierIsUniqueAndMinLength5", + "targetType": "targetObjectsOf", + "constraint": "maxCount[1]", + "focusNode": "\"id0\"", + "message": "maxCount[1]: Invalid cardinality: expected max 1: Got count = 2", + "value": "", + "messageTransformed": "Expect count <= 1 (got 2)" + }, + { + "severity": "Info", + "path": "\/", + "targetObject": "http:\/\/DeliveryBasketExample#FruitBasket", + "sourceShape": "http:\/\/DeliveryBasketExample#FruitBasketIdentifierLength", + "targetType": "targetClass", + "constraint": "MinLengthConstraint[5]", + "focusNode": "http:\/\/DeliveryBasketExample#basket5", + "message": "MinLengthConstraint[5]: String too short: id0", + "value": "\"id0\"", + "messageTransformed": "Expect string length >= 5 (\"id0\")" + }, + { + "severity": "Info", + "path": "\/", + "targetObject": "http:\/\/DeliveryBasketExample#FruitBasket", + "sourceShape": "http:\/\/DeliveryBasketExample#FruitBasketIdentifierLength", + "targetType": "targetClass", + "constraint": "MinLengthConstraint[5]", + "focusNode": "http:\/\/DeliveryBasketExample#basket6", + "message": "MinLengthConstraint[5]: String too short: id0", + "value": "\"id0\"", + "messageTransformed": "Expect string length >= 5 (\"id0\")" + }, + { + "severity": "Violation", + "path": "", + "targetObject": "http:\/\/DeliveryBasketExample#basket100", + "sourceShape": "http:\/\/DeliveryBasketExample#Basket100HasTwoPeachesAndCapacity", + "targetType": "targetNode", + "constraint": "DatatypeConstraint[xsd:int]", + "focusNode": "http:\/\/DeliveryBasketExample#basket100", + "message": "DatatypeConstraint[xsd:int]: Expected xsd:int : Actual xsd:double : Node \"10\"^^xsd:double", + "value": "\"10\"^^http:\/\/www.w3.org\/2001\/XMLSchema#double", + "messageTransformed": "Expect datatype xsd:int (got \"10\"^^xsd:double, type xsd:double)" + }, + { + "severity": "Violation", + "path": "", + "targetObject": "http:\/\/DeliveryBasketExample#basket100", + "sourceShape": "http:\/\/DeliveryBasketExample#Basket100HasTwoPeachesAndCapacity", + "targetType": "targetNode", + "constraint": "ClassConstraint[]", + "focusNode": "http:\/\/DeliveryBasketExample#basket100", + "message": "ClassConstraint[]: Expected class : for ", + "value": "http:\/\/DeliveryBasketExample#fruit100d", + "messageTransformed": "Expect to be an instance of " + }, + { + "severity": "Violation", + "path": "", + "targetObject": "http:\/\/DeliveryBasketExample#basket100", + "sourceShape": "http:\/\/DeliveryBasketExample#Basket100HasTwoPeachesAndCapacity", + "targetType": "targetNode", + "constraint": "maxCount[2]", + "focusNode": "http:\/\/DeliveryBasketExample#basket100", + "message": "maxCount[2]: Invalid cardinality: expected max 2: Got count = 4", + "value": "", + "messageTransformed": "Expect count <= 2 (got 4)" + }, + { + "severity": "Violation", + "path": "", + "targetObject": "http:\/\/DeliveryBasketExample#Address", + "sourceShape": "http:\/\/DeliveryBasketExample#AddressIsNot53217-1234", + "targetType": "targetClass", + "constraint": "Not", + "focusNode": "http:\/\/DeliveryBasketExample#addressTwoZips", + "message": "Zip code cannot be 53217-1234", + "value": "http:\/\/DeliveryBasketExample#addressTwoZips", + "messageTransformed": "" + } + ] +} \ No newline at end of file diff --git a/sparqlGraphLibrary/src/test/resources/DeliveryBasketExample-shacl.ttl b/sparqlGraphLibrary/src/test/resources/DeliveryBasketExample-shacl.ttl new file mode 100644 index 000000000..ed1251578 --- /dev/null +++ b/sparqlGraphLibrary/src/test/resources/DeliveryBasketExample-shacl.ttl @@ -0,0 +1,252 @@ +### +### Sample SHACL, demonstrating the use of various types of shapes, targets, paths, constraints +### Accompanies the model/data in DeliveryBasketExample.owl +### + +@prefix sh: . +@prefix xsd: . +@prefix owl: . +@prefix dbex: . + +### A FruitBasket must include between 1 and 3 fruits +### A FruitBasket expiration date must be later than pack date +### A FruitBasket expiration date must be later or equal to than pack date +dbex:FruitBasketConforms + a sh:NodeShape; + sh:targetClass dbex:FruitBasket; + sh:property [ + sh:path dbex:includes; + sh:minCount 1; + sh:maxCount 3; + ]; + sh:property [ + sh:path dbex:packDate; + sh:lessThan dbex:expirationDate; + ]; + sh:property [ + sh:path dbex:packDate; + sh:lessThanOrEquals dbex:expirationDate; + ]; + . + +### An identifier of anything that FruitBasket includes (e.g. a Fruit) must have min length 5 +### Note: demonstrates sequence path +dbex:FruitBasketIdentifierLength + a sh:NodeShape; + sh:targetClass dbex:FruitBasket; + sh:property [ + sh:path ( dbex:includes dbex:identifier ); + sh:minLength 5; + sh:severity sh:Info; + ]; + . + +### A specific FruitBasket must include at least 2 peaches +### Note: demonstrates targetNode +dbex:FruitBasketIncludesAtLeastTwoPeaches + a sh:NodeShape; + sh:targetNode dbex:basket4; + sh:property [ + sh:message "Must include at least 2 peaches"; + sh:path dbex:includes; + sh:qualifiedValueShape [ + sh:class dbex:Peach; + ]; + sh:qualifiedMinCount 2; + sh:severity sh:Warning; + ]; + . + +### A FruitBasket may include at most 2 peaches +dbex:FruitBasketIncludesAtMostTwoPeaches + a sh:NodeShape; + sh:targetNode dbex:basket100; + sh:property [ + sh:message "Must include at most 2 peaches"; + sh:path dbex:includes; + sh:qualifiedValueShape [ + sh:class dbex:Peach; + ]; + sh:qualifiedMaxCount 2; + sh:severity sh:Warning; + ]; + . + +### A Fruit must be held in exactly one FruitBasket +### Note: demonstrates inverse path +dbex:FruitIsInABasket + a sh:NodeShape; + sh:targetClass dbex:Fruit; + sh:property [ + sh:path [ sh:inversePath dbex:includes]; + sh:minCount 1; + sh:maxCount 1; + ]; + . + +### URI must contain "peach" +### Note: demonstrates multiple targets +dbex:URIContainsPeach + a sh:NodeShape; + sh:targetNode dbex:basket100; + sh:targetClass dbex:Peach; + sh:pattern "(.+)peach(.+)"; + sh:severity sh:Info; + . + +### For anything that "includes" something, if it has a capacity it must be at least 1 +### Note: demonstrates targetSubjectsOf +dbex:SomethingThatIncludesHasCapacity + a sh:NodeShape; + sh:targetSubjectsOf dbex:includes; + sh:property [ + sh:path dbex:capacity; + sh:minInclusive 1; + sh:severity sh:Warning; + ]; + . + +### Identifier length must be at least 5 +### Identifiers must be unique (2 subjects cannot have the same identifier) +### Note: demonstrates targetObjectsOf +dbex:IdentifierIsUniqueAndMinLength5 + a sh:NodeShape; + sh:targetObjectsOf dbex:identifier; + sh:minLength 5; + sh:property [ + sh:path [ sh:inversePath dbex:identifier]; + sh:maxCount 1; + ]; + . + +### A specific FruitBasket cannot include more than 2 objects, and they must be Peaches +### A specific FruitBasket's capacity must be an int +dbex:Basket100HasTwoPeachesAndCapacity + a sh:NodeShape; + sh:targetNode dbex:basket100; + sh:property [ + sh:path dbex:includes; + sh:maxCount 2; + sh:class dbex:Peach; + ]; + sh:property [ + sh:path dbex:capacity; + sh:datatype xsd:int; + ]; + . + +### A specific FruitBasket cannot have any properties besides those in sh:property and sh:ignoredProperties +dbex:Basket100HasNoExtraProperties + a sh:NodeShape; + sh:targetNode dbex:basket100; + sh:closed true; + sh:ignoredProperties (); + sh:property [ + sh:path dbex:includes; + ]; + . + +### An Address must conform to a shape (below) requiring that an address have at least 1 recipient +### An Address must have a zipCode or zipCodePlusFour, but not both +### An Address recipient must be the same as care-of person +dbex:AddressConforms + a sh:NodeShape; + sh:targetClass dbex:Address; + sh:node dbex:HasAtLeastOneRecipient; + sh:xone ( + [ + sh:property [ + sh:path dbex:zipCode ; + sh:minCount 1 ; + ] + ] + [ + sh:property [ + sh:path dbex:zipCodePlusFour ; + sh:minCount 1 ; + ] + ] + ); + sh:property [ + sh:path dbex:recipientName; + sh:equals dbex:careOfName; + sh:severity sh:Warning; + ]; + . + +### An Address zip code cannot be 53217-1234 +dbex:AddressIsNot53217-1234 + a sh:NodeShape; + sh:targetClass dbex:Address; + sh:message "Zip code cannot be 53217-1234"; + sh:not[ + sh:path dbex:zipCodePlusFour; + sh:hasValue "53217-1234"; + ]; + . + +### An Address recipient and care-of person both must be of form "Firstname Lastname" (both capitalized) +### Note: demonstrates a PropertyShape with a target +### Note: demonstrates alternative path +dbex:AddressPersonsHaveFirstLastNameOnly + a sh:PropertyShape; + sh:targetClass dbex:Address; + sh:property [ + sh:path [ sh:alternativePath ( dbex:recipientName dbex:careOfName) ]; + sh:pattern "^[A-Z][a-z]* [A-Z][a-z]*$"; + sh:severity sh:Warning; + ]; + . + +### A Delivery must have an address that conforms to a shape (below) requiring that an address have at least 1 recipient +### The identifier of anything that is (recursively) included in the delivery must be at least length 10 (no error if identifier not present) +### Note: demonstrates one-or-more path +dbex:DeliveryHasRecipientAndLongEnoughIdentifiers + a sh:NodeShape ; + sh:targetClass dbex:Delivery; + sh:property [ + sh:path dbex:address; + sh:node dbex:HasAtLeastOneRecipient; + ]; + sh:property [ + sh:path ( [ sh:oneOrMorePath dbex:includes ] dbex:identifier ); + sh:minLength 10; + sh:severity sh:Info; + ] + . + +### An instance must have at least 1 recipient +### Note: this shape has no target - it is used in other shape-based constraints +dbex:HasAtLeastOneRecipient + a sh:NodeShape; + sh:property [ + sh:path dbex:recipientName; + sh:minCount 1; + ]; + . + +### An Address recipient name cannot contain 'Rex' +### Note: this is a SPARQLConstraint. It is followed by a prefix declaration needed in order for "dbex" to be recognized +dbex:AddressRecipientIsNotRex + a sh:NodeShape ; + sh:targetClass dbex:Address; + sh:sparql [ + a sh:SPARQLConstraint; + sh:message "Recipient name cannot contain 'Rex', but it does"; + sh:prefixes dbex: ; + sh:select """ + SELECT $this (dbex:recipientName AS ?path) ?value + WHERE { + $this dbex:recipientName ?value . + FILTER contains(?value,"Rex") + } + """ ; + ] . +dbex: + a owl:Ontology ; + owl:imports sh: ; + sh:declare [ + sh:prefix "dbex" ; + sh:namespace "http://DeliveryBasketExample#"^^xsd:anyURI ; + ]; + . diff --git a/sparqlGraphLibrary/src/test/resources/DeliveryBasketExample.owl b/sparqlGraphLibrary/src/test/resources/DeliveryBasketExample.owl new file mode 100644 index 000000000..b519d3ff6 --- /dev/null +++ b/sparqlGraphLibrary/src/test/resources/DeliveryBasketExample.owl @@ -0,0 +1,279 @@ + + + + + + This ontology was created from a SADL file 'DeliveryBasketExample.sadl' and should not be directly edited. + + + + + + + + 1 + + + + + + + + + + + 1 + + + + + + + + + 1 + + + + + + + + + 1 + + + + + + + + + 1 + + + + + + + + + + 3 + + + + + + + + + 1 + + + + + + + + + 1 + + + + + + + + + 1 + + + + + + + + + + + + + + + + + + + 1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0.5 + + + + + + + + id0 + + + + + + + + + + + + + + + + + 10 + + + + + 1999-01-01 + 2023-01-01 + + + + + 10027 + + + + + + + 2023-02-01 + 2023-01-01 + + + fruit1 + + + basket1 + + + + + 53217 + Carey careof + Rebecca Recipient + + + + + id0 + + + + + + 53217-1234 + 53217 + Rex Recipient + Rex Recipient + + diff --git a/sparqlGraphLibrary/src/test/resources/DeliveryBasketExample.sadl b/sparqlGraphLibrary/src/test/resources/DeliveryBasketExample.sadl new file mode 100644 index 000000000..4cc8a9abb --- /dev/null +++ b/sparqlGraphLibrary/src/test/resources/DeliveryBasketExample.sadl @@ -0,0 +1,105 @@ +uri "http://DeliveryBasketExample" alias dbex. + +// ===================== model ================================= + +Thing is a class, + described by identifier with a single value of type string. + +Fruit is a type of Thing. +Peach is a type of Fruit. + +Address is a class, + described by recipientName with a single value of type string, + described by careOfName with a single value of type string, + described by zipCode with a single value of type int, + described by zipCodePlusFour with a single value of type string. // e.g. "53217-1234" + +FruitBasket is a type of Thing, + described by includes with values of type Fruit, + described by capacity with a single value of type double, + described by packDate with a single value of type date, + described by expirationDate with a single value of type date. +includes of FruitBasket has at most 3 values. + +Delivery is a class, + described by address with a single value of type Address, + described by includes with values of type FruitBasket. + +// ===================== instance data ================================= + +// a basket with 1 fruit +fruit1 is a Fruit + has identifier "fruit1". +basket1 is a FruitBasket, + has identifier "basket1", + has includes fruit1, + has packDate "2023-01-01", + has expirationDate "2023-02-01". + +// a fruit with no basket +peach1 is a Peach. + +// a basket with no fruit +basket3 is a FruitBasket, + has packDate "2023-01-01", + has expirationDate "1999-01-01". + +// a basket with a peach +peach4 is a Peach. +basket4 is a FruitBasket, + has includes peach4. + +// a basket with 4 fruits +peach100a is a Peach. +peach100b is a Peach. +peach100c is a Peach. +fruit100d is a Fruit. +basket100 is a FruitBasket, + has capacity 10, + has includes peach100a, + has includes peach100b, + has includes peach100c, + has includes fruit100d. + +// a basket with a small capacity...and a fruit squeezed into it anyway +fruitSqueezed is a Fruit. +basketTooSmall is a FruitBasket, + has includes fruitSqueezed, + has capacity 0.5. + +// two items with the same identifier +fruit5 is a Fruit, + has identifier "id0". +fruit6 is a Fruit, + has identifier "id0". + +// 2 baskets holding the same fruit +basket5 is a FruitBasket, + has includes fruit5. +basket6 is a FruitBasket, + has includes fruit5. + +// an address +address1 is a Address, + has recipientName "Rebecca Recipient", + has careOfName "Carey careof", + has zipCode 53217. + +// a delivery +delivery1 is a Delivery, + has address address1, + has includes basket1. + +// a delivery with an address with no recipient +addressWithoutRecipient is a Address, + has zipCode 10027. +deliveryWithoutRecipient is a Delivery, + has address addressWithoutRecipient, + has includes basket3. + +// an address with both zips +addressTwoZips is a Address, + has recipientName "Rex Recipient", + has careOfName "Rex Recipient", + has zipCode 53217, + has zipCodePlusFour "53217-1234". \ No newline at end of file diff --git a/sparqlGraphLibrary/src/test/resources/musicTestDataset-shacl.ttl b/sparqlGraphLibrary/src/test/resources/musicTestDataset-shacl.ttl new file mode 100644 index 000000000..330b3c41c --- /dev/null +++ b/sparqlGraphLibrary/src/test/resources/musicTestDataset-shacl.ttl @@ -0,0 +1,11 @@ +@prefix : . +@prefix sh: . +@prefix xsd: . +@prefix ns: . +:shape1 + a sh:NodeShape; + sh:targetClass ns:AlbumTrack; + sh:property [ + sh:path ns:durationInSeconds; + sh:maxExclusive 300; + ] . \ No newline at end of file diff --git a/sparqlGraphWeb/sparqlGraph/js/exploretab.js b/sparqlGraphWeb/sparqlGraph/js/exploretab.js index 489d41d8b..a8352c0cb 100644 --- a/sparqlGraphWeb/sparqlGraph/js/exploretab.js +++ b/sparqlGraphWeb/sparqlGraph/js/exploretab.js @@ -32,11 +32,13 @@ define([ // properly require.config'ed 'sparqlgraph/js/modaliidx', 'sparqlgraph/js/msiclientnodegroupexec', 'sparqlgraph/js/msiclientontologyinfo', + 'sparqlgraph/js/msiclientutility', 'sparqlgraph/js/msiclientstatus', 'sparqlgraph/js/msiclientresults', 'sparqlgraph/js/msiresultset', 'sparqlgraph/js/ontologyinfo', 'sparqlgraph/js/restrictiontree', + 'sparqlgraph/js/shacltree', 'sparqlgraph/js/visjshelper', 'jquery', @@ -51,17 +53,18 @@ define([ // properly require.config'ed ], // TODO: this isn't leveraging VisJsHelper properly. Code duplication. - function(IIDXHelper, ModalIidx, MsiClientNodeGroupExec, MsiClientOntologyInfo, MsiClientStatus, MsiClientResults, MsiResultSet, OntologyInfo, RestrictionTree, VisJsHelper, $, vis) { + function(IIDXHelper, ModalIidx, MsiClientNodeGroupExec, MsiClientOntologyInfo, MsiClientUtility, MsiClientStatus, MsiClientResults, MsiResultSet, OntologyInfo, RestrictionTree, ShaclTree, VisJsHelper, $, vis) { //============ local object ExploreTab ============= - var ExploreTab = function(treediv, canvasdiv, buttondiv, topControlForm, oInfoClientURL) { + var ExploreTab = function(treediv, canvasdiv, buttondiv, topControlForm, oInfoClientURL, utilityClientURL) { this.controlDivParent = treediv; this.canvasDivParent = canvasdiv; this.botomButtonDiv = buttondiv; this.topControlForm = topControlForm; this.oInfoClientURL = oInfoClientURL; + this.utilityClientURL = utilityClientURL; this.infospan = document.createElement("span"); this.infospan.style.marginRight = "3ch"; @@ -93,7 +96,8 @@ define([ // properly require.config'ed ExploreTab.MODE_INSTANCE = "Instance Data"; ExploreTab.MODE_STATS = "Instance Counts"; ExploreTab.MODE_RESTRICTIONS = "Restrictions"; - ExploreTab.MODES = [ExploreTab.MODE_ONTOLOGY_CLASSES, ExploreTab.MODE_ONTOLOGY_DETAIL, ExploreTab.MODE_INSTANCE, ExploreTab.MODE_STATS, ExploreTab.MODE_RESTRICTIONS]; + ExploreTab.MODE_SHACL = "SHACL Validation"; + ExploreTab.MODES = [ExploreTab.MODE_ONTOLOGY_CLASSES, ExploreTab.MODE_ONTOLOGY_DETAIL, ExploreTab.MODE_INSTANCE, ExploreTab.MODE_STATS, ExploreTab.MODE_RESTRICTIONS, ExploreTab.MODE_SHACL]; ExploreTab.prototype = { @@ -133,8 +137,9 @@ define([ // properly require.config'ed this.networkHash[m] = new vis.Network(this.canvasDivHash[m], {}, this.getDefaultOptions(m)); } - // in restriction mode, enable double-click to expand node + // in some modes, enable double-click to expand node this.networkHash[ExploreTab.MODE_RESTRICTIONS].on('doubleClick', VisJsHelper.expandSelectedNodes.bind(this, this.canvasDivHash[ExploreTab.MODE_RESTRICTIONS], this.networkHash[ExploreTab.MODE_RESTRICTIONS])); + this.networkHash[ExploreTab.MODE_SHACL].on('doubleClick', VisJsHelper.expandSelectedNodes.bind(this, this.canvasDivHash[ExploreTab.MODE_SHACL], this.networkHash[ExploreTab.MODE_SHACL])); return this.networkHash; }, @@ -238,6 +243,7 @@ define([ // properly require.config'ed this.oTree.setOInfo(this.oInfo); this.restrictionTree.clear(); + this.shaclTree.clear(); this.clearNetwork(ExploreTab.MODES); }, @@ -276,7 +282,7 @@ define([ // properly require.config'ed var bold = document.createElement("b"); td.appendChild(bold); bold.innerHTML = "Explore mode: "; - var select = IIDXHelper.createSelect("etSelect", [ExploreTab.MODE_ONTOLOGY_CLASSES, ExploreTab.MODE_ONTOLOGY_DETAIL, ExploreTab.MODE_INSTANCE, ExploreTab.MODE_STATS, ExploreTab.MODE_RESTRICTIONS], [ExploreTab.MODE_ONTOLOGY_CLASSES]); + var select = IIDXHelper.createSelect("etSelect", [ExploreTab.MODE_ONTOLOGY_CLASSES, ExploreTab.MODE_ONTOLOGY_DETAIL, ExploreTab.MODE_INSTANCE, ExploreTab.MODE_STATS, ExploreTab.MODE_RESTRICTIONS, ExploreTab.MODE_SHACL], [ExploreTab.MODE_ONTOLOGY_CLASSES]); select.onchange = this.draw.bind(this); td.appendChild(select); @@ -317,6 +323,7 @@ define([ // properly require.config'ed this.initControlDiv_OntologyDetailMode(); this.initControlDiv_StatsMode(); this.initControlDiv_RestrictionsMode(); + this.initControlDiv_ShaclMode(); }, // Initialize control div for ontology classes mode @@ -387,7 +394,7 @@ define([ // properly require.config'ed // add 2 dropdowns, using a table for formatting // dropdown to show violations only (exceeds maximum cardinality) or also incomplete data (does not meet minimum cardinality) - var modeSelectDropdown = IIDXHelper.createSelect("restrictionTreeModeSelect", [["violations only",0], ["violations and incomplete data",1]], ["multi"], false, "input-large"); + var modeSelectDropdown = IIDXHelper.createSelect("restrictionTreeModeSelect", [["violations only",0], ["violations and incomplete data",1]], ["violations and incomplete data"], false, "input-large"); modeSelectDropdown.onchange = function() { this.restrictionTree.setExceedsOnlyMode(parseInt(document.getElementById("restrictionTreeModeSelect").value) == 0); this.restrictionTree.draw(); @@ -397,33 +404,83 @@ define([ // properly require.config'ed sortSelectDropdown.onchange = function() { this.restrictionTree.setSortMode(parseInt(document.getElementById("restrictionTreeSortSelect").value)); this.restrictionTree.sort(); - }.bind(this); - var table = document.createElement("table"); - col = document.createElement("col"); - col.style.width = "25%"; - table.appendChild(col); - tr = document.createElement("tr"); - td1 = document.createElement("td"); - td1.innerHTML = "Show:"; - tr.append(td1); - td2 = document.createElement("td"); - td2.append(modeSelectDropdown); - tr.append(td2); - table.appendChild(tr); - tr2 = document.createElement("tr"); - td1 = document.createElement("td"); - td1.innerHTML = "Sort by:"; - tr2.append(td1); - td2 = document.createElement("td"); - td2.append(sortSelectDropdown); - tr2.append(td2); - table.appendChild(tr2); - div.appendChild(table); + }.bind(this); + var table = div.appendChild(document.createElement("table")); + table.appendChild(document.createElement("col")); + var tr = table.insertRow(); + tr.insertCell().appendChild(document.createTextNode("Show:")); + tr.insertCell().appendChild(modeSelectDropdown); // severity dropdown + tr = table.insertRow(); + tr.insertCell().appendChild(document.createTextNode("Sort by:")); + tr.insertCell().appendChild(sortSelectDropdown); // sort-by dropdown - // initialize network + // initialize tree this.initDynaTree_RestrictionsMode(); this.restrictionTree.setSortMode(0); // set sort to default }, + + // Initialize control div for SHACL mode + initControlDiv_ShaclMode: function() { + var div = this.controlDivHash[ExploreTab.MODE_SHACL]; + div.style.margin = "1ch"; + + // button to select SHACL rules file + var runSelectedShaclFile = function(e) { + // after user selects file, query for shacl results and display them in tree + if (e.target.files.length > 0) { // If user hit cancel, then get 0 files here. File picker disallows multiple files. + var shaclCallback = this.buildStatusResultsCallback( + this.shaclTree.draw.bind(this.shaclTree), // after retrieving table, draw the tree + MsiClientResults.prototype.execGetJsonBlobRes + ).bind(this); + + this.clearNetwork(); // clear the graph + this.shaclTree.clear(); + var client = new MsiClientUtility(this.utilityClientURL, ModalIidx.alert.bind(this, "Error")); + client.execGetShaclResults(gConn, e.target.files[0], "Info", shaclCallback); + shaclTtlFileUploader.value = null; // reset so that reload the same file triggers the change event + } + }.bind(this); + var shaclTtlFileUploader = document.createElement("input"); + shaclTtlFileUploader.type = "file"; + shaclTtlFileUploader.accept = ".ttl"; // accept files with ttl extension only + shaclTtlFileUploader.style = "color: rgba(0, 0, 0, 0)"; // hides the "No file chosen" text + shaclTtlFileUploader.addEventListener('change', runSelectedShaclFile, false); + + // dropdown for severity + var modeSelectDropdown = IIDXHelper.createSelect("shaclTreeSeverityModeSelect", [ + ["violations only", ShaclTree.SEVERITYMODE_VIOLATION], + ["violations & warnings", ShaclTree.SEVERITYMODE_WARNING], + ["violations, warnings & info", ShaclTree.SEVERITYMODE_INFO] + ], ["violations, warnings & info"], false, "input-large"); + modeSelectDropdown.onchange = function() { + this.shaclTree.setSeverityMode(document.getElementById("shaclTreeSeverityModeSelect").value); + this.shaclTree.draw(); + }.bind(this); + + // dropdown to pick sort option + var sortSelectDropdown = IIDXHelper.createSelect("shaclTreeSortSelect", [["shape", 0], ["count", 1]], ["multi"], false, "input-large"); + sortSelectDropdown.onchange = function() { + this.shaclTree.setSortMode(parseInt(document.getElementById("shaclTreeSortSelect").value)); + this.shaclTree.sort(); + }.bind(this); + + // put the 3 elements above into a table + var table = div.appendChild(document.createElement("table")); + table.appendChild(document.createElement("col")); + var tr = table.insertRow(); + tr.insertCell().appendChild(document.createTextNode("Select SHACL file:")); + tr.insertCell().appendChild(shaclTtlFileUploader); // file uploader button + tr = table.insertRow(); + tr.insertCell().appendChild(document.createTextNode("Show:")); + tr.insertCell().appendChild(modeSelectDropdown); // severity dropdown + tr = table.insertRow(); + tr.insertCell().appendChild(document.createTextNode("Sort by:")); + tr.insertCell().appendChild(sortSelectDropdown); // sort-by dropdown + + // initialize tree + this.initDynaTree_ShaclMode(); + this.shaclTree.setSortMode(0); // set sort to default + }, /* * Initialize an empty dynatree into this.controlDivHash[ExploreTab.MODE_INSTANCE] @@ -488,20 +545,35 @@ define([ // properly require.config'ed this.controlDivParent.appendChild(this.controlDivHash[ExploreTab.MODE_RESTRICTIONS]); var treeSelector = "#" + this.controlDivHash[ExploreTab.MODE_RESTRICTIONS].id; $(treeSelector).dynatree({ - onSelect: function(flag, node) { this.modifyNetwork_RestrictionsMode(flag, node); // user selects/deselects a node }.bind(this), - persist: false, // true causes a cookie error with large trees selectMode: 1, // 1 single, 2 multi, 3 multi hierarchical checkbox: true, }); - this.restrictionTree = new RestrictionTree($(treeSelector).dynatree("getTree")); - this.restrictionTree.setExceedsOnlyMode(true); // to match initial state of dropdown }, + /* + * Initialize an empty dynatree into this.controlDivHash[ExploreTab.MODE_SHACL] + */ + initDynaTree_ShaclMode: function() { + + this.controlDivParent.innerHtml = ""; + this.controlDivParent.appendChild(this.controlDivHash[ExploreTab.MODE_SHACL]); + var treeSelector = "#" + this.controlDivHash[ExploreTab.MODE_SHACL].id; + $(treeSelector).dynatree({ + onSelect: function(flag, node) { + this.modifyNetwork_ShaclMode(flag, node); // user selects/deselects a node + }.bind(this), + persist: false, // true causes a cookie error with large trees + selectMode: 1, // 1 single, 2 multi, 3 multi hierarchical + checkbox: true, + }); + this.shaclTree = new ShaclTree($(treeSelector).dynatree("getTree")); + }, + /** * For restrictions mode, construct network when user selects a URI. Clear the network if user de-selects a URI. * flag - true if selected, false if de-selected @@ -537,6 +609,40 @@ define([ // properly require.config'ed client.execAsyncConstructInstanceWithPredicates(uri, classUri, [predicate], gConn, resultsCallback, networkFailureCallback.bind(this, canvasDiv)); }, + /** + * For SHACL mode, construct network when user selects an offending item. Clear the network if user de-selects. + * flag - true if selected, false if de-selected + * node - the node + */ + modifyNetwork_ShaclMode: function(flag, node) { + + const canvasDiv = this.canvasDivHash[ExploreTab.MODE_SHACL]; + const network = this.networkHash[ExploreTab.MODE_SHACL]; + + if (!flag) { + // URI was de-selected, clear the network (tree allows only one URI selected at a time) + this.clearNetwork(); + return; + } + this.clearNetwork(); // clear any prior network (tree allows only one URI selected at a time) + VisJsHelper.networkBusy(canvasDiv, true); + + var client = new MsiClientNodeGroupExec(g.service.nodeGroupExec.url, g.longTimeoutMsec); + var resultsCallback = MsiClientNodeGroupExec.buildJsonLdOrTriplesCallback( + VisJsHelper.addTriples.bind(this, canvasDiv, network), // add triples to graph + networkFailureCallback.bind(this, canvasDiv), + function() { }, // no status updates + function() { }, // no check for cancel + g.service.status.url, + g.service.results.url + ); + + // construct all connected data + const nodeTitle = node.data.title; // this could be an instance URI or a literal + // TODO line below uses instance type "node_uri" even though the nodeTitle may be a literal. Currently works for literal strings (e.g. "id0") but need to revisit. + client.execAsyncConstructConnectedData(nodeTitle, "node_uri", SemanticNodeGroup.RT_NTRIPLES, VisJsHelper.ADD_TRIPLES_MAX, gConn, resultsCallback, networkFailureCallback.bind(this, canvasDiv)); + }, + // main section of buttons along the bottom. // Should have controls that are useful for all modes @@ -707,6 +813,10 @@ define([ // properly require.config'ed ); client.execGetCardinalityViolations(gConn, MAX_RESTRICTION_ROWS, true, cardinalityCallback); } + } else if (this.getMode() == ExploreTab.MODE_SHACL) { + + // don't draw anything, user must select a SHACL file + this.clearGraphInfoBar(); // remove nodes/predicates info bar } this.networkHash[this.getMode()].fit(); // fit the graph to the canvas diff --git a/sparqlGraphWeb/sparqlGraph/js/msiclientutility.js b/sparqlGraphWeb/sparqlGraph/js/msiclientutility.js index a93640749..928ff2936 100644 --- a/sparqlGraphWeb/sparqlGraph/js/msiclientutility.js +++ b/sparqlGraphWeb/sparqlGraph/js/msiclientutility.js @@ -43,6 +43,14 @@ define([ // properly require.config'ed bootstrap-modal }); this.msi.postAndCheckSuccess("processPlotSpec", data, "application/json", successCallback, this.optFailureCallback, this.optTimeout); }, + + execGetShaclResults : function (conn, shaclTtlFile, severity, successCallback) { + var formdata = new FormData(); + formdata.append("conn", JSON.stringify(conn.toJson())); + formdata.append("shaclTtlFile", shaclTtlFile); + formdata.append("severity", severity); + this.msi.postToEndpoint("getShaclResults", formdata, MicroServiceInterface.FORM_CONTENT, successCallback, this.optFailureCallback, this.optTimeout); + }, }; diff --git a/sparqlGraphWeb/sparqlGraph/js/restrictiontree.js b/sparqlGraphWeb/sparqlGraph/js/restrictiontree.js index 3a0c8d431..4276e3ef9 100644 --- a/sparqlGraphWeb/sparqlGraph/js/restrictiontree.js +++ b/sparqlGraphWeb/sparqlGraph/js/restrictiontree.js @@ -14,15 +14,15 @@ ** limitations under the License. */ define([ // properly require.config'ed + 'sparqlgraph/js/tree', + 'sparqlgraph/js/visjshelper', 'jquery', // shimmed 'sparqlgraph/dynatree-1.2.5/jquery.dynatree', - - ], - function($) { + function(Tree, VisJsHelper, $) { /* * RestrictionTree * A tree of cardinality restriction violations that appears in Restriction Explore mode. @@ -31,6 +31,7 @@ define([ // properly require.config'ed */ var RestrictionTree = function(dynaTree) { this.tree = dynaTree; + this.setExceedsOnlyMode(0); // matches initial dropdown selection }; // column indexes in the violations table that is used to populate the tree @@ -44,20 +45,6 @@ define([ // properly require.config'ed RestrictionTree.prototype = { - /** - * Remove all tree nodes - */ - clear : function(){ - this.tree.getRoot().removeChildren(); - }, - - /** - * Determine if the tree is empty or not - */ - isEmpty : function(){ - return !this.tree.getRoot().hasChildren(); - }, - /** * Set/unset exceed-only mode */ @@ -95,13 +82,6 @@ define([ // properly require.config'ed } }, - // compare function for sorting - compareTitle : function (a, b) { - a = a.data.title.toLowerCase(); - b = b.data.title.toLowerCase(); - return a > b ? 1 : a < b ? -1 : 0; - }, - // compare function for sorting compareClassViolationCount : function (a, b) { a = a.data.classViolationCount; @@ -115,71 +95,10 @@ define([ // properly require.config'ed b = b.data.classViolationPercentage; return a > b ? -1 : a < b ? 1 : 0; }, - - /** - * Add a node to the tree - * - * title - node label - * tooltip - node tooltip - * isFolder - true if this is not a leaf node - * optParentNode - adds new node as child node to this node if provided, else uses root as parent - */ - addNode : function(title, tooltip, isFolder, optParentNode) { - - // if parent node not provided, use root - // generate new id for this node - var parentNode; - var id; - if(typeof optParentNode === "undefined"){ - parentNode = this.tree.getRoot(); - id = title; - }else{ - parentNode = optParentNode; - id = parentNode.data.id + title; - } - // if node already exists then return it without adding - if(this.getNode(id) != undefined){ - return this.getNode(id); - } - - // add the node - parentNode.addChild({ - title: title, - id: id, - tooltip: tooltip, - isFolder: isFolder, - hideCheckbox: isFolder, // only want checkboxes on leaf nodes - icon: false - }); - - return this.getNode(id); - }, - - /** - * Set a node attribute - */ - setNodeAttribute: function(node, key, value) { - node.data[key] = value; - }, - - /** - * Get a node by its unique id - */ - getNode : function(id) { - var ret = undefined; - this.tree.getRoot().visit(function(node){ - if (node.data.id === id) { - ret = node; - } - }); - return ret; - }, /** * Build the restriction tree - * * tableRes - results object containing the violation table - * sortModeInt - sort mode */ draw: function(tableRes) { @@ -188,11 +107,6 @@ define([ // properly require.config'ed return className.includes("/") ? className.slice(className.lastIndexOf("/") + 1, className.length) : className; } - // remove prefix (e.g. http://item#description => description) - var stripPrefix = function(uri) { - return (uri.indexOf("#") > -1) ? uri.split("#")[1] : uri; - } - // get human-readable restriction text (e.g. mincardinality => "at least") var getReadableRestriction = function(restriction) { return (restriction.includes("max")) ? "at most" : (restriction.includes("min") ? "at least" : "exactly"); @@ -241,7 +155,7 @@ define([ // properly require.config'ed const classViolationPercentage = Math.min(100, Math.round((classViolationCount / classInstanceCount) * 100)); // may be >100% if a single instance has multiple violations...cap at 100% const limit = row[RestrictionTree.COLINDEX_LIMIT]; const predicate = row[RestrictionTree.COLINDEX_PREDICATE]; - const predicateLocal = stripPrefix(predicate); + const predicateLocal = VisJsHelper.stripPrefix(predicate); const restrictionDescription = "restriction \"has " + getReadableRestriction(row[RestrictionTree.COLINDEX_RESTRICTION]) + " " + limit + " " + predicateLocal + "\""; const actualCount = row[RestrictionTree.COLINDEX_ACTUALCOUNT]; @@ -253,7 +167,7 @@ define([ // properly require.config'ed continue; } - let node1 = this.addNode(getLocalClassname(className) + " (" + classViolationCount + " problems across " + classViolationPercentage.toFixed(0) + "% of instances)", className, true, undefined); // tree node indicating class + let node1 = this.addNode(getLocalClassname(className) + " (" + classViolationCount + " items across " + classViolationPercentage.toFixed(0) + "% of instances)", className, true, undefined); // tree node indicating class this.setNodeAttribute(node1, "classViolationCount", classViolationCount); // used for sorting this.setNodeAttribute(node1, "classViolationPercentage", classViolationPercentage); // used for sorting let node2 = this.addNode(restrictionDescription, "", true, node1); // tree node describing restriction (e.g. "must have at least 2 codes") @@ -279,6 +193,10 @@ define([ // properly require.config'ed this.sort(); }, } + + // RestrictionTree extends Tree + Object.setPrototypeOf(RestrictionTree.prototype, Tree.prototype); + return RestrictionTree; } diff --git a/sparqlGraphWeb/sparqlGraph/js/shacltree.js b/sparqlGraphWeb/sparqlGraph/js/shacltree.js new file mode 100644 index 000000000..c3ba92603 --- /dev/null +++ b/sparqlGraphWeb/sparqlGraph/js/shacltree.js @@ -0,0 +1,190 @@ +/** + ** Copyright 2023 General Electric Company + ** + ** Licensed under the Apache License, Version 2.0 (the "License"); + ** you may not use this file except in compliance with the License. + ** You may obtain a copy of the License at + ** + ** http://www.apache.org/licenses/LICENSE-2.0 + ** + ** Unless required by applicable law or agreed to in writing, software + ** distributed under the License is distributed on an "AS IS" BASIS, + ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ** See the License for the specific language governing permissions and + ** limitations under the License. + */ +define([ // properly require.config'ed + 'sparqlgraph/js/tree', + 'sparqlgraph/js/visjshelper', + + 'jquery', + + // shimmed + 'sparqlgraph/dynatree-1.2.5/jquery.dynatree', + ], + function(Tree, VisJsHelper, $) { + /* + * ShaclTree + * A tree of SHACL validation results that appears in Shacl Validation Explore mode. + */ + var ShaclTree = function(dynaTree) { + this.tree = dynaTree; + this.setSeverityMode(ShaclTree.SEVERITYMODE_INFO); // default to info, matches initial dropdown selection + }; + + // json keys + ShaclTree.JSON_KEY_SOURCESHAPE = "sourceShape"; + ShaclTree.JSON_KEY_TARGETTYPE = "targetType"; + ShaclTree.JSON_KEY_TARGETOBJECT = "targetObject"; + ShaclTree.JSON_KEY_PATH = "path"; + ShaclTree.JSON_KEY_SEVERITY = "severity"; + ShaclTree.JSON_KEY_FOCUSNODE = "focusNode"; + ShaclTree.JSON_KEY_MESSAGE = "message"; + ShaclTree.JSON_KEY_MESSAGETRANSFORMED = "messageTransformed"; + + // severity modes + ShaclTree.SEVERITYMODE_VIOLATION = "Violation"; + ShaclTree.SEVERITYMODE_WARNING = "Warning"; + ShaclTree.SEVERITYMODE_INFO = "Info"; + ShaclTree.prototype = { + + /** + * Set severity mode + */ + setSeverityMode : function(severity){ + this.severityMode = severity; + }, + + /** + * Get severity mode + */ + getSeverityMode : function(){ + return this.severityMode; + }, + + /** + * Set sort mode + * sortModeInt - an integer indicating how to sort (0 for title, 1 for count) + */ + setSortMode : function(sortModeInt){ + this.sortMode = sortModeInt; + }, + + /** + * Sort the tree + */ + sort : function(){ + if(this.sortMode == 0){ + this.tree.getRoot().sortChildren(this.compareTitle, false); + }else if(this.sortMode == 1){ + this.tree.getRoot().sortChildren(this.compareLeafCount.bind(this), false); + }else{ + alert("Error: unrecognized sort mode"); + } + }, + + /** + * Compare severities + */ + compareSeverity : function (a, b) { + + // confirm that each is a valid severity level + const severities = [ShaclTree.SEVERITYMODE_VIOLATION, ShaclTree.SEVERITYMODE_WARNING, ShaclTree.SEVERITYMODE_INFO] + if(a == null || b == null || !severities.includes(a) || !severities.includes(b)){ + throw new Error("Cannot compare severity '" + a + "' to severity '" + b + "'"); + } + + if(a == b){ + return 0; + }else if(a == ShaclTree.SEVERITYMODE_VIOLATION){ + return 1; // a is Violation, b is Warning or Info + }else if (a == ShaclTree.SEVERITYMODE_WARNING){ + if(b == ShaclTree.SEVERITYMODE_VIOLATION){ + return -1; // a is Warning, b is Violation + }else{ + return 1; // a is Warning, b is Info + } + }else{ + return -1; // a is Info, b is Warning or Violation + } + }, + + /** + * Build the tree + * json - object containing an xhr element with the SHACL results + */ + draw: function(json) { + + // get results json + if(typeof json !== 'undefined'){ + this.shaclJsonSaved = json.xhr; // store it for future reuse + } + if(typeof this.shaclJsonSaved === 'undefined'){ + console.error("Error: no SHACL results available"); + return; + } + + this.clear(); // clear tree + + for (var key in this.shaclJsonSaved.reportEntries) { + var entry = this.shaclJsonSaved.reportEntries[key]; + + const sourceShape = entry[ShaclTree.JSON_KEY_SOURCESHAPE]; // e.g. http://DeliveryBasketExample#Shape_Fruit + const targetType = entry[ShaclTree.JSON_KEY_TARGETTYPE]; // One of: targetNode, targetClass, targetSubjectsOf, targetObjectsOf + const targetObject = entry[ShaclTree.JSON_KEY_TARGETOBJECT]; // If targetClass, expect a class URI. If targetSubjectsOf/targetObjectsOf, expect a predicate URI. If targetNode, expect a URI or literal + const path = entry[ShaclTree.JSON_KEY_PATH]; // e.g. , /, more + const message = entry[ShaclTree.JSON_KEY_MESSAGE]; // e.g. "maxCount[3]: Invalid cardinality: expected max 3: Got count = 4" (this one is auto-generated, may be custom if provided) + var messageTransformed = entry[ShaclTree.JSON_KEY_MESSAGETRANSFORMED]; // e.g. more user-friendly version of the original message (if empty use original) + const severity = entry[ShaclTree.JSON_KEY_SEVERITY]; // e.g. "Info", "Warning", "Violation" + const focusNode = entry[ShaclTree.JSON_KEY_FOCUSNODE]; + + // screen entries for severity level + if(this.compareSeverity(severity, this.getSeverityMode()) < 0){ + continue; + } + + // level 0 node (source shape) + let node0 = this.addNode("Shape: " + VisJsHelper.stripPrefix(sourceShape), "", true, undefined); + + // level 1 node (target type + target object) + // use human-friendly text for target type (e.g. targetObjectsOf => Objects Of) + let node1 = this.addNode("Target: " + targetType.substring(6).replace(/([A-Z])/g, ' $1') + " " + VisJsHelper.stripPrefix(targetObject), targetType + " " + targetObject, true, node0); + + // level 2 node (path) + let node2 = this.addNode("Path: " + path, path, true, node1); + + // level 3 node (severity + message) + if(messageTransformed == null || messageTransformed == ""){ + messageTransformed = message; // if empty, use original message + } + let node3 = this.addNode(severity.toUpperCase() + ": " + messageTransformed, message, true, node2); + + // level 4 node (focus node) + this.addNode(focusNode, "", false, node3); + } + + // add leaf counts (e.g. "5 items") to top-level nodes + this.addLeafCountsToRootChildren(); + + // if no tree entries, then add a "none" entry + if(!this.tree.getRoot().hasChildren()){ + this.tree.getRoot().addChild({ + title: "None", + isFolder: false, + hideCheckbox: true, // don't need a checkbox + icon: false + }); + } + + // sort + this.sort(); + }, + } + + // ShaclTree extends Tree + Object.setPrototypeOf(ShaclTree.prototype, Tree.prototype); + + return ShaclTree; + } + +); \ No newline at end of file diff --git a/sparqlGraphWeb/sparqlGraph/js/sparqlgraph.js b/sparqlGraphWeb/sparqlGraph/js/sparqlgraph.js index 560cbbd72..11172d09b 100644 --- a/sparqlGraphWeb/sparqlGraph/js/sparqlgraph.js +++ b/sparqlGraphWeb/sparqlGraph/js/sparqlgraph.js @@ -117,7 +117,8 @@ document.getElementById("exploreCanvasDiv"), document.getElementById("exploreButtonDiv"), document.getElementById("exploreSearchForm"), - g.service.ontologyInfo.url + g.service.ontologyInfo.url, + g.service.utility.url ); setTabButton("explore-tab-but", false); diff --git a/sparqlGraphWeb/sparqlGraph/js/tree.js b/sparqlGraphWeb/sparqlGraph/js/tree.js new file mode 100644 index 000000000..0dab638f1 --- /dev/null +++ b/sparqlGraphWeb/sparqlGraph/js/tree.js @@ -0,0 +1,157 @@ +/** + ** Copyright 2023 General Electric Company + ** + ** Licensed under the Apache License, Version 2.0 (the "License"); + ** you may not use this file except in compliance with the License. + ** You may obtain a copy of the License at + ** + ** http://www.apache.org/licenses/LICENSE-2.0 + ** + ** Unless required by applicable law or agreed to in writing, software + ** distributed under the License is distributed on an "AS IS" BASIS, + ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ** See the License for the specific language governing permissions and + ** limitations under the License. + */ +define([ // properly require.config'ed + + 'jquery', + + // shimmed + 'sparqlgraph/dynatree-1.2.5/jquery.dynatree', + ], + function($) { + /* + * Tree + * A tree that will be extended (e.g. for the control pane of various explore modes) + */ + var Tree = function(dynaTree) { + this.tree = dynaTree; + }; + + Tree.prototype = { + + /** + * Remove all tree nodes + */ + clear : function(){ + this.tree.getRoot().removeChildren(); + }, + + /** + * Determine if the tree is empty or not + */ + isEmpty : function(){ + return !this.tree.getRoot().hasChildren(); + }, + + /** + * Compare titles (for sorting) + */ + compareTitle : function (a, b) { + a = a.data.title.toLowerCase(); + b = b.data.title.toLowerCase(); + return a > b ? 1 : a < b ? -1 : 0; + }, + + /** + * Compare num leaf nodes (for sorting) + */ + compareLeafCount : function (a, b) { + a = this.getNumLeaves(a); + b = this.getNumLeaves(b); + return a > b ? -1 : a < b ? 1 : 0; // descending order + }, + + /** + * Add a node to the tree + * + * title - node label + * tooltip - node tooltip + * isFolder - true if this is not a leaf node + * optParentNode - adds new node as child node to this node if provided, else uses root as parent + */ + addNode : function(title, tooltip, isFolder, optParentNode) { + + // format node title + title = title.replaceAll("<","<").replaceAll(">",">"); + + // if parent node not provided, use root + // generate new id for this node + var parentNode; + var id; + if(typeof optParentNode === "undefined"){ + parentNode = this.tree.getRoot(); + id = title; + }else{ + parentNode = optParentNode; + id = parentNode.data.id + title; + } + // if node already exists then return it without adding + if(this.getNode(id) != undefined){ + return this.getNode(id); + } + + // add the node + parentNode.addChild({ + title: title, + id: id, + tooltip: tooltip, + isFolder: isFolder, + hideCheckbox: isFolder, // only want checkboxes on leaf nodes + icon: false + }); + + return this.getNode(id); + }, + + /** + * Set a node attribute + */ + setNodeAttribute: function(node, key, value) { + node.data[key] = value; + }, + + /** + * Get a node by its unique id + */ + getNode : function(id) { + var ret = undefined; + this.tree.getRoot().visit(function(node){ + if (node.data.id === id) { + ret = node; + } + }); + return ret; + }, + + /** + * Count the number of leaf nodes under a given node + */ + getNumLeaves : function(node){ + if(node.hasChildren() === false){ // use triple-equals per dynatree documentation + return 1; // it's a leaf + }else{ + var count = 0; + for(const n of node.getChildren()){ + count += this.getNumLeaves(n); + } + return count; + } + }, + + /** + * To each child of the root node, append leaf count "(X items)" + */ + addLeafCountsToRootChildren: function() { + if(this.tree.getRoot().hasChildren()){ + for(const node of this.tree.getRoot().getChildren()){ + node.setTitle(node.data.title + " (" + this.getNumLeaves(node) + " items)"); // sets attribute and also redraws + } + } + }, + } + return Tree; + } + +); \ No newline at end of file diff --git a/sparqlGraphWeb/sparqlGraph/js/visjshelper.js b/sparqlGraphWeb/sparqlGraph/js/visjshelper.js index 3b6d52684..6a5e4f407 100644 --- a/sparqlGraphWeb/sparqlGraph/js/visjshelper.js +++ b/sparqlGraphWeb/sparqlGraph/js/visjshelper.js @@ -565,6 +565,13 @@ define([ // properly require.config'ed return p.toLowerCase().endsWith("#type") || p.toLowerCase().endsWith("#type>"); }; + /** + * Removes prefix from URI (e.g. http://item#description => description) if present + */ + VisJsHelper.stripPrefix = function(uri) { + return (uri.indexOf("#") > -1) ? uri.split("#")[1] : uri; + } + /* * create a unique id with base name */ diff --git a/utilityService/src/main/java/com/ge/research/semtk/services/utility/UtilityServiceRestController.java b/utilityService/src/main/java/com/ge/research/semtk/services/utility/UtilityServiceRestController.java index 959bf7868..5b5cf68f2 100755 --- a/utilityService/src/main/java/com/ge/research/semtk/services/utility/UtilityServiceRestController.java +++ b/utilityService/src/main/java/com/ge/research/semtk/services/utility/UtilityServiceRestController.java @@ -42,12 +42,15 @@ import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; import org.apache.commons.io.FileUtils; +import org.apache.jena.shacl.validation.Severity; import org.json.simple.JSONObject; import com.ge.research.semtk.auth.AuthorizationManager; import com.ge.research.semtk.auth.ThreadAuthenticator; import com.ge.research.semtk.belmont.NodeGroup; import com.ge.research.semtk.belmont.runtimeConstraints.RuntimeConstraintManager; +import com.ge.research.semtk.edc.JobTracker; +import com.ge.research.semtk.edc.client.ResultsClient; import com.ge.research.semtk.load.config.ManifestConfig; import com.ge.research.semtk.load.utility.SparqlGraphJson; import com.ge.research.semtk.plotting.PlotlyPlotSpec; @@ -57,6 +60,7 @@ import com.ge.research.semtk.resultSet.Table; import com.ge.research.semtk.resultSet.TableResultSet; import com.ge.research.semtk.sparqlX.SparqlConnection; +import com.ge.research.semtk.sparqlX.SparqlResultTypes; import com.ge.research.semtk.sparqlX.client.SparqlQueryAuthClientConfig; import com.ge.research.semtk.sparqlX.client.SparqlQueryClient; import com.ge.research.semtk.springutillib.headers.HeadersManager; @@ -65,11 +69,13 @@ import com.ge.research.semtk.springutillib.properties.NodegroupExecutionServiceProperties; import com.ge.research.semtk.springutillib.properties.ServicesGraphProperties; import com.ge.research.semtk.springutillib.properties.QueryServiceUserPasswordProperties; +import com.ge.research.semtk.springutillib.properties.ResultsServiceProperties; import com.ge.research.semtk.springutillib.properties.NodegroupStoreServiceProperties; import com.ge.research.semtk.springutillib.properties.QueryServiceProperties; import com.ge.research.semtk.utility.LocalLogger; import com.ge.research.semtk.utility.Logger; import com.ge.research.semtk.utility.Utility; +import com.ge.research.semtk.validate.ShaclRunner; import io.swagger.v3.oas.annotations.Operation; @@ -102,6 +108,8 @@ public class UtilityServiceRestController { private QueryServiceProperties query_prop; @Autowired private QueryServiceUserPasswordProperties query_credentials_prop; + @Autowired + private ResultsServiceProperties results_prop; @PostConstruct public void init() { @@ -115,6 +123,7 @@ public void init() { query_prop.validateWithExit(); query_credentials_prop.validateWithExit(); servicesgraph_prop.validateWithExit(); + results_prop.validateWithExit(); lock = new ReentrantLock(); } @@ -452,7 +461,59 @@ public void loadIngestionPackage( @RequestParam("serverAndPort") String serverAn } } + @Operation( + summary="Use SHACL to validate the contents of a connection", + description="Async. Returns a jobID." + ) + @CrossOrigin + @RequestMapping(value="/getShaclResults", method=RequestMethod.POST, consumes=MediaType.MULTIPART_FORM_DATA_VALUE) + public JSONObject getShaclResults( @RequestParam("shaclTtlFile") MultipartFile shaclTtlFile, + @RequestParam("conn") String connJsonStr, + @RequestParam("severity") Severity severity, + @RequestHeader HttpHeaders headers){ + HeadersManager.setHeaders(headers); + final String ENDPOINT_NAME = "getShaclResults"; + SimpleResultSet retval = new SimpleResultSet(false); + try { + String jobId = JobTracker.generateJobId(); + JobTracker tracker = new JobTracker(servicesgraph_prop.buildSei()); + ResultsClient rclient = results_prop.getClient(); + + tracker.createJob(jobId); + tracker.setJobPercentComplete(jobId, 1); + + // spin up an async thread + new Thread(() -> { + try { + HeadersManager.setHeaders(headers); + ShaclRunner shaclRunner = new ShaclRunner(shaclTtlFile.getInputStream(), new SparqlConnection(connJsonStr), tracker, jobId, 20, 80); + if(tracker != null) { tracker.setJobPercentComplete(jobId, 85, "Gathering SHACL results"); } + JSONObject resultsJson = shaclRunner.getResults(severity); + rclient.execStoreBlobResults(jobId, resultsJson); + tracker.setJobSuccess(jobId); + } catch (Exception e) { + try { + LocalLogger.printStackTrace(e); + tracker.setJobFailure(jobId, e.getMessage()); + } catch (Exception ee) { + LocalLogger.logToStdErr(ENDPOINT_NAME + " error accessing job tracker"); + LocalLogger.printStackTrace(ee); + } + } + }).start(); + + retval.addJobId(jobId); + retval.addResultType(SparqlResultTypes.TABLE); + retval.setSuccess(true); + } catch (Exception e) { + retval.addRationaleMessage(SERVICE_NAME, ENDPOINT_NAME, e); + retval.setSuccess(false); + LocalLogger.printStackTrace(e); + } + return retval.toJson(); + } + /** * Determine if an EDC mnemonic exists in the services config */ diff --git a/utilityService/src/main/resources/application.properties b/utilityService/src/main/resources/application.properties index 13b49e64d..15a5df036 100644 --- a/utilityService/src/main/resources/application.properties +++ b/utilityService/src/main/resources/application.properties @@ -17,6 +17,9 @@ auth.refreshFreqSeconds=${AUTH_REFRESH_FREQ_SEC} auth.usernameKey=${AUTH_USERNAME_KEY} auth.groupKey=${AUTH_GROUP_KEY} +results.service.protocol=${RESULTS_SERVICE_PROTOCOL} +results.service.server=${RESULTS_SERVICE_HOST} +results.service.port=${PORT_SPARQLGRAPH_RESULTS_SERVICE} ngexec.service.protocol=${NODEGROUPEXECUTION_SERVICE_PROTOCOL} ngexec.service.server=${NODEGROUPEXECUTION_SERVICE_HOST} ngexec.service.port=${PORT_NODEGROUPEXECUTION_SERVICE}