diff --git a/content-api/content-actors/pom.xml b/content-api/content-actors/pom.xml index 45e1e286a..c0f084a7b 100644 --- a/content-api/content-actors/pom.xml +++ b/content-api/content-actors/pom.xml @@ -93,6 +93,19 @@ 2.5.22 test + + + org.sunbird + graph-dac + + + org.apache.commons + commons-lang3 + + + 1.0-SNAPSHOT + jar + diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala index 837810741..8151b7897 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala @@ -15,8 +15,9 @@ import org.sunbird.content.review.mgr.ReviewManager import org.sunbird.content.upload.mgr.UploadManager import org.sunbird.content.util._ import org.sunbird.graph.OntologyEngineContext -import org.sunbird.graph.dac.model.Node +import org.sunbird.graph.dac.model.{Node, Vertex} import org.sunbird.graph.nodes.DataNode +import org.sunbird.graph.vertex.DataVertex import org.sunbird.graph.utils.NodeUtil import org.sunbird.managers.HierarchyManager import org.sunbird.managers.HierarchyManager.hierarchyPrefix @@ -66,7 +67,7 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe def create(request: Request): Future[Response] = { populateDefaultersForCreation(request) RequestUtil.restrictProperties(request) - DataNode.create(request, dataModifier).map(node => { + DataVertex.create(request, vertexDataModifier).map(node => { ResponseHandler.OK.put(ContentConstants.IDENTIFIER, node.getIdentifier).put("node_id", node.getIdentifier) .put("versionKey", node.getMetadata.get("versionKey")) }) @@ -296,6 +297,20 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe node } + def vertexDataModifier(vertex: Vertex): Vertex = { + if (vertex.getMetadata.containsKey("trackable") && + vertex.getMetadata.getOrDefault("trackable", new java.util.HashMap[String, AnyRef]).asInstanceOf[java.util.Map[String, AnyRef]].containsKey("enabled") && + "Yes".equalsIgnoreCase(vertex.getMetadata.getOrDefault("trackable", new java.util.HashMap[String, AnyRef]).asInstanceOf[java.util.Map[String, AnyRef]].getOrDefault("enabled", "").asInstanceOf[String])) { + vertex.getMetadata.put("contentType", "Course") + } + + //TODO: Below fix to be reviewed when the fix for null to Stringify in ExternalStore.scala is implemented + if (vertex.getExternalData != null && vertex.getExternalData.containsKey("relational_metadata") && vertex.getExternalData.get("relational_metadata") == null) { + vertex.getExternalData.put("relational_metadata", "{}") + } + vertex + } + def getImportConfig(): ImportConfig = { val requiredProps = Platform.getStringList("import.required_props", java.util.Arrays.asList("name", "code", "mimeType", "contentType", "artifactUrl", "framework")).asScala.toList val validStages = Platform.getStringList("import.valid_stages", java.util.Arrays.asList("create", "upload", "review", "publish")).asScala.toList diff --git a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala index a1d672b9b..718ed8b0f 100644 --- a/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala +++ b/content-api/content-actors/src/main/scala/org/sunbird/content/actors/EventActor.scala @@ -6,7 +6,7 @@ import org.sunbird.common.dto.{Request, Response, ResponseHandler} import org.sunbird.common.exception.{ClientException, ResponseCode} import org.sunbird.content.util.ContentConstants import org.sunbird.graph.OntologyEngineContext -import org.sunbird.graph.dac.model.{Node, Relation} +import org.sunbird.graph.dac.model.{Node, Relation, Vertex} import org.sunbird.graph.nodes.DataNode import java.util @@ -79,4 +79,13 @@ class EventActor @Inject()(implicit oec: OntologyEngineContext, ss: StorageServi node } + override def vertexDataModifier(vertex: Vertex): Vertex = { + if (vertex.getMetadata.containsKey("trackable") && + vertex.getMetadata.getOrDefault("trackable", new java.util.HashMap[String, AnyRef]).asInstanceOf[java.util.Map[String, AnyRef]].containsKey("enabled") && + "Yes".equalsIgnoreCase(vertex.getMetadata.getOrDefault("trackable", new java.util.HashMap[String, AnyRef]).asInstanceOf[java.util.Map[String, AnyRef]].getOrDefault("enabled", "").asInstanceOf[String])) { + vertex.getMetadata.put("contentType", "Event") + } + vertex + } + } \ No newline at end of file diff --git a/ontology-engine/graph-common/src/main/java/org/sunbird/graph/common/enums/GraphDACParams.java b/ontology-engine/graph-common/src/main/java/org/sunbird/graph/common/enums/GraphDACParams.java index 230c04005..4b853c3cb 100644 --- a/ontology-engine/graph-common/src/main/java/org/sunbird/graph/common/enums/GraphDACParams.java +++ b/ontology-engine/graph-common/src/main/java/org/sunbird/graph/common/enums/GraphDACParams.java @@ -8,5 +8,5 @@ public enum GraphDACParams { MERGE, nodes, RETURN, keys, rootNode, nodeId, WHERE, startNodeId, endNodeId, relationType, startNodeIds, endNodeIds, collectionId, collection, indexProperty, taskId, input, getTags, searchCriteria, paramMap, cypherQuery, paramValueMap, queryStatementMap, SYS_INTERNAL_LAST_UPDATED_ON, - CONSUMER_ID, consumerId, CHANNEL_ID, channel, APP_ID, appId, Nodes_Count, Relations_Count; + CONSUMER_ID, consumerId, CHANNEL_ID, channel, APP_ID, appId, Nodes_Count, Relations_Count, vertex; } diff --git a/ontology-engine/graph-core_2.12/pom.xml b/ontology-engine/graph-core_2.12/pom.xml index 729c430e4..2f20c92b4 100644 --- a/ontology-engine/graph-core_2.12/pom.xml +++ b/ontology-engine/graph-core_2.12/pom.xml @@ -28,6 +28,18 @@ 1.0-SNAPSHOT jar + + org.sunbird + graph-dac + + + org.apache.commons + commons-lang3 + + + 1.0-SNAPSHOT + jar + org.sunbird schema-validator diff --git a/ontology-engine/graph-core_2.12/src/main/scala/org/sunbird/graph/GraphService.scala b/ontology-engine/graph-core_2.12/src/main/scala/org/sunbird/graph/GraphService.scala index e090a8591..412465d23 100644 --- a/ontology-engine/graph-core_2.12/src/main/scala/org/sunbird/graph/GraphService.scala +++ b/ontology-engine/graph-core_2.12/src/main/scala/org/sunbird/graph/GraphService.scala @@ -12,6 +12,7 @@ import java.lang import scala.concurrent.{ExecutionContext, Future} class GraphService { + implicit val ec: ExecutionContext = ExecutionContext.global val isrRelativePathEnabled: lang.Boolean = Platform.getBoolean("cloudstorage.metadata.replace_absolute_path", false) diff --git a/ontology-engine/graph-core_2.12/src/main/scala/org/sunbird/graph/JanusGraphService.scala b/ontology-engine/graph-core_2.12/src/main/scala/org/sunbird/graph/JanusGraphService.scala new file mode 100644 index 000000000..9f09b1f9f --- /dev/null +++ b/ontology-engine/graph-core_2.12/src/main/scala/org/sunbird/graph/JanusGraphService.scala @@ -0,0 +1,75 @@ +package org.sunbird.graph + +import org.sunbird.common.Platform +import org.sunbird.common.dto.{Property, Request, Response} +import org.sunbird.graph.dac.model.{Vertex, VertexSubGraph} +import org.sunbird.graph.util.CSPMetaUtil +import org.sunbird.janus.service.operation.{EdgeOperations, SearchOperations, VertexOperations} + +import java.lang +import scala.concurrent.{ExecutionContext, Future} +class JanusGraphService { + + private val VertexOperations = new VertexOperations() + private val EdgeOperations = new EdgeOperations() + private val SearchOperations = new SearchOperations() + + implicit val ec: ExecutionContext = ExecutionContext.global + val isrRelativePathEnabled: lang.Boolean = Platform.getBoolean("cloudstorage.metadata.replace_absolute_path", false) + + + def addVertex(graphId: String, vertex: Vertex): Future[Vertex] = { + if (isrRelativePathEnabled) { + val metadata = CSPMetaUtil.updateRelativePath(vertex.getMetadata) + vertex.setMetadata(metadata) + } + VertexOperations.addVertex(graphId, vertex).map(resVertex => if (isrRelativePathEnabled) CSPMetaUtil.updateAbsolutePath(resVertex) else resVertex) + + } + + def createEdges(graphId: String, edgeMap: java.util.List[java.util.Map[String, AnyRef]]): Future[Response] = { + EdgeOperations.createEdges(graphId, edgeMap) + } + + def removeEdges(graphId: String, edgeMap: java.util.List[java.util.Map[String, AnyRef]]): Future[Response] = { + EdgeOperations.removeEdges(graphId, edgeMap) + } + + def getNodeByUniqueId(graphId: String, vertexId: String, getTags: Boolean, request: Request): Future[Vertex] = { + SearchOperations.getNodeByUniqueId(graphId, vertexId, getTags, request).map(vertex => if (isrRelativePathEnabled) CSPMetaUtil.updateAbsolutePath(vertex) else vertex) + } + + def deleteNode(graphId: String, vertexId: String, request: Request): Future[java.lang.Boolean] = { + VertexOperations.deleteVertex(graphId, vertexId, request) + } + + def upsertVertex(graphId: String, vertex: Vertex, request: Request): Future[Vertex] = { + if (isrRelativePathEnabled) { + val metadata = CSPMetaUtil.updateRelativePath(vertex.getMetadata) + vertex.setMetadata(metadata) + } + VertexOperations.upsertVertex(graphId, vertex, request) + .map(resVertex => if (isrRelativePathEnabled) CSPMetaUtil.updateAbsolutePath(resVertex) else resVertex) + } + + def upsertRootNode(graphId: String, request: Request): Future[Vertex] = { + VertexOperations.upsertRootVertex(graphId, request) + } + + def updateVertexes(graphId: String, identifiers: java.util.List[String], metadata: java.util.Map[String, AnyRef]): Future[java.util.Map[String, Vertex]] = { + val updatedMetadata = if (isrRelativePathEnabled) CSPMetaUtil.updateRelativePath(metadata) else metadata + VertexOperations.updateVertexes(graphId, identifiers, updatedMetadata) + } + + def getNodeProperty(graphId: String, identifier: String, property: String): Future[Property] = { + SearchOperations.getNodeProperty(graphId, identifier, property).map(property => if (isrRelativePathEnabled) CSPMetaUtil.updateAbsolutePath(property) else property) + } + + def checkCyclicLoop(graphId: String, endNodeId: String, startNodeId: String, relationType: String) = { + SearchOperations.checkCyclicLoop(graphId, endNodeId, relationType, startNodeId) + } + + def getSubGraph(graphId: String, nodeId: String, depth: Int): Future[VertexSubGraph] = { + EdgeOperations.getSubGraph(graphId, nodeId, depth) + } +} \ No newline at end of file diff --git a/ontology-engine/graph-core_2.12/src/main/scala/org/sunbird/graph/OntologyEngineContext.scala b/ontology-engine/graph-core_2.12/src/main/scala/org/sunbird/graph/OntologyEngineContext.scala index 167eff5ad..106410c66 100644 --- a/ontology-engine/graph-core_2.12/src/main/scala/org/sunbird/graph/OntologyEngineContext.scala +++ b/ontology-engine/graph-core_2.12/src/main/scala/org/sunbird/graph/OntologyEngineContext.scala @@ -10,11 +10,15 @@ class OntologyEngineContext { private val dialGraphDB = new DialGraphService private val hUtil = new HttpUtil private lazy val kfClient = new KafkaClient - + private lazy val janusGraphDB = new JanusGraphService def graphService = { graphDB } + def janusGraphService = { + janusGraphDB + } + def dialgraphService = { dialGraphDB } diff --git a/ontology-engine/graph-core_2.12/src/main/scala/org/sunbird/graph/util/CSPMetaUtil.scala b/ontology-engine/graph-core_2.12/src/main/scala/org/sunbird/graph/util/CSPMetaUtil.scala index c8c4b51d3..d8771da22 100644 --- a/ontology-engine/graph-core_2.12/src/main/scala/org/sunbird/graph/util/CSPMetaUtil.scala +++ b/ontology-engine/graph-core_2.12/src/main/scala/org/sunbird/graph/util/CSPMetaUtil.scala @@ -1,13 +1,12 @@ package org.sunbird.graph.util import java.util - import org.apache.commons.collections4.MapUtils import org.apache.commons.lang3.StringUtils import org.slf4j.LoggerFactory import org.sunbird.common.dto.Property import org.sunbird.common.{JsonUtils, Platform} -import org.sunbird.graph.dac.model.Node +import org.sunbird.graph.dac.model.{Node, Vertex} import scala.collection.JavaConverters._ import scala.collection.immutable.Map @@ -43,6 +42,12 @@ object CSPMetaUtil { node } + def updateAbsolutePath(vertex: Vertex): Vertex = { + val metadata = updateAbsolutePath(vertex.getMetadata) + vertex.setMetadata(metadata) + vertex + } + def updateAbsolutePath(nodes: java.util.List[Node]): java.util.List[Node] = { nodes.asScala.toList.map(node => { updateAbsolutePath(node) diff --git a/ontology-engine/graph-dac-api/pom.xml b/ontology-engine/graph-dac-api/pom.xml index 0c51b8f68..2bb1e628a 100644 --- a/ontology-engine/graph-dac-api/pom.xml +++ b/ontology-engine/graph-dac-api/pom.xml @@ -86,6 +86,21 @@ 1.17.6 test + + org.apache.tinkerpop + gremlin-driver + 3.7.2 + + + org.janusgraph + janusgraph-core + 1.0.0 + + + org.janusgraph + janusgraph-inmemory + 1.0.0 + diff --git a/ontology-engine/graph-dac-api/src/main/java/org/sunbird/graph/dac/model/Edges.java b/ontology-engine/graph-dac-api/src/main/java/org/sunbird/graph/dac/model/Edges.java new file mode 100644 index 000000000..e2aabd6ce --- /dev/null +++ b/ontology-engine/graph-dac-api/src/main/java/org/sunbird/graph/dac/model/Edges.java @@ -0,0 +1,288 @@ +package org.sunbird.graph.dac.model; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.sunbird.common.exception.ServerException; +import org.sunbird.graph.common.enums.SystemProperties; +import org.sunbird.graph.dac.enums.GraphDACErrorCodes; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Edges implements Serializable { + + private static final long serialVersionUID = -7207054262120122453L; + private Object id; + private String graphId; + private String edgeType; + private String startVertexId; + private String endVertexId; + private String startVertexName; + private String endVertexName; + private String startVertexType; + private String endVertexType; + private String startVertexObjectType; + private String endVertexObjectType; + private Map metadata; + private Map startVertexMetadata; + private Map endVertexMetadata; + + public Edges() { + + } + + public Edges(String startVertexId, String edgeType, String endVertexId) { + this.startVertexId = startVertexId; + this.endVertexId = endVertexId; + this.edgeType = edgeType; + } + + public Edges(String graphId, Edge edge) { + if (null == edge) + throw new ServerException(GraphDACErrorCodes.ERR_GRAPH_NULL_DB_REL.name(), + "Failed to create relation object. Relation from database is null."); + this.graphId = graphId; + + Vertex startVertex = edge.inVertex(); + Vertex endVertex = edge.outVertex(); + this.startVertexId = (String) startVertex.property(SystemProperties.IL_UNIQUE_ID.name()).value(); + this.endVertexId = (String) endVertex.property(SystemProperties.IL_UNIQUE_ID.name()).value(); + this.startVertexName = getName(startVertex); + this.endVertexName = getName(endVertex); + this.startVertexType = getVertexType(startVertex); + this.endVertexType = getVertexType(endVertex); + this.startVertexObjectType = getObjectType(startVertex); + this.endVertexObjectType = getObjectType(endVertex); + this.edgeType = edge.label(); + this.metadata = new HashMap(); + this.startVertexMetadata = getNodeMetadata(edge.outVertex()); + this.endVertexMetadata = getNodeMetadata(edge.inVertex()); + edge.keys().forEach(key -> this.metadata.put(key, edge.value(key))); + } + + public Edges(String graphId, Edge edge, Map startNodeMap, Map endNodeMap) { + if (null == edge) + throw new ServerException(GraphDACErrorCodes.ERR_GRAPH_NULL_DB_REL.name(), + "Failed to create relation object. Relation from database is null."); + this.id = edge.id(); + this.graphId = graphId; + + Vertex startNode = (Vertex) startNodeMap.get(edge.outVertex().id()); + Vertex endNode = (Vertex) endNodeMap.get(edge.inVertex().id()); + + this.startVertexId = startNode.property(SystemProperties.IL_UNIQUE_ID.name()).value().toString(); + this.endVertexId = endNode.property(SystemProperties.IL_UNIQUE_ID.name()).value().toString(); + this.startVertexName = getName(startNode); + this.endVertexName = getName(endNode); + this.startVertexType = getVertexType(startNode); + this.endVertexType = getVertexType(endNode); + this.startVertexObjectType = getObjectType(startNode); + this.endVertexObjectType = getObjectType(endNode); + this.edgeType = edge.label(); + this.metadata = new HashMap(); + this.startVertexMetadata = getNodeMetadata(startNode); + this.endVertexMetadata = getNodeMetadata(endNode); + edge.keys().forEach(key -> { + Object value = edge.value(key); + if(null != value){ + if (value instanceof List) { + List list = (List) value; + if (!list.isEmpty()) { + Object obj = list.get(0); + if (obj instanceof String) { + this.metadata.put(key, list.toArray(new String[0])); + } else if (obj instanceof Number) { + this.metadata.put(key, list.toArray(new Number[0])); + } else if (obj instanceof Boolean) { + this.metadata.put(key, list.toArray(new Boolean[0])); + } else { + this.metadata.put(key, list.toArray(new Object[0])); + } + } + } else + this.metadata.put(key, value); + } + }); + } + + + private String getName(Vertex vertex) { + String name = vertex.property("name").isPresent() ? vertex.property("name").value().toString() : null; + if (StringUtils.isBlank(name)) { + name = vertex.property("title").isPresent() ? vertex.property("title").value().toString() : null; + if (StringUtils.isBlank(name)) { + name = vertex.property(SystemProperties.IL_FUNC_OBJECT_TYPE.name()).isPresent() ? vertex.property(SystemProperties.IL_FUNC_OBJECT_TYPE.name()).value().toString() : null; + if (StringUtils.isBlank(name)) + name = vertex.property(SystemProperties.IL_SYS_NODE_TYPE.name()).isPresent() ? vertex.property(SystemProperties.IL_SYS_NODE_TYPE.name()).value().toString() : null; + } + } + return name; + } + + + private String getVertexType(Vertex vertex) { + return vertex.property(SystemProperties.IL_SYS_NODE_TYPE.name()).isPresent() ? vertex.property(SystemProperties.IL_SYS_NODE_TYPE.name()).value().toString() : null; + } + + private String getObjectType(Vertex vertex) { + return vertex.property(SystemProperties.IL_FUNC_OBJECT_TYPE.name()).isPresent() ? vertex.property(SystemProperties.IL_FUNC_OBJECT_TYPE.name()).value().toString() : null; + } + + private Map getNodeMetadata(Vertex vertex) { + Map metadata = new HashMap<>(); + if (vertex != null) { + vertex.keys().forEach(key -> { + Object value = vertex.value(key); + if (value instanceof List) { + List list = (List) value; + if (!list.isEmpty()) { + Object firstElement = list.get(0); + if (firstElement instanceof String) { + metadata.put(key, list.toArray(new String[0])); + } else if (firstElement instanceof Number) { + metadata.put(key, list.toArray(new Number[0])); + } else if (firstElement instanceof Boolean) { + metadata.put(key, list.toArray(new Boolean[0])); + } else { + metadata.put(key, list.toArray(new Object[0])); + } + } + } else { + metadata.put(key, value); + } + }); + } + return metadata; + } + + public String getEdgeType() { + return edgeType; + } + + public void setEdgeType(String edgeType) { + this.edgeType = edgeType; + } + + public String getStartVertexId() { + return startVertexId; + } + + public void setStartVertexId(String startVertexId) { + this.startVertexId = startVertexId; + } + + public String getEndVertexId() { + return endVertexId; + } + + public void setEndVertexId(String endVertexId) { + this.endVertexId = endVertexId; + } + + public Map getMetadata() { + if (!MapUtils.isEmpty(metadata)) + return metadata; + else + return new HashMap(); + } + + public Edges updateMetadata(Map metadata) { + if (!MapUtils.isEmpty(metadata)) + this.metadata = metadata; + return this; + } + + public void setMetadata(Map metadata) { + this.metadata = metadata; + } + + public String getGraphId() { + return graphId; + } + + public void setGraphId(String graphId) { + this.graphId = graphId; + } + + public Object getId() { + return id; + } + + public void setId(Object id) { + this.id = id; + } + + public String getStartVertexName() { + return startVertexName; + } + + public void setStartVertexName(String startVertexName) { + this.startVertexName = startVertexName; + } + + public String getEndVertexName() { + return endVertexName; + } + + public void setEndVertexName(String endVertexName) { + this.endVertexName = endVertexName; + } + + public String getStartVertexType() { + return startVertexType; + } + + public void setStartVertexType(String startVertexType) { + this.startVertexType = startVertexType; + } + + public String getEndVertexType() { + return endVertexType; + } + + public void setEndVertexType(String endVertexType) { + this.endVertexType = endVertexType; + } + + public String getStartVertexObjectType() { + return startVertexObjectType; + } + + public void setStartVertexObjectType(String startVertexObjectType) { + this.startVertexObjectType = startVertexObjectType; + } + + public String getEndVertexObjectType() { + return endVertexObjectType; + } + + public void setEndVertexObjectType(String endVertexObjectType) { + this.endVertexObjectType = endVertexObjectType; + } + + @JsonIgnore + public Map getStartVertexMetadata() { + return startVertexMetadata; + } + + @JsonIgnore + public void setStartVertexMetadata(Map startVertexMetadata) { + this.startVertexMetadata = startVertexMetadata; + } + + @JsonIgnore + public Map getEndVertexMetadata() { + return endVertexMetadata; + } + + @JsonIgnore + public void setEndVertexMetadata(Map endVertexMetadata) { + this.endVertexMetadata = endVertexMetadata; + } + +} diff --git a/ontology-engine/graph-dac-api/src/main/java/org/sunbird/graph/dac/model/Vertex.java b/ontology-engine/graph-dac-api/src/main/java/org/sunbird/graph/dac/model/Vertex.java new file mode 100644 index 000000000..ab211e4cf --- /dev/null +++ b/ontology-engine/graph-dac-api/src/main/java/org/sunbird/graph/dac/model/Vertex.java @@ -0,0 +1,180 @@ +package org.sunbird.graph.dac.model; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.sunbird.graph.common.enums.SystemProperties; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class Vertex implements Serializable { + + private static final long serialVersionUID = 252337826576516976L; + + private Object id; + private String graphId; + private String identifier; + private String vertexType; + private String objectType; + private Map metadata; + private List outEdges; + private List inEdges; + private List addedEdges; + private List deletedEdges; + private Map edgeVertices; + private Map externalData; + + public Vertex() { + addedEdges = new ArrayList<>(); + deletedEdges = new ArrayList<>(); + } + + public Vertex(String identifier, String vertexType, String objectType) { + this.identifier = identifier; + this.vertexType = vertexType; + this.objectType = objectType; + addedEdges = new ArrayList<>(); + deletedEdges = new ArrayList<>(); + } + + public Vertex(String graphId, Map metadata) { + this.graphId = graphId; + this.metadata = metadata; + if (null != metadata && !metadata.isEmpty()) { + if (null != metadata.get(SystemProperties.IL_UNIQUE_ID.name())) + this.identifier = metadata.get(SystemProperties.IL_UNIQUE_ID.name()).toString(); + if (null != metadata.get(SystemProperties.IL_SYS_NODE_TYPE.name())) + this.vertexType = metadata.get(SystemProperties.IL_SYS_NODE_TYPE.name()).toString(); + if (null != metadata.get(SystemProperties.IL_FUNC_OBJECT_TYPE.name())) + this.objectType = metadata.get(SystemProperties.IL_FUNC_OBJECT_TYPE.name()).toString(); + } + addedEdges = new ArrayList<>(); + deletedEdges = new ArrayList<>(); + } + + + public Object getId() { + return id; + } + + public void setId(Object id) { + this.id = id; + } + + @JsonIgnore + public String getGraphId() { + return graphId; + } + + public void setGraphId(String graphId) { + this.graphId = graphId; + } + + public String getIdentifier() { + if (StringUtils.isBlank(identifier) && null != metadata) + this.identifier = (String) metadata.get(SystemProperties.IL_UNIQUE_ID.name()); + return identifier; + } + + public void setIdentifier(String identifier) { + this.identifier = identifier; + } + + public String getVertexType() { + if (StringUtils.isBlank(vertexType) && null != metadata) + this.vertexType = (String) metadata.get(SystemProperties.IL_SYS_NODE_TYPE.name()); + return vertexType; + } + + public void setVertexType(String vertexType) { + this.vertexType = vertexType; + } + + public String getObjectType() { + if (StringUtils.isBlank(objectType) && null != metadata) + this.objectType = (String) metadata.get(SystemProperties.IL_FUNC_OBJECT_TYPE.name()); + return objectType; + } + + public void setObjectType(String objectType) { + this.objectType = objectType; + } + + public Map getMetadata() { + return metadata; + } + + public void setMetadata(Map metadata) { + this.metadata = metadata; + } + + public List getOutEdges() { + if (!CollectionUtils.isEmpty(outEdges)) + return outEdges; + else return new ArrayList<>(); + } + + public void setOutEdges(List outEdges) { + this.outEdges = outEdges; + } + + public List getInEdges() { + if (!CollectionUtils.isEmpty(inEdges)) + return inEdges; + else return new ArrayList<>(); + } + + public void setInEdges(List inEdges) { + this.inEdges = inEdges; + } + + public List getAddedEdges() { + return addedEdges; + } + + public void setAddedEdges(List addedEdges) { + if(CollectionUtils.isEmpty(this.addedEdges)) + this.addedEdges = new ArrayList<>(); + this.addedEdges.addAll(addedEdges); + } + + public List getDeletedEdges() { + return deletedEdges; + } + + public void setDeletedEdges(List deletedEdges) { + this.deletedEdges = deletedEdges; + } + + public Map getExternalData() { + return externalData; + } + + public Map getEdgeVertices() { + return edgeVertices; + } + + public void setEdgeVertices(Map edgeVertices) { + this.edgeVertices = edgeVertices; + } + + public void setExternalData(Map externalData) { + this.externalData = externalData; + } + + public Vertex getVertex() { + return (Vertex) this; + } + + public Vertex getRelationNode(String identifier) { + return edgeVertices.get(identifier); + } + + public String getArtifactUrl() { + return (String) this.metadata.getOrDefault("artifactUrl", ""); + } + +} diff --git a/ontology-engine/graph-dac-api/src/main/java/org/sunbird/graph/dac/model/VertexSubGraph.java b/ontology-engine/graph-dac-api/src/main/java/org/sunbird/graph/dac/model/VertexSubGraph.java new file mode 100644 index 000000000..5df0ed7cd --- /dev/null +++ b/ontology-engine/graph-dac-api/src/main/java/org/sunbird/graph/dac/model/VertexSubGraph.java @@ -0,0 +1,28 @@ +package org.sunbird.graph.dac.model; + +import java.util.List; +import java.util.Map; + +public class VertexSubGraph { + private Map vertexs; + private List edges; + + public VertexSubGraph(Map vertexs, List edges) { + this.vertexs = vertexs; + this.edges = edges; + } + + public Map getVertexs() { return vertexs; } + + public void setVertexs(Map vertexs) { + this.vertexs = vertexs; + } + + public List getEdges() { + return edges; + } + + public void setEdges(List edges) { + this.edges = edges; + } +} diff --git a/ontology-engine/graph-dac/pom.xml b/ontology-engine/graph-dac/pom.xml new file mode 100644 index 000000000..c1e8a963e --- /dev/null +++ b/ontology-engine/graph-dac/pom.xml @@ -0,0 +1,150 @@ + + 4.0.0 + + ontology-engine + org.sunbird + 1.0-SNAPSHOT + + graph-dac + + + org.scala-lang + scala-library + ${scala.version} + + + org.sunbird + graph-dac-api + + + org.apache.commons + commons-lang3 + + + 1.0-SNAPSHOT + jar + + + org.sunbird + graph-common + 1.0-SNAPSHOT + jar + + + io.netty + netty-codec + 4.1.68.Final + + + org.scala-lang.modules + scala-java8-compat_${scala.maj.version} + 0.9.0 + + + org.powermock + powermock-api-mockito + 1.7.4 + test + + + org.powermock + powermock-module-junit4 + 1.7.4 + test + + + org.testcontainers + testcontainers + 1.17.6 + test + + + org.janusgraph + janusgraph-driver + 1.0.0 + + + org.janusgraph + janusgraph-core + 1.0.0 + + + org.apache.tinkerpop + gremlin-driver + 3.7.2 + + + org.janusgraph + janusgraph-inmemory + 1.0.0 + + + org.janusgraph + janusgraph-cql + 1.0.0 + + + org.apache.commons + commons-text + 1.9 + + + + + src/main/scala + + + net.alchim31.maven + scala-maven-plugin + 3.2.2 + + ${scala.version} + false + + + + scala-compile-first + process-resources + + add-source + compile + + + + scala-test-compile + process-test-resources + + testCompile + + + + + + org.scalatest + scalatest-maven-plugin + 2.0.0 + + + test + test + + test + + + + + + org.scoverage + scoverage-maven-plugin + ${scoverage.plugin.version} + + ${scala.version} + true + true + + + + + + \ No newline at end of file diff --git a/ontology-engine/graph-dac/src/main/conf/janusgraph-inmemory.properties b/ontology-engine/graph-dac/src/main/conf/janusgraph-inmemory.properties new file mode 100644 index 000000000..affaf236e --- /dev/null +++ b/ontology-engine/graph-dac/src/main/conf/janusgraph-inmemory.properties @@ -0,0 +1,35 @@ +# Copyright 2020 JanusGraph Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# JanusGraph configuration sample: in-memory +# +# This file connects to an in-memory storage backend + +# The implementation of graph factory that will be used by gremlin server +# +# Default: org.janusgraph.core.JanusGraphFactory +# Data Type: String +# Mutability: LOCAL +gremlin.graph=org.janusgraph.core.JanusGraphFactory + +# The primary persistence provider used by JanusGraph. This is required. +# It should be set one of JanusGraph's built-in shorthand names for its +# standard storage backends (shorthands: berkeleyje, cql, hbase, inmemory, +# scylla) or to the full package and classname of a custom/third-party +# StoreManager implementation. +# +# Default: (no default value) +# Data Type: String +# Mutability: LOCAL +storage.backend=inmemory diff --git a/ontology-engine/graph-dac/src/main/conf/remote-graph.properties b/ontology-engine/graph-dac/src/main/conf/remote-graph.properties new file mode 100644 index 000000000..74b4d6c0c --- /dev/null +++ b/ontology-engine/graph-dac/src/main/conf/remote-graph.properties @@ -0,0 +1,3 @@ +gremlin.remote.remoteConnectionClass=org.apache.tinkerpop.gremlin.driver.remote.DriverRemoteConnection +gremlin.remote.driver.clusterFile=/Users/admin/Documents/workspace/knowledge-platform/ontology-engine/graph-dac/src/main/conf/remote-objects.yaml +gremlin.remote.driver.sourceName=g \ No newline at end of file diff --git a/ontology-engine/graph-dac/src/main/conf/remote-objects.yaml b/ontology-engine/graph-dac/src/main/conf/remote-objects.yaml new file mode 100644 index 000000000..8aecad8d0 --- /dev/null +++ b/ontology-engine/graph-dac/src/main/conf/remote-objects.yaml @@ -0,0 +1,5 @@ +hosts: [localhost] +port: 8182 +serializer: { + className: org.apache.tinkerpop.gremlin.util.ser.GraphBinaryMessageSerializerV1, + config: { ioRegistries: [org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistry] }} \ No newline at end of file diff --git a/ontology-engine/graph-dac/src/main/scala/org/sunbird/janus/dac/util/GremlinVertexUtil.scala b/ontology-engine/graph-dac/src/main/scala/org/sunbird/janus/dac/util/GremlinVertexUtil.scala new file mode 100644 index 000000000..cd0e668ec --- /dev/null +++ b/ontology-engine/graph-dac/src/main/scala/org/sunbird/janus/dac/util/GremlinVertexUtil.scala @@ -0,0 +1,73 @@ +package org.sunbird.janus.dac.util + +import org.apache.commons.lang3.StringUtils +import org.sunbird.graph.dac.model.{Edges, Vertex} +import org.sunbird.common.exception.ServerException +import org.sunbird.graph.common.enums.SystemProperties +import org.sunbird.graph.dac.enums.GraphDACErrorCodes + +import java.{lang, util} + +class GremlinVertexUtil { + + def getNode(graphId: String, gremlinVertex: org.apache.tinkerpop.gremlin.structure.Vertex, edgeMap: util.Map[Object, AnyRef], + startNodeMap: util.Map[Object, AnyRef], endNodeMap: util.Map[Object, AnyRef]): Vertex = { + if (null == gremlinVertex) + throw new ServerException(GraphDACErrorCodes.ERR_GRAPH_NULL_DB_NODE.name(), + "Failed to create node object. Node from database is null.") + + val vertex: Vertex = new Vertex() + vertex.setGraphId(graphId) + vertex.setId(gremlinVertex.id()) + + val metadata = new util.HashMap[String, Object]() + gremlinVertex.keys().forEach { key => + if (StringUtils.equalsIgnoreCase(key, SystemProperties.IL_UNIQUE_ID.name())) + vertex.setIdentifier(gremlinVertex.values(key).next().asInstanceOf[String]) + else if (StringUtils.equalsIgnoreCase(key, SystemProperties.IL_SYS_NODE_TYPE.name())) + vertex.setVertexType(gremlinVertex.values(key).next().asInstanceOf[String]) + else if (StringUtils.equalsIgnoreCase(key, SystemProperties.IL_FUNC_OBJECT_TYPE.name())) + vertex.setObjectType(gremlinVertex.values(key).next().asInstanceOf[String]) + else { + val value = gremlinVertex.values(key) + if (null != value) { + value match { + case list: util.List[_] => + if (null != list && list.size() > 0) { + + val obj = list.get(0) + obj match { + case _: String => metadata.put(key, list.toArray(new Array[String](list.size()))) + case _: Number => metadata.put(key, list.toArray(new Array[Number](list.size()))) + case _: lang.Boolean => metadata.put(key, list.toArray(new Array[lang.Boolean](list.size()))) + case _ => metadata.put(key, list.toArray(new Array[AnyRef](list.size()))) + } + + } + case _ => metadata.put(key, value.next()) + } + } + } + } + vertex.setMetadata(metadata) + + if (null != edgeMap && !edgeMap.isEmpty && null != startNodeMap && !startNodeMap.isEmpty && null != endNodeMap && !endNodeMap.isEmpty) { + val inEdges = new util.ArrayList[Edges]() + val outEdges = new util.ArrayList[Edges]() + + edgeMap.forEach { (id, rel) => + val edge = rel.asInstanceOf[org.apache.tinkerpop.gremlin.structure.Edge] + if (edge.inVertex().id() == gremlinVertex.id()) { + outEdges.add(new Edges(graphId, edge, startNodeMap, endNodeMap)) + } + if (edge.outVertex().id() == gremlinVertex.id()) { + inEdges.add(new Edges(graphId, edge, startNodeMap, endNodeMap)) + } + } + vertex.setInEdges(inEdges) + vertex.setOutEdges(outEdges) + } + + vertex + } +} diff --git a/ontology-engine/graph-dac/src/main/scala/org/sunbird/janus/service/operation/EdgeOperations.scala b/ontology-engine/graph-dac/src/main/scala/org/sunbird/janus/service/operation/EdgeOperations.scala new file mode 100644 index 000000000..85b6a6be9 --- /dev/null +++ b/ontology-engine/graph-dac/src/main/scala/org/sunbird/janus/service/operation/EdgeOperations.scala @@ -0,0 +1,173 @@ +package org.sunbird.janus.service.operation + +import org.apache.commons.collections4.CollectionUtils +import org.apache.commons.lang3.StringUtils +import org.apache.tinkerpop.gremlin.groovy.jsr223.dsl.credential.__ +import org.apache.tinkerpop.gremlin.groovy.jsr223.dsl.credential.__.{bothE, outE} +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource +import org.apache.tinkerpop.gremlin.structure.{Edge, Graph, Vertex} +import org.janusgraph.core.JanusGraph +import org.sunbird.common.dto.{Response, ResponseHandler} +import org.sunbird.common.exception.ClientException +import org.sunbird.graph.common.enums.SystemProperties +import org.sunbird.graph.dac.model.{Edges, Node, VertexSubGraph} +import org.sunbird.graph.service.common.{CypherQueryConfigurationConstants, DACErrorCodeConstants, DACErrorMessageConstants} +import org.sunbird.janus.dac.util.GremlinVertexUtil +import org.sunbird.janus.service.util.JanusConnectionUtil +import org.sunbird.telemetry.logger.TelemetryManager + +import java.util +import java.util.{HashMap, List, Map, Set} +import java.util.stream.Collectors +import scala.collection.JavaConverters.{asJavaIterableConverter, asScalaBufferConverter, asScalaIteratorConverter, mapAsScalaMapConverter} +import scala.collection.convert.ImplicitConversions.{`collection AsScalaIterable`, `map AsScala`} +import scala.collection.immutable.HashSet +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future +import scala.collection.JavaConverters._ + +class EdgeOperations { + + val graphConnection = new JanusConnectionUtil + val gremlinVertexUtil = new GremlinVertexUtil + def createEdges(graphId: String, edgeData: util.List[util.Map[String, AnyRef]]): Future[Response] = { + Future{ + + if (StringUtils.isBlank(graphId)) + throw new ClientException(DACErrorCodeConstants.INVALID_GRAPH.name, + DACErrorMessageConstants.INVALID_GRAPH_ID + " | [Create Node Operation Failed.]") + + if (CollectionUtils.isEmpty(edgeData)) + throw new ClientException(DACErrorCodeConstants.INVALID_RELATION.name, + DACErrorMessageConstants.INVALID_NODE + " | [Create Relation Operation Failed.]") + + graphConnection.initialiseGraphClient() + val g: GraphTraversalSource = graphConnection.getGraphTraversalSource + + createBulkRelations(g, graphId, edgeData) + ResponseHandler.OK() + } + } + + def removeEdges(graphId: String, edgeData: util.List[util.Map[String, AnyRef]]): Future[Response] = { + Future { + if (StringUtils.isBlank(graphId)) + throw new ClientException(DACErrorCodeConstants.INVALID_GRAPH.name, + DACErrorMessageConstants.INVALID_GRAPH_ID + " | [Create Node Operation Failed.]") + + if (CollectionUtils.isEmpty(edgeData)) + throw new ClientException(DACErrorCodeConstants.INVALID_RELATION.name, + DACErrorMessageConstants.INVALID_NODE + " | [Create Relation Operation Failed.]") + + graphConnection.initialiseGraphClient() + val g: GraphTraversalSource = graphConnection.getGraphTraversalSource + + deleteBulkRelations(g, graphId, edgeData) + ResponseHandler.OK() + } + } + + def createBulkRelations(g: GraphTraversalSource, graphId: String, edgeData: util.List[util.Map[String, AnyRef]]): Unit = { + for (row <- edgeData.asScala.distinct) { + val startNodeId = row.get("startNodeId").toString + val endNodeId = row.get("endNodeId").toString + val relation = row.get("relation").toString + val relMetadata = row.get("relMetadata").asInstanceOf[util.Map[AnyRef, AnyRef]] + val startNode: Vertex = g.V().hasLabel(graphId).has(SystemProperties.IL_UNIQUE_ID.name, startNodeId).next() + val endNode: Vertex = g.V().hasLabel(graphId).has(SystemProperties.IL_UNIQUE_ID.name, endNodeId).next() + + val edgeTraversal = g.V(startNode.id()).as("a").addE(relation).to(__.V(endNode.id()).as("b")) + for ((key, value) <- relMetadata) { + edgeTraversal.property(key, value) + } + val addedEdge = edgeTraversal.next() + } + + } + + private def deleteBulkRelations(g: GraphTraversalSource, graphId: String, edgeData: util.List[util.Map[String, AnyRef]]): Unit = { + for (row <- edgeData.asScala) { + val startNodeId = row.get("startNodeId").toString + val endNodeId = row.get("endNodeId").toString + val relation = row.get("relation").toString + + g.V().hasLabel(graphId).has(SystemProperties.IL_UNIQUE_ID.name, startNodeId) + .outE().as(CypherQueryConfigurationConstants.DEFAULT_CYPHER_RELATION_OBJECT).inV().has(SystemProperties.IL_UNIQUE_ID.name, endNodeId) + .select(CypherQueryConfigurationConstants.DEFAULT_CYPHER_RELATION_OBJECT).drop().iterate() + + } + } + + def getSubGraph(graphId: String, nodeId: String, depth: Integer) = { + if (StringUtils.isBlank(graphId)) + throw new ClientException(DACErrorCodeConstants.INVALID_GRAPH.name, DACErrorMessageConstants.INVALID_GRAPH_ID + " | [Get SubGraph Operation Failed.]") + if (StringUtils.isBlank(nodeId)) + throw new ClientException(DACErrorCodeConstants.INVALID_IDENTIFIER.name, DACErrorMessageConstants.INVALID_IDENTIFIER + " | [Please Provide Node Identifier.]") + var effectiveDepth:Integer = if (depth == null) 5 else depth + + graphConnection.initialiseGraphClient() + val g: GraphTraversalSource = graphConnection.getGraphTraversalSource + TelemetryManager.log("Driver Initialised. | [Graph Id: " + graphId + "]") + + val relationMap = new util.HashMap[Object, AnyRef]() + var nodes = new util.HashSet[org.sunbird.graph.dac.model.Vertex] + var relations = new util.HashSet[org.sunbird.graph.dac.model.Edges] + val startNodeMap = new util.HashMap[Object, AnyRef] + val endNodeMap = new util.HashMap[Object, AnyRef] + + var results = g.V().hasLabel(graphId).has(SystemProperties.IL_UNIQUE_ID.name, nodeId).as("n") + .emit().repeat(outE().inV().simplePath()).times(5).as("m") + .outE().as("r2").inV().as("l") + .select("n", "m", "r2", "l") + .dedup() + .project("relationName", "relationMetadata", "startNode", "endNode") + .by(__.select("r2").label()) + .by(__.select("r2").elementMap()) + .by(__.select("m")) + .by(__.select("l")) + .dedup("startNode", "endNode") + .toList() + + + for (result <- results.asScala) { + val startNode = result.get("startNode").asInstanceOf[Vertex] + val endNode = result.get("endNode").asInstanceOf[Vertex] + val relationName = result.get("relationName").toString + val relationMetadata = result.get("relationMetadata").asInstanceOf[util.Map[String, Object]] + + nodes.add(gremlinVertexUtil.getNode(graphId, startNode, relationMap, startNodeMap, endNodeMap)) + nodes.add(gremlinVertexUtil.getNode(graphId, endNode, relationMap, startNodeMap, endNodeMap)) + + // Relation Metadata + val relData = new Edges( + startNode.property(SystemProperties.IL_UNIQUE_ID.name).value().toString, + relationName, + endNode.property(SystemProperties.IL_UNIQUE_ID.name).value().toString + ) + relData.setMetadata(relationMetadata) + relData.setStartVertexObjectType(startNode.property(SystemProperties.IL_FUNC_OBJECT_TYPE.name).value().toString) + relData.setStartVertexName(startNode.property("name").value().toString) + relData.setStartVertexType(startNode.property(SystemProperties.IL_SYS_NODE_TYPE.name).value().toString) + relData.setEndVertexObjectType(endNode.property(SystemProperties.IL_FUNC_OBJECT_TYPE.name).value().toString) + relData.setEndVertexName(endNode.property("name").value().toString) + relData.setEndVertexType(endNode.property(SystemProperties.IL_SYS_NODE_TYPE.name).value().toString) + relations.add(relData) + } + + // Group nodes by their identifier and get the first node for each identifier + val uniqNodes = nodes.groupBy(_.getIdentifier).mapValues(_.head).values.toSet + + // Create a map with the node identifier as the key and the node itself as the value + val nodeMap= uniqNodes.map(node => node.getIdentifier -> node).toMap + val relationsList= relations.toList + + // Convert Scala collections to Java collections + val javaNodeMap: java.util.Map[String, org.sunbird.graph.dac.model.Vertex] = nodeMap.asJava + val javaRelationsList: java.util.List[org.sunbird.graph.dac.model.Edges] = relationsList.asJava + + // Create a VertexSubGraph instance + Future { + new VertexSubGraph(javaNodeMap, javaRelationsList) + } + } +} diff --git a/ontology-engine/graph-dac/src/main/scala/org/sunbird/janus/service/operation/SearchOperations.scala b/ontology-engine/graph-dac/src/main/scala/org/sunbird/janus/service/operation/SearchOperations.scala new file mode 100644 index 000000000..181bcbb59 --- /dev/null +++ b/ontology-engine/graph-dac/src/main/scala/org/sunbird/janus/service/operation/SearchOperations.scala @@ -0,0 +1,327 @@ +package org.sunbird.janus.service.operation + +import org.apache.commons.collections4.CollectionUtils +import org.apache.commons.lang3.StringUtils +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.{GraphTraversal, GraphTraversalSource} +import org.sunbird.janus.dac.util.GremlinVertexUtil +import org.apache.tinkerpop.gremlin.groovy.jsr223.dsl.credential.__._ +import org.sunbird.graph.dac.model.Vertex +import org.apache.tinkerpop.gremlin.structure.Edge +import org.sunbird.common.dto.{Property, Request} +import org.sunbird.common.exception.{ClientException, MiddlewareException, ResourceNotFoundException, ServerException} +import org.sunbird.graph.common.enums.{GraphDACParams, SystemProperties} +import org.sunbird.graph.service.common.{CypherQueryConfigurationConstants, DACErrorCodeConstants, DACErrorMessageConstants} +import org.sunbird.janus.service.util.JanusConnectionUtil +import org.sunbird.telemetry.logger.TelemetryManager + +import java.util +import scala.concurrent.{ExecutionContext, Future} +import ExecutionContext.Implicits.global +import scala.collection.JavaConverters.{asJavaIterableConverter, asScalaBufferConverter} + + +class SearchOperations { + + val graphConnection = new JanusConnectionUtil + val gremlinVertexUtil = new GremlinVertexUtil + + def getNodeByUniqueId(graphId: String, vertexId: String, getTags: Boolean, request: Request): Future[Vertex] = { + Future { + TelemetryManager.log("Graph Id: " + graphId + "\nVertex Id: " + vertexId + "\nGet Tags:" + getTags) + + if (StringUtils.isBlank(graphId)) + throw new ClientException(DACErrorCodeConstants.INVALID_GRAPH.name, + DACErrorMessageConstants.INVALID_GRAPH_ID + " | ['Get Node By Unique Id' Operation Failed.]") + + if (StringUtils.isBlank(vertexId)) + throw new ClientException(DACErrorCodeConstants.INVALID_IDENTIFIER.name, + DACErrorMessageConstants.INVALID_IDENTIFIER + " | ['Get Node By Unique Id' Operation Failed.]") + + TelemetryManager.log("Driver Initialised. | [Graph Id: " + graphId + "]") + try { + graphConnection.initialiseGraphClient() + val g: GraphTraversalSource = graphConnection.getGraphTraversalSource + + val parameterMap = new util.HashMap[String, AnyRef] + parameterMap.put(GraphDACParams.graphId.name, graphId) + parameterMap.put(GraphDACParams.nodeId.name, vertexId) + parameterMap.put(GraphDACParams.getTags.name, getTags.asInstanceOf[java.lang.Boolean]) + parameterMap.put(GraphDACParams.request.name, request) + + val retrievedVertices = getVertexByUniqueId(parameterMap, g) + var newVertex: Vertex = null + if (CollectionUtils.isEmpty(retrievedVertices)) + throw new ResourceNotFoundException(DACErrorCodeConstants.NOT_FOUND.name, + DACErrorMessageConstants.NODE_NOT_FOUND + " | [Invalid Node Id.]: " + vertexId, vertexId) + + val vertexMap = new util.HashMap[Object, AnyRef] + val relationMap = new util.HashMap[Object, AnyRef] + val startNodeMap = new util.HashMap[Object, AnyRef] + val endNodeMap = new util.HashMap[Object, AnyRef] + + retrievedVertices.forEach { result => + if (null != result) + getRecordValues(result, vertexMap, relationMap, startNodeMap, endNodeMap) + } + + if (!vertexMap.isEmpty) { + val entry = vertexMap.entrySet().iterator().next() + newVertex = gremlinVertexUtil.getNode(graphId, entry.getValue.asInstanceOf[org.apache.tinkerpop.gremlin.structure.Vertex], relationMap, startNodeMap, endNodeMap) + } + newVertex + } + catch { + case ex: MiddlewareException => throw ex + case e: Throwable => + e.printStackTrace() + throw new ServerException(DACErrorCodeConstants.CONNECTION_PROBLEM.name(), + DACErrorMessageConstants.CONNECTION_PROBLEM + " | " + e.getMessage, e) + } + } + } + + def getNodeProperty(graphId: String, vertexId: String, key: String): Future[Property] = { + Future { + TelemetryManager.log("Graph Id: " + graphId + "\nNode Id: " + vertexId + "\nProperty (Key): " + key) + + if (StringUtils.isBlank(graphId)) + throw new ClientException(DACErrorCodeConstants.INVALID_GRAPH.name, + DACErrorMessageConstants.INVALID_GRAPH_ID + " | ['Get Node Property' Operation Failed.]") + + if (StringUtils.isBlank(vertexId)) + throw new ClientException(DACErrorCodeConstants.INVALID_IDENTIFIER.name, + DACErrorMessageConstants.INVALID_IDENTIFIER + " | ['Get Node Property' Operation Failed.]") + + if (StringUtils.isBlank(key)) + throw new ClientException(DACErrorCodeConstants.INVALID_PROPERTY.name, + DACErrorMessageConstants.INVALID_PROPERTY_KEY + " | ['Get Node Property' Operation Failed.]") + + val property = new Property() + try { + graphConnection.initialiseGraphClient() + val g: GraphTraversalSource = graphConnection.getGraphTraversalSource + + val parameterMap = new util.HashMap[String, AnyRef] + parameterMap.put(GraphDACParams.graphId.name, graphId) + parameterMap.put(GraphDACParams.nodeId.name, vertexId) + parameterMap.put(GraphDACParams.key.name, key) + + val nodeProperty = executeGetNodeProperty(parameterMap, g) + val elementMap = nodeProperty.elementMap().next() + if (null != elementMap && null != elementMap.get(key)){ + property.setPropertyName(key) + property.setPropertyValue(elementMap.get(key)) + } + property + } + catch { + case e: Throwable => + e.printStackTrace() + e.getCause match { + case _: NoSuchElementException | _: ResourceNotFoundException => + throw new ResourceNotFoundException(DACErrorCodeConstants.NOT_FOUND.name, DACErrorMessageConstants.NODE_NOT_FOUND + " | [Invalid Node Id.]: " + vertexId, vertexId) + case _ => + throw new ServerException(DACErrorCodeConstants.CONNECTION_PROBLEM.name, DACErrorMessageConstants.CONNECTION_PROBLEM + " | " + e.getMessage, e) + } + } + } + + } + + def checkCyclicLoop(graphId: String, startNodeId: String, relationType: String, endNodeId: String): util.Map[String, AnyRef] = { + if (StringUtils.isBlank(graphId)) + throw new ClientException(DACErrorCodeConstants.INVALID_GRAPH.name, + DACErrorMessageConstants.INVALID_GRAPH_ID + " | ['Check Cyclic Loop' Operation Failed.]") + + if (StringUtils.isBlank(startNodeId)) + throw new ClientException(DACErrorCodeConstants.INVALID_IDENTIFIER.name, + DACErrorMessageConstants.INVALID_START_NODE_ID + " | ['Check Cyclic Loop' Operation Failed.]") + + if (StringUtils.isBlank(relationType)) + throw new ClientException(DACErrorCodeConstants.INVALID_RELATION.name, + DACErrorMessageConstants.INVALID_RELATION_TYPE + " | ['Check Cyclic Loop' Operation Failed.]") + + if (StringUtils.isBlank(endNodeId)) + throw new ClientException(DACErrorCodeConstants.INVALID_IDENTIFIER.name, + DACErrorMessageConstants.INVALID_END_NODE_ID + " | ['Check Cyclic Loop' Operation Failed.]") + + val cyclicLoopMap = new util.HashMap[String, AnyRef] + + try { + graphConnection.initialiseGraphClient() + val g: GraphTraversalSource = graphConnection.getGraphTraversalSource + + val parameterMap = new util.HashMap[String, AnyRef] + parameterMap.put(GraphDACParams.graphId.name, graphId) + parameterMap.put(GraphDACParams.startNodeId.name, startNodeId) + parameterMap.put(GraphDACParams.relationType.name, relationType) + parameterMap.put(GraphDACParams.endNodeId.name, endNodeId) + + val result = generateCheckCyclicLoopTraversal(parameterMap, g) + if (null != result && result.hasNext) { + cyclicLoopMap.put(GraphDACParams.loop.name, new Boolean(true)) + cyclicLoopMap.put(GraphDACParams.message.name, startNodeId + " and " + endNodeId + " are connected by relation: " + relationType) + } + else + cyclicLoopMap.put(GraphDACParams.loop.name, new Boolean(false)) + + } + TelemetryManager.log("Returning Cyclic Loop Map: ", cyclicLoopMap) + cyclicLoopMap + } + + + def generateCheckCyclicLoopTraversal(parameterMap: util.Map[String, AnyRef], g: GraphTraversalSource) = { + + if (null != parameterMap) { + val graphId = parameterMap.get(GraphDACParams.graphId.name).asInstanceOf[String] + if (StringUtils.isBlank(graphId)) + throw new ClientException(DACErrorCodeConstants.INVALID_GRAPH.name, + DACErrorMessageConstants.INVALID_GRAPH_ID + " | ['Check Cyclic Loop' Query Generation Failed.]") + + val startNodeId = parameterMap.get(GraphDACParams.startNodeId.name).asInstanceOf[String] + if (StringUtils.isBlank(startNodeId)) + throw new ClientException(DACErrorCodeConstants.INVALID_IDENTIFIER.name, + DACErrorMessageConstants.INVALID_START_NODE_ID + " | ['Check Cyclic Loop' Query Generation Failed.]") + + val relationType = parameterMap.get(GraphDACParams.relationType.name).asInstanceOf[String] + if (StringUtils.isBlank(relationType)) + throw new ClientException(DACErrorCodeConstants.INVALID_RELATION.name, + DACErrorMessageConstants.INVALID_RELATION_TYPE + " | ['Check Cyclic Loop' Query Generation Failed.]") + + val endNodeId = parameterMap.get(GraphDACParams.endNodeId.name).asInstanceOf[String] + if (StringUtils.isBlank(endNodeId)) + throw new ClientException(DACErrorCodeConstants.INVALID_IDENTIFIER.name, + DACErrorMessageConstants.INVALID_END_NODE_ID + " | ['Check Cyclic Loop' Query Generation Failed.]") + + val cyclicTraversal = g.V().hasLabel(graphId).has(SystemProperties.IL_UNIQUE_ID.name(), startNodeId) + .repeat(outE(relationType).inV().simplePath()).until(has(SystemProperties.IL_UNIQUE_ID.name(), endNodeId)) + .hasLabel(graphId) + cyclicTraversal + } + else throw new ClientException(DACErrorCodeConstants.INVALID_PARAMETER.name, DACErrorMessageConstants.INVALID_PARAM_MAP) + } + + + def executeGetNodeProperty(parameterMap: util.Map[String, AnyRef], g: GraphTraversalSource) = { + try { + if (null != parameterMap) { + val graphId = parameterMap.get(GraphDACParams.graphId.name).asInstanceOf[String] + if (StringUtils.isBlank(graphId)) + throw new ClientException(DACErrorCodeConstants.INVALID_GRAPH.name, + DACErrorMessageConstants.INVALID_GRAPH_ID + " | ['Get Node Property' Query Generation Failed.]") + + val nodeId = parameterMap.get(GraphDACParams.nodeId.name).asInstanceOf[String] + if (StringUtils.isBlank(nodeId)) + throw new ClientException(DACErrorCodeConstants.INVALID_IDENTIFIER.name, + DACErrorMessageConstants.INVALID_IDENTIFIER + " | ['Get Node Property' Query Generation Failed.]") + + val key = parameterMap.get(GraphDACParams.key.name).asInstanceOf[String] + if (StringUtils.isBlank(key)) + throw new ClientException(DACErrorCodeConstants.INVALID_PROPERTY.name, + DACErrorMessageConstants.INVALID_PROPERTY_KEY + " | ['Get Node Property' Query Generation Failed.]") + + g.V().hasLabel(graphId).has(SystemProperties.IL_UNIQUE_ID.name, nodeId).values(key) + + } + else throw new ClientException(DACErrorCodeConstants.INVALID_PARAMETER.name, DACErrorMessageConstants.INVALID_PARAM_MAP) + } + catch { + case e: Exception => + throw new ServerException(DACErrorCodeConstants.SERVER_ERROR.name, "Error! Something went wrong while creating node object. ", e.getCause); + } + } + + private def getVertexByUniqueId(parameterMap: util.Map[String, AnyRef], g: GraphTraversalSource): util.List[util.Map[String, AnyRef]] = { + try { + if (null != parameterMap) { + val graphId = parameterMap.getOrDefault(GraphDACParams.graphId.name, "").asInstanceOf[String] + if (StringUtils.isBlank(graphId)) + throw new ClientException(DACErrorCodeConstants.INVALID_GRAPH.name, + DACErrorMessageConstants.INVALID_GRAPH_ID + " | ['Get Node By Id' Query Generation Failed.]") + + val vertexId = parameterMap.get(GraphDACParams.nodeId.name).asInstanceOf[String] + if (StringUtils.isBlank(vertexId)) + throw new ClientException(DACErrorCodeConstants.INVALID_IDENTIFIER.name, + DACErrorMessageConstants.INVALID_IDENTIFIER + " | ['Get Node By Unique Id' Query Generation Failed.]") + val result = g.V().hasLabel(graphId).has("IL_UNIQUE_ID", vertexId).as("ee") + .flatMap( + coalesce( + union( + outE().dedup().as("r").inV().dedup().as("__endNode").select("ee", "r", "__endNode"), + inE().dedup().as("r").outV().dedup().as("__startNode").select("ee", "r", "__startNode") + ), + project("ee", "r", "__startNode", "__endNode") + .by(select("ee")) + .by(constant(null)) + .by(constant(null)) + .by(constant(null)) + ) + ) + .dedup().toList.asInstanceOf[util.List[util.Map[String, AnyRef]]] + + val finalList = new util.ArrayList[util.Map[String, AnyRef]] + result.asScala.map { tr => + val ee = tr.get("ee").asInstanceOf[org.apache.tinkerpop.gremlin.structure.Vertex] + val r = tr.get("r") + val startNode = tr.get("__startNode") match { + case null => ee + case node => node + } + val endNode = tr.get("__endNode") match { + case null => ee + case node => node + } + val resMap = new util.HashMap[String, AnyRef] + resMap.put("ee", ee) + resMap.put("r", r) + resMap.put("__startNode", startNode) + resMap.put("__endNode", endNode) + finalList.add(resMap) + }.asJava + + finalList + } + else throw new ClientException(DACErrorCodeConstants.INVALID_PARAMETER.name, DACErrorMessageConstants.INVALID_PARAM_MAP ) + } + catch { + case e :Exception => + e.printStackTrace() + throw new ServerException(DACErrorCodeConstants.SERVER_ERROR.name, "Error! Something went wrong while creating node object. ", e.getCause); + } + } + + private def getRecordValues(result: util.Map[String, AnyRef], nodeMap :util.Map[Object, AnyRef], relationMap :util.Map[Object, AnyRef], startNodeMap :util.Map[Object, AnyRef], endNodeMap :util.Map[Object, AnyRef] ): Unit = { + if (null != nodeMap) { + val vertexValue = result.get(CypherQueryConfigurationConstants.DEFAULT_CYPHER_NODE_OBJECT) + if (null != vertexValue && vertexValue.isInstanceOf[org.apache.tinkerpop.gremlin.structure.Vertex]) { + val gremlinVertex : org.apache.tinkerpop.gremlin.structure.Vertex = result.get(CypherQueryConfigurationConstants.DEFAULT_CYPHER_NODE_OBJECT).asInstanceOf[org.apache.tinkerpop.gremlin.structure.Vertex] + nodeMap.put(gremlinVertex.id(), gremlinVertex) + } + } + if (null != relationMap) { + val edgeValue = result.get(CypherQueryConfigurationConstants.DEFAULT_CYPHER_RELATION_OBJECT) + if (null != edgeValue && edgeValue.isInstanceOf[org.apache.tinkerpop.gremlin.structure.Edge]) { + val edge: org.apache.tinkerpop.gremlin.structure.Edge = result.get(CypherQueryConfigurationConstants.DEFAULT_CYPHER_RELATION_OBJECT).asInstanceOf[Edge] + relationMap.put(edge.id(), edge) + } + } + + if (null != startNodeMap) { + val startVertexValue = result.get(CypherQueryConfigurationConstants.DEFAULT_CYPHER_START_NODE_OBJECT) + + if (null != startVertexValue && startVertexValue.isInstanceOf[org.apache.tinkerpop.gremlin.structure.Vertex]) { + val startVertex: org.apache.tinkerpop.gremlin.structure.Vertex = result.get(CypherQueryConfigurationConstants.DEFAULT_CYPHER_START_NODE_OBJECT).asInstanceOf[org.apache.tinkerpop.gremlin.structure.Vertex] + startNodeMap.put(startVertex.id(), startVertex) + } + } + if (null != endNodeMap) { + val endVertexValue = result.get(CypherQueryConfigurationConstants.DEFAULT_CYPHER_END_NODE_OBJECT) + if (null != endVertexValue && endVertexValue.isInstanceOf[org.apache.tinkerpop.gremlin.structure.Vertex]) { + val endVertex: org.apache.tinkerpop.gremlin.structure.Vertex = result.get(CypherQueryConfigurationConstants.DEFAULT_CYPHER_END_NODE_OBJECT).asInstanceOf[org.apache.tinkerpop.gremlin.structure.Vertex] + endNodeMap.put(endVertex.id(), endVertex) + } + } + } + +} diff --git a/ontology-engine/graph-dac/src/main/scala/org/sunbird/janus/service/operation/VertexOperations.scala b/ontology-engine/graph-dac/src/main/scala/org/sunbird/janus/service/operation/VertexOperations.scala new file mode 100644 index 000000000..f02d44662 --- /dev/null +++ b/ontology-engine/graph-dac/src/main/scala/org/sunbird/janus/service/operation/VertexOperations.scala @@ -0,0 +1,489 @@ +package org.sunbird.janus.service.operation + +import org.apache.commons.collections4.MapUtils +import org.apache.commons.lang3.{BooleanUtils, StringUtils} +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.{GraphTraversal, GraphTraversalSource} +import org.sunbird.common.dto.Request +import org.sunbird.common.exception.{ClientException, ResourceNotFoundException, ServerException} +import org.sunbird.common.{DateUtils, JsonUtils} +import org.sunbird.graph.common.Identifier +import org.sunbird.graph.common.enums.{AuditProperties, GraphDACParams, SystemProperties} +import org.sunbird.graph.dac.enums.SystemNodeTypes +import org.sunbird.graph.dac.model.Vertex +import org.sunbird.graph.service.common.{DACErrorCodeConstants, DACErrorMessageConstants} +import org.sunbird.janus.service.util.JanusConnectionUtil +import org.sunbird.telemetry.logger.TelemetryManager + +import java.util +import scala.collection.convert.ImplicitConversions.{`collection AsScalaIterable`, `map AsJavaMap`, `map AsScala`} +import scala.collection.immutable.HashMap +import scala.collection.mutable +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future + +class VertexOperations { + + val graphConnection = new JanusConnectionUtil + def addVertex(graphId: String, vertex: Vertex): Future[Vertex] = { + Future { + if (StringUtils.isBlank(graphId)) + throw new ClientException(DACErrorCodeConstants.INVALID_GRAPH.name, + DACErrorMessageConstants.INVALID_GRAPH_ID + " | [Create Node Operation Failed.]") + + if (null == vertex) + throw new ClientException(DACErrorCodeConstants.INVALID_NODE.name, + DACErrorMessageConstants.INVALID_NODE + " | [Create Node Operation Failed.]") + + val parameterMap = new util.HashMap[String, AnyRef] + parameterMap.put(GraphDACParams.graphId.name, graphId) + parameterMap.put(GraphDACParams.vertex.name, setPrimitiveData(vertex)) + + try { + graphConnection.initialiseGraphClient() + val g: GraphTraversalSource = graphConnection.getGraphTraversalSource + + val addedVertex = createVertexTraversal(parameterMap, g) + val vertexElementMap = addedVertex.elementMap().next() + + vertex.setGraphId(graphId) + vertex.setIdentifier(vertexElementMap.get(SystemProperties.IL_UNIQUE_ID.name)) + vertex.getMetadata.put(GraphDACParams.versionKey.name, vertexElementMap.get(GraphDACParams.versionKey.name)) + vertex + } + catch { + case e: Throwable => + e.printStackTrace() + e.getCause match { + case cause: org.apache.tinkerpop.gremlin.driver.exception.ResponseException => + throw new ClientException( + DACErrorCodeConstants.CONSTRAINT_VALIDATION_FAILED.name(), + DACErrorMessageConstants.CONSTRAINT_VALIDATION_FAILED + vertex.getIdentifier + ) + case cause => + throw new ServerException(DACErrorCodeConstants.CONNECTION_PROBLEM.name, DACErrorMessageConstants.CONNECTION_PROBLEM + " | " + e.getMessage, e) + } + } + } + } + + def deleteVertex(graphId: String, vertexId: String, request: Request): Future[java.lang.Boolean] = { + Future { + if (StringUtils.isBlank(graphId)) + throw new ClientException(DACErrorCodeConstants.INVALID_GRAPH.name, + DACErrorMessageConstants.INVALID_GRAPH_ID + " | [Remove Property Values Operation Failed.]") + + if (StringUtils.isBlank(vertexId)) + throw new ClientException(DACErrorCodeConstants.INVALID_IDENTIFIER.name, + DACErrorMessageConstants.INVALID_IDENTIFIER + " | [Remove Property Values Operation Failed.]") + + try { + graphConnection.initialiseGraphClient() + val g: GraphTraversalSource = graphConnection.getGraphTraversalSource + + val parameterMap = new util.HashMap[String, AnyRef] + parameterMap.put(GraphDACParams.graphId.name, graphId) + parameterMap.put(GraphDACParams.nodeId.name, vertexId) + parameterMap.put(GraphDACParams.request.name, request) + + executeVertexDeletion(parameterMap, g) + } + catch { + case e: Throwable => + e.printStackTrace() + throw new ServerException(DACErrorCodeConstants.CONNECTION_PROBLEM.name, + DACErrorMessageConstants.CONNECTION_PROBLEM + " | " + e.getMessage, e) + } + } + } + + private def executeVertexDeletion(parameterMap: util.Map[String, AnyRef], g: GraphTraversalSource): Boolean = { + if (null != parameterMap) { + val graphId = parameterMap.get(GraphDACParams.graphId.name).asInstanceOf[String] + if (StringUtils.isBlank(graphId)) + throw new ClientException(DACErrorCodeConstants.INVALID_GRAPH.name, + DACErrorMessageConstants.INVALID_GRAPH_ID + " | [Remove Property Values Query Generation Failed.]") + + val nodeId = parameterMap.get(GraphDACParams.nodeId.name).asInstanceOf[String] + if (StringUtils.isBlank(nodeId)) + throw new ClientException(DACErrorCodeConstants.INVALID_IDENTIFIER.name, + DACErrorMessageConstants.INVALID_IDENTIFIER + " | [Remove Property Values Query Generation Failed.]") + + val traversal = g.V().hasLabel(graphId).has(SystemProperties.IL_UNIQUE_ID.name(), nodeId) + if (traversal.hasNext) { + traversal.drop().iterate() + true + } else { + throw new ResourceNotFoundException(DACErrorCodeConstants.NOT_FOUND.name, + DACErrorMessageConstants.NODE_NOT_FOUND + " | [Invalid Node Id.]: " + nodeId, nodeId) + } + } else { + throw new ClientException(DACErrorCodeConstants.INVALID_PARAMETER.name, DACErrorMessageConstants.INVALID_PARAM_MAP) + } + } + + private def createVertexTraversal(parameterMap: util.Map[String, AnyRef], g: GraphTraversalSource): GraphTraversal[org.apache.tinkerpop.gremlin.structure.Vertex, org.apache.tinkerpop.gremlin.structure.Vertex] = { + if (null != parameterMap) { + val graphId = parameterMap.getOrDefault(GraphDACParams.graphId.name,"").asInstanceOf[String] + val vertex = parameterMap.getOrDefault(GraphDACParams.vertex.name, null).asInstanceOf[Vertex] + + if (StringUtils.isBlank(graphId)) + throw new ClientException(DACErrorCodeConstants.INVALID_GRAPH.name, + DACErrorMessageConstants.INVALID_GRAPH_ID + " | ['Create Node' Query Generation Failed.]") + + if (null == vertex) + throw new ClientException(DACErrorCodeConstants.INVALID_NODE.name, + DACErrorMessageConstants.INVALID_NODE + " | [Create Node Query Generation Failed.]") + + val date: String = DateUtils.formatCurrentDate + + val mpMap :util.Map[String, AnyRef] = getMetadataCypherQueryMap(vertex) + val spMap :util.Map[String, AnyRef] = getSystemPropertyMap(vertex, date) + val apMap :util.Map[String, AnyRef] = getAuditPropertyMap(vertex, date, false) + val vpMap :util.Map[String, AnyRef] = getVersionPropertyMap(vertex, date) + + val combinedMap: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] + combinedMap.putAll(mpMap) + combinedMap.putAll(spMap) + combinedMap.putAll(apMap) + combinedMap.putAll(vpMap) + + parameterMap.put(GraphDACParams.paramValueMap.name, combinedMap) + + val newVertexTraversal = g.addV(vertex.getGraphId) + val finalMap = parameterMap.getOrDefault(GraphDACParams.paramValueMap.name, new util.HashMap[String, AnyRef]).asInstanceOf[util.Map[String, AnyRef]] + + finalMap.foreach { case (key, value) => newVertexTraversal.property(key, value) } + + newVertexTraversal + } + else { + throw new ClientException(DACErrorCodeConstants.INVALID_PARAMETER.name, DACErrorMessageConstants.INVALID_PARAM_MAP ) + } + } + + def getMetadataCypherQueryMap(node: Vertex): util.Map[String, AnyRef] = { + val metadataPropertyMap = new util.HashMap[String, AnyRef] + if (null != node && null != node.getMetadata && !node.getMetadata.isEmpty) { + node.getMetadata.foreach { case (key, value) => metadataPropertyMap.put(key, value) } + } + metadataPropertyMap + } + + def getSystemPropertyMap(node: Vertex, date: String): util.Map[String, AnyRef] = { + val systemPropertyMap = new util.HashMap[String, AnyRef] + if (null != node && StringUtils.isNotBlank(date)) { + if (StringUtils.isBlank(node.getIdentifier)) + node.setIdentifier(Identifier.getIdentifier(node.getGraphId, Identifier.getUniqueIdFromTimestamp)) + systemPropertyMap.put(SystemProperties.IL_UNIQUE_ID.name, node.getIdentifier) + systemPropertyMap.put(SystemProperties.IL_SYS_NODE_TYPE.name, node.getVertexType) + systemPropertyMap.put(SystemProperties.IL_FUNC_OBJECT_TYPE.name, node.getObjectType) + } + systemPropertyMap + } + + def getAuditPropertyMap(node: Vertex, date: String, isUpdateOnly: Boolean):util.Map[String, AnyRef] = { + val auditPropertyMap = new util.HashMap[String, AnyRef] + if(null != node && StringUtils.isNotBlank(date)) { + if (BooleanUtils.isFalse(isUpdateOnly)) { + auditPropertyMap.put(AuditProperties.createdOn.name, + if (node.getMetadata.containsKey(AuditProperties.createdOn.name)) + node.getMetadata.get(AuditProperties.createdOn.name) + else date) + } + if (null != node.getMetadata && null == node.getMetadata.get(GraphDACParams.SYS_INTERNAL_LAST_UPDATED_ON.name)) + auditPropertyMap.put(AuditProperties.lastUpdatedOn.name, date) + } + auditPropertyMap + } + + def getVersionPropertyMap(node: Vertex, date: String): util.Map[String, AnyRef] = { + val versionPropertyMap = new util.HashMap[String, AnyRef] + if (null != node && StringUtils.isNotBlank(date)) + versionPropertyMap.put(GraphDACParams.versionKey.name, DateUtils.parse(date).getTime.toString) + versionPropertyMap + } + + def upsertVertex(graphId: String, vertex: Vertex, request: Request): Future[Vertex] = { + Future { + if (StringUtils.isBlank(graphId)) + throw new ClientException(DACErrorCodeConstants.INVALID_GRAPH.name, + DACErrorMessageConstants.INVALID_GRAPH_ID + " | [Create Node Operation Failed.]") + + if (null == vertex) + throw new ClientException(DACErrorCodeConstants.INVALID_NODE.name, + DACErrorMessageConstants.INVALID_NODE + " | [Create Node Operation Failed.]") + + val parameterMap = new util.HashMap[String, AnyRef] + parameterMap.put(GraphDACParams.graphId.name, graphId) + parameterMap.put(GraphDACParams.vertex.name, setPrimitiveData(vertex)) + parameterMap.put(GraphDACParams.request.name, request) + + prepareUpsertMap(parameterMap) + try { + graphConnection.initialiseGraphClient() + val g: GraphTraversalSource = graphConnection.getGraphTraversalSource + + val existingVertex = g.V().has(SystemProperties.IL_UNIQUE_ID.name, vertex.getIdentifier) + val finalMap = parameterMap.getOrDefault(GraphDACParams.paramValueMap.name, new util.HashMap[String, AnyRef]).asInstanceOf[util.Map[String, AnyRef]] + finalMap.foreach { case (key, value) => + if (!key.equals(GraphDACParams.graphId.name) && !key.equals(GraphDACParams.request.name)) { + existingVertex.property(key, value) + } + } + + val retrieveVertex = existingVertex.elementMap().next() + vertex.setIdentifier(retrieveVertex.get(SystemProperties.IL_UNIQUE_ID.name)) + vertex.getMetadata.put(GraphDACParams.versionKey.name, retrieveVertex.get(GraphDACParams.versionKey.name)) + vertex + } catch { + case e: Throwable => + e.printStackTrace() + e.getCause match { + case cause: org.apache.tinkerpop.gremlin.driver.exception.ResponseException => + throw new ClientException( + DACErrorCodeConstants.CONSTRAINT_VALIDATION_FAILED.name(), + DACErrorMessageConstants.CONSTRAINT_VALIDATION_FAILED + vertex.getIdentifier + ) + case cause => + throw new ServerException(DACErrorCodeConstants.CONNECTION_PROBLEM.name, DACErrorMessageConstants.CONNECTION_PROBLEM + " | " + e.getMessage, e) + } + } + } + } + + def prepareUpsertMap(parameterMap: util.Map[String, AnyRef]) = { + if (null != parameterMap) { + val vertex = parameterMap.getOrDefault("vertex", null).asInstanceOf[Vertex] + if (null == vertex) + throw new ClientException(DACErrorCodeConstants.INVALID_NODE.name, + DACErrorMessageConstants.INVALID_NODE + " | [Upsert Node Query Generation Failed.]") + if (StringUtils.isBlank(vertex.getIdentifier)) + vertex.setIdentifier(Identifier.getIdentifier(vertex.getGraphId, Identifier.getUniqueIdFromTimestamp)) + val date: String = DateUtils.formatCurrentDate + + val ocsMap: util.Map[String, AnyRef] = getOnCreateSetMap( vertex, date) + val omsMap: util.Map[String, AnyRef] = getOnMatchSetMap( vertex,date, merge = true) + val combinedMap: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] + combinedMap.putAll(ocsMap) + combinedMap.putAll(omsMap) + parameterMap.put(GraphDACParams.paramValueMap.name, combinedMap) + } + } + + private def getOnCreateSetMap(node: Vertex, date: String): util.Map[String, Object] = { + val paramMap = new util.HashMap[String, Object]() + + if (node != null && StringUtils.isNotBlank(date)) { + if (StringUtils.isBlank(node.getIdentifier)) { + node.setIdentifier(Identifier.getIdentifier(node.getGraphId, Identifier.getUniqueIdFromTimestamp)) + } + + paramMap.put(SystemProperties.IL_UNIQUE_ID.name, node.getIdentifier) + + paramMap.put(SystemProperties.IL_SYS_NODE_TYPE.name, node.getVertexType) + + if (StringUtils.isNotBlank(node.getObjectType)) { + paramMap.put(SystemProperties.IL_FUNC_OBJECT_TYPE.name, node.getObjectType) + } + paramMap.put(AuditProperties.createdOn.name, date) + + if (!node.getMetadata.containsKey(GraphDACParams.SYS_INTERNAL_LAST_UPDATED_ON.name)) { + paramMap.put(AuditProperties.lastUpdatedOn.name, date) + } + val versionKey = DateUtils.parse(date).getTime.toString + paramMap.put(GraphDACParams.versionKey.name, versionKey) + } + + paramMap + } + + private def getOnMatchSetMap(node: Vertex, date: String, merge: Boolean): util.Map[String, Object] = { + val paramMap = new util.HashMap[String, Object]() + + if (node != null && StringUtils.isNotBlank(date)) { + + if (node.getMetadata != null) { + node.getMetadata.foreach { + case (key, value) => if (key != GraphDACParams.versionKey.name) paramMap.put(key, value) + } + } + + // Set lastUpdatedOn property if not already set + if (!node.getMetadata.containsKey(GraphDACParams.SYS_INTERNAL_LAST_UPDATED_ON.name)) { + paramMap.put(AuditProperties.lastUpdatedOn.name, date) + } + + // Set versionKey property if missing in metadata + if (node.getMetadata != null && + StringUtils.isBlank(node.getMetadata.get(GraphDACParams.versionKey.name()).toString)) { + val versionKey = DateUtils.parse(date).getTime.toString + paramMap.put(GraphDACParams.versionKey.name, versionKey) + } + } + + paramMap + } + + def upsertRootVertex(graphId: String, request: AnyRef): Future[Vertex] = { + Future { + if (StringUtils.isBlank(graphId)) + throw new ClientException(DACErrorCodeConstants.INVALID_GRAPH.name(), + DACErrorMessageConstants.INVALID_GRAPH_ID + " | [Upsert Root Node Operation Failed.]") + + val g = graphConnection.getGraphTraversalSource + + val rootNodeUniqueId = Identifier.getIdentifier(graphId, SystemNodeTypes.ROOT_NODE.name()) + + val vertex = new Vertex + vertex.setIdentifier(rootNodeUniqueId) + vertex.getMetadata().put(SystemProperties.IL_UNIQUE_ID.name, rootNodeUniqueId) + vertex.getMetadata().put(SystemProperties.IL_SYS_NODE_TYPE.name, SystemNodeTypes.ROOT_NODE.name) + vertex.getMetadata().put(AuditProperties.createdOn.name, DateUtils.formatCurrentDate()) + vertex.getMetadata().put(GraphDACParams.Nodes_Count.name, 0: Integer) + vertex.getMetadata().put(GraphDACParams.Relations_Count.name, 0: Integer) + + val parameterMap = Map( + GraphDACParams.graphId.name -> graphId, + GraphDACParams.rootNode.name -> vertex, + GraphDACParams.request.name -> request + ) + + try { + val existingRootNode = g.V().has(SystemProperties.IL_UNIQUE_ID.name, rootNodeUniqueId).next() + + val updatedVertex = existingRootNode.property(AuditProperties.createdOn.name, DateUtils.formatCurrentDate()) + + val identifier = updatedVertex.property(SystemProperties.IL_UNIQUE_ID.name).value().toString + val versionKey = Option(updatedVertex.property(GraphDACParams.versionKey.name)).map(_.value().toString).getOrElse("") + + vertex.setIdentifier(identifier) + vertex.setGraphId(graphId) + if (StringUtils.isNotBlank(versionKey)) + vertex.getMetadata.put(GraphDACParams.versionKey.name, versionKey) + + vertex + + } + catch { + case e: Throwable => + throw new ServerException(DACErrorCodeConstants.CONNECTION_PROBLEM.name, + DACErrorMessageConstants.CONNECTION_PROBLEM + " | " + e.getMessage, e) + } + } + } + + def updateVertexes(graphId: String, identifiers: java.util.List[String], data: java.util.Map[String, AnyRef]): Future[util.Map[String, Vertex]] = { + Future { + if (StringUtils.isBlank(graphId)) + throw new ClientException(DACErrorCodeConstants.INVALID_GRAPH.name, + DACErrorMessageConstants.INVALID_GRAPH_ID + " | [Invalid or 'null' Graph Id.]") + + + if (identifiers.isEmpty) + throw new ClientException(DACErrorCodeConstants.INVALID_IDENTIFIER.name, + DACErrorMessageConstants.INVALID_IDENTIFIER + " | [Please Provide Node Identifier.]") + + if (MapUtils.isEmpty(data)) + throw new ClientException(DACErrorCodeConstants.INVALID_METADATA.name, + DACErrorMessageConstants.INVALID_METADATA + " | [Please Provide Valid Node Metadata]") + + val parameterMap = generateUpdateVerticesQuery(graphId, identifiers, setPrimitiveData(data)) + + try { + graphConnection.initialiseGraphClient() + val g: GraphTraversalSource = graphConnection.getGraphTraversalSource + + val updatedVertices = identifiers.foldLeft(List.empty[Vertex]) { + (acc: List[Vertex], identifier: String) => + val existingVertex = g.V().has(SystemProperties.IL_UNIQUE_ID.toString, identifier) + val finalMap = parameterMap.getOrDefault(GraphDACParams.paramValueMap.name, new util.HashMap[String, AnyRef]).asInstanceOf[util.Map[String, AnyRef]] + + finalMap.foreach { case (key, value) => + if (!key.equals(GraphDACParams.graphId.name) && !key.equals(GraphDACParams.request.name)) { + existingVertex.property(key, value) + } + } + + val updatedVertex = existingVertex.toList().head.asInstanceOf[Vertex] + + acc :+ updatedVertex + } + + + val resultMap = updatedVertices.map(vertex => { + val identifier = vertex.getMetadata.get(SystemProperties.IL_UNIQUE_ID.name).asInstanceOf[String] + val newVertex = new Vertex + newVertex.setIdentifier(identifier) + newVertex + }) + + resultMap.asInstanceOf[Map[String, Vertex]] + } + catch { + case e: Throwable => + e.printStackTrace() + e match { + case cause: org.apache.tinkerpop.gremlin.driver.exception.ResponseException => + throw new ClientException( + DACErrorCodeConstants.CONSTRAINT_VALIDATION_FAILED.name(), + DACErrorMessageConstants.CONSTRAINT_VALIDATION_FAILED + " | Updating multiple nodes failed." + ) + case cause => + val errorMessage = DACErrorMessageConstants.CONNECTION_PROBLEM + " | " + e.getMessage + throw new ServerException(DACErrorCodeConstants.CONNECTION_PROBLEM.name, errorMessage, e) + } + } + } + } + + private def setPrimitiveData(metadata: java.util.Map[String, AnyRef]): mutable.Map[String, Object] = { + metadata.flatMap { case (key, value) => + val processedValue = value match { + case map: Map[Any, Any] => + try { + JsonUtils.serialize(map) + } catch { + case e: Exception => + TelemetryManager.error("Exception Occurred While Processing Primitive Data Types | Exception is : " + e.getMessage(), e) + value + } + case list: List[_] if list.nonEmpty && list.head.isInstanceOf[Map[Any, Any]] => + try { + JsonUtils.serialize(list) + } catch { + case e: Exception => + TelemetryManager.error("Exception Occurred While Processing Primitive Data Types | Exception is : " + e.getMessage(), e) + value + } + case _ => value + } + Some((key, processedValue)) + } + } + + def generateUpdateVerticesQuery(graphId: String, identifiers: java.util.List[String], data: mutable.Map[String, AnyRef]): Map[String, Object] = { + val parameterMap = new HashMap[String, Object] + parameterMap.put("identifiers", identifiers); + parameterMap.putAll(data); + parameterMap; + } + + private def setPrimitiveData(vertex: Vertex): Vertex = { + val metadata: util.Map[String, AnyRef] = vertex.getMetadata + metadata.forEach((key, value) => { + try { + value match { + case v: util.Map[String, AnyRef] => metadata.put(key, JsonUtils.serialize(v)) + case v: util.List[util.Map[String, AnyRef]] if (!v.isEmpty && v.isInstanceOf[util.Map[String, AnyRef]]) => metadata.put(key, JsonUtils.serialize(v)) + case _ => + } + } catch { + case e: Exception => TelemetryManager.error(s"Exception Occurred While Processing Primitive Data Types | Exception is : ${e.getMessage}", e) + } + }) + vertex + } + + + +} diff --git a/ontology-engine/graph-dac/src/main/scala/org/sunbird/janus/service/util/JanusConnectionUtil.scala b/ontology-engine/graph-dac/src/main/scala/org/sunbird/janus/service/util/JanusConnectionUtil.scala new file mode 100644 index 000000000..f602d8528 --- /dev/null +++ b/ontology-engine/graph-dac/src/main/scala/org/sunbird/janus/service/util/JanusConnectionUtil.scala @@ -0,0 +1,52 @@ +package org.sunbird.janus.service.util + +import org.apache.tinkerpop.gremlin.driver.remote.DriverRemoteConnection +import org.apache.tinkerpop.gremlin.driver.{Client, Cluster} +import org.apache.tinkerpop.gremlin.process.traversal.AnonymousTraversalSource.traversal +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource +import org.apache.tinkerpop.gremlin.structure.io.binary.TypeSerializerRegistry +import org.apache.tinkerpop.gremlin.structure.util.empty.EmptyGraph +import org.apache.tinkerpop.gremlin.util.ser.GraphBinaryMessageSerializerV1 +import org.janusgraph.core.{JanusGraph, JanusGraphFactory} +import org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistry +import org.sunbird.telemetry.logger.TelemetryManager + +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future +class JanusConnectionUtil { + + var g: GraphTraversalSource = null + var graph: JanusGraph = null + + + + @throws[Exception] + def initialiseGraphClient(): Unit = { + try { + if (null == g) { + println("GraphTraversalSource: "+g) + g = traversal.withRemote("/Users/admin/Documents/workspace/knowledge-platform/ontology-engine/graph-dac/src/main/conf/remote-graph.properties") + } + if (null == graph) graph = JanusGraphFactory.open("/Users/admin/Documents/workspace/knowledge-platform/ontology-engine/graph-dac/src/main/conf/janusgraph-inmemory.properties") + + } + catch { + case e: Exception => + TelemetryManager.log("JanusConnectionUtil --> Exception: " + e.getCause) + e.printStackTrace() + } + } + + @throws[Exception] + def getGraphTraversalSource: GraphTraversalSource = g + + @throws[Exception] + def getGraph: JanusGraph = graph + + @throws[Exception] + def closeClient(): Unit = { + g.close() + graph.close() + } + +} diff --git a/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/nodes/DataNode.scala b/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/nodes/DataNode.scala index 1cd022a11..ca3b62d83 100644 --- a/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/nodes/DataNode.scala +++ b/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/nodes/DataNode.scala @@ -1,8 +1,5 @@ package org.sunbird.graph.nodes -import java.util -import java.util.Optional -import java.util.concurrent.CompletionException import org.apache.commons.collections4.{CollectionUtils, MapUtils} import org.apache.commons.lang3.StringUtils import org.sunbird.common.DateUtils @@ -10,12 +7,15 @@ import org.sunbird.common.dto.{Request, Response} import org.sunbird.common.exception.{ClientException, ErrorCodes, ResponseCode} import org.sunbird.graph.OntologyEngineContext import org.sunbird.graph.common.enums.SystemProperties -import org.sunbird.graph.dac.model.{Filter, MetadataCriterion, Node, Relation, SearchConditions, SearchCriteria} +import org.sunbird.graph.dac.model._ import org.sunbird.graph.schema.{DefinitionDTO, DefinitionFactory, DefinitionNode} import org.sunbird.parseq.Task -import scala.collection.convert.ImplicitConversions._ +import java.util +import java.util.Optional +import java.util.concurrent.CompletionException import scala.collection.JavaConverters._ +import scala.collection.convert.ImplicitConversions._ import scala.concurrent.{ExecutionContext, Future} @@ -112,7 +112,7 @@ object DataNode { oec.graphService.deleteNode(request.graphId, identifier, request) } - private def saveExternalProperties(identifier: String, externalProps: util.Map[String, AnyRef], context: util.Map[String, AnyRef], objectType: String)(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Response] = { + def saveExternalProperties(identifier: String, externalProps: util.Map[String, AnyRef], context: util.Map[String, AnyRef], objectType: String)(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Response] = { if (MapUtils.isNotEmpty(externalProps)) { externalProps.put("identifier", identifier) val request = new Request(context, externalProps, "", objectType) @@ -122,7 +122,7 @@ object DataNode { } } - private def updateExternalProperties(identifier: String, externalProps: util.Map[String, AnyRef], context: util.Map[String, AnyRef], objectType: String, request: Request)(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Response] = { + def updateExternalProperties(identifier: String, externalProps: util.Map[String, AnyRef], context: util.Map[String, AnyRef], objectType: String, request: Request)(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Response] = { if (MapUtils.isNotEmpty(externalProps)) { val req = new Request(request) req.put("identifier", identifier) @@ -185,11 +185,13 @@ object DataNode { } list } + private def defaultDataModifier(node: Node) = { node } + @throws[Exception] def systemUpdate(request: Request, nodeList: util.List[Node], hierarchyKey: String, hierarchyFunc: Option[Request => Future[Response]] = None)(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Node] = { val data: util.Map[String, AnyRef] = request.getRequest diff --git a/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/path/DataSubGraph.scala b/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/path/DataSubGraph.scala index e7605e4e4..960fde586 100644 --- a/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/path/DataSubGraph.scala +++ b/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/path/DataSubGraph.scala @@ -5,7 +5,7 @@ import org.apache.commons.lang3.StringUtils import org.sunbird.common.dto.Request import org.sunbird.graph.OntologyEngineContext import org.sunbird.graph.common.enums.SystemProperties -import org.sunbird.graph.dac.model.{Node, Relation, SubGraph} +import org.sunbird.graph.dac.model.{Node, Relation, SubGraph, VertexSubGraph} import org.sunbird.graph.nodes.DataNode import org.sunbird.graph.schema.{DefinitionFactory, DefinitionNode, ObjectCategoryDefinition} import org.sunbird.graph.utils.NodeUtil @@ -28,6 +28,13 @@ object DataSubGraph { subGraph } + def readVertex(request: Request)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[VertexSubGraph] = { + val identifier: String = request.get("identifier").asInstanceOf[String] + val depth: Int = request.getOrDefault("depth", 5).asInstanceOf[Int] + val subGraph: Future[VertexSubGraph] = oec.janusGraphService.getSubGraph(request.graphId, identifier, depth) + subGraph + } + def readSubGraph(request: Request)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Unit] = { val identifier: String = request.get("identifier").asInstanceOf[String] val schemaName: String = request.getContext.get("schemaName").asInstanceOf[String] diff --git a/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/schema/DefinitionDTO.scala b/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/schema/DefinitionDTO.scala index 70413b352..16c125015 100644 --- a/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/schema/DefinitionDTO.scala +++ b/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/schema/DefinitionDTO.scala @@ -8,7 +8,7 @@ import org.sunbird.common.exception.{ClientException, ResponseCode} import org.sunbird.graph.OntologyEngineContext import org.sunbird.graph.common.Identifier import org.sunbird.graph.dac.enums.SystemNodeTypes -import org.sunbird.graph.dac.model.Node +import org.sunbird.graph.dac.model.{Node, Vertex} import org.sunbird.graph.schema.validator._ import scala.collection.JavaConverters._ @@ -32,6 +32,20 @@ class DefinitionDTO(graphId: String, schemaName: String, version: String = "1.0" node } + def getVertex(identifier: String, input: java.util.Map[String, AnyRef], vertexType: String): Vertex = { + val result = schemaValidator.getStructuredData(input) + val objectType = schemaValidator.getConfig.getString("objectType") + val vertex = new Vertex(identifier, objectType, vertexType) + vertex.setGraphId(graphId) + vertex.setVertexType(SystemNodeTypes.DATA_NODE.name) + vertex.setObjectType(objectType) + if (MapUtils.isNotEmpty(input)) vertex.setMetadata(result.getMetadata) else vertex.setMetadata(new util.HashMap[String, AnyRef]()) + if (StringUtils.isBlank(vertex.getIdentifier)) vertex.setIdentifier(Identifier.getIdentifier(graphId, Identifier.getUniqueIdFromTimestamp)) + setEdges(vertex, result.getRelations) + if (MapUtils.isNotEmpty(result.getExternalData)) vertex.setExternalData(result.getExternalData) else vertex.setExternalData(new util.HashMap[String, AnyRef]()) + vertex + } + def getExternalProps(): List[String] = { if (schemaValidator.getConfig.hasPath("external.properties")) { val propsSet = Set.empty ++ schemaValidator.getConfig.getObject("external.properties").keySet().asScala diff --git a/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/schema/DefinitionNode.scala b/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/schema/DefinitionNode.scala index 7471d5635..42d4886af 100644 --- a/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/schema/DefinitionNode.scala +++ b/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/schema/DefinitionNode.scala @@ -10,7 +10,7 @@ import org.sunbird.cache.impl.RedisCache import org.sunbird.common.JsonUtils import org.sunbird.common.dto.Request import org.sunbird.graph.OntologyEngineContext -import org.sunbird.graph.dac.model.{Node, Relation} +import org.sunbird.graph.dac.model.{Edges, Node, Relation, Vertex} import scala.collection.JavaConversions._ import scala.collection.JavaConverters._ @@ -31,6 +31,19 @@ object DefinitionNode { definition.validate(inputNode, "create", setDefaultValue) recoverWith { case e: CompletionException => throw e.getCause } } + def validates(request: Request, setDefaultValue: Boolean = true)(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Vertex] = { + val graphId: String = request.getContext.get("graph_id").asInstanceOf[String] + val version: String = request.getContext.get("version").asInstanceOf[String] + val schemaName: String = request.getContext.get("schemaName").asInstanceOf[String] + val objectCategoryDefinition: ObjectCategoryDefinition = getObjectCategoryDefinition(request.getRequest.getOrDefault("primaryCategory", "").asInstanceOf[String], + schemaName, request.getContext.getOrDefault("channel", "all").asInstanceOf[String]) + val definition = DefinitionFactory.getDefinition(graphId, schemaName, version, objectCategoryDefinition) + definition.validateRequest(request) + val inputNode = definition.getVertex(request.getRequest) + updateEdgeMetadata(inputNode) + definition.validateVertex(inputNode, "create", setDefaultValue) recoverWith { case e: CompletionException => throw e.getCause } + } + def getExternalProps(graphId: String, version: String, schemaName: String, ocd: ObjectCategoryDefinition = ObjectCategoryDefinition())(implicit ec: ExecutionContext, oec: OntologyEngineContext): List[String] = { val definition = DefinitionFactory.getDefinition(graphId, schemaName, version, ocd) definition.getExternalProps() @@ -77,6 +90,14 @@ object DefinitionNode { definition.getNode(request.get("identifier").asInstanceOf[String], "read", if (request.getRequest.containsKey("mode")) request.get("mode").asInstanceOf[String] else "read", None, disableCache) } + def getVertex(request: Request)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Vertex] = { + val schemaName: String = request.getContext.get("schemaName").asInstanceOf[String] + val definition = DefinitionFactory.getDefinition(request.getContext.get("graph_id").asInstanceOf[String] + , schemaName, request.getContext.get("version").asInstanceOf[String]) + val disableCache: Option[Boolean] = if (request.getRequest.containsKey("disableCache")) request.get("disableCache").asInstanceOf[Option[Boolean]] else None + definition.getVertex(request.get("identifier").asInstanceOf[String], "read", if (request.getRequest.containsKey("mode")) request.get("mode").asInstanceOf[String] else "read", None, disableCache) + } + @throws[Exception] def validate(identifier: String, request: Request)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Node] = { val graphId: String = request.getContext.get("graph_id").asInstanceOf[String] @@ -121,6 +142,51 @@ object DefinitionNode { }).flatMap(f => f) } + @throws[Exception] + def validates(identifier: String, request: Request)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Vertex] = { + val graphId: String = request.getContext.get("graph_id").asInstanceOf[String] + val version: String = request.getContext.get("version").asInstanceOf[String] + val schemaName: String = request.getContext.get("schemaName").asInstanceOf[String].replaceAll("image", "") + val reqVersioning: String = request.getContext.getOrDefault("versioning", "").asInstanceOf[String] + val versioning = if (StringUtils.isBlank(reqVersioning)) None else Option(reqVersioning) + val req: util.HashMap[String, AnyRef] = new util.HashMap[String, AnyRef](request.getRequest) + val skipValidation: Boolean = { + if (request.getContext.containsKey("skipValidation")) request.getContext.get("skipValidation").asInstanceOf[Boolean] else false + } + val definition = DefinitionFactory.getDefinition(graphId, schemaName, version) + val removeProps = request.getContext.getOrDefault("removeProps", new util.ArrayList[String]()).asInstanceOf[util.List[String]] + + definition.getVertex(identifier, "update", null, versioning, None).map(dbVertex => { + val schema = dbVertex.getObjectType.toLowerCase.replace("image", "") + val primaryCategory: String = if (null != dbVertex.getMetadata) dbVertex.getMetadata.getOrDefault("primaryCategory", "").asInstanceOf[String] else "" + val objectCategoryDefinition: ObjectCategoryDefinition = getObjectCategoryDefinition(primaryCategory, schema, request.getContext.getOrDefault("channel", "all").asInstanceOf[String]) + val categoryDefinition = DefinitionFactory.getDefinition(graphId, schema, version, objectCategoryDefinition) + categoryDefinition.validateRequest(request) + resetVertexJsonProperties(dbVertex, graphId, version, schema, objectCategoryDefinition) + val inputVertex: Vertex = categoryDefinition.getVertex(dbVertex.getIdentifier, request.getRequest, dbVertex.getVertexType) + val dbRels = getDBEdges(graphId, schema, version, req, dbVertex, objectCategoryDefinition) + setEdges(dbVertex, inputVertex, dbRels) + if (dbVertex.getIdentifier.endsWith(".img") && StringUtils.equalsAnyIgnoreCase("Yes", dbVertex.getMetadata.getOrDefault("isImageNodeCreated", "").asInstanceOf[String])) { + inputVertex.getMetadata.put("versionKey", dbVertex.getMetadata.getOrDefault("versionKey", "")) + dbVertex.getMetadata.remove("isImageNodeCreated") + } + dbVertex.getMetadata.putAll(inputVertex.getMetadata) + if (MapUtils.isNotEmpty(inputVertex.getExternalData)) { + if (MapUtils.isNotEmpty(dbVertex.getExternalData)) + dbVertex.getExternalData.putAll(inputVertex.getExternalData) + else + dbVertex.setExternalData(inputVertex.getExternalData) + } + if (!removeProps.isEmpty) removeProps.toList.foreach(prop => dbVertex.getMetadata.remove(prop)) + val validatedVertex = if (!skipValidation) categoryDefinition.validateVertex(dbVertex, "update") else Future(dbVertex) + validatedVertex.map(node => { + if (!removeProps.isEmpty) removeProps.toList.foreach(prop => dbVertex.getMetadata.put(prop, null)) + node + }) + + }).flatMap(f => f) + } + def postProcessor(request: Request, node: Node)(implicit ec: ExecutionContext, oec: OntologyEngineContext): Node = { val graphId: String = request.getContext.get("graph_id").asInstanceOf[String] val version: String = request.getContext.get("version").asInstanceOf[String] @@ -146,6 +212,31 @@ object DefinitionNode { node } + def postProcessor(request: Request, vertex: Vertex)(implicit ec: ExecutionContext, oec: OntologyEngineContext): Vertex = { + val graphId: String = request.getContext.get("graph_id").asInstanceOf[String] + val version: String = request.getContext.get("version").asInstanceOf[String] + val schemaName: String = request.getContext.get("schemaName").asInstanceOf[String] + val primaryCategory: String = if (null != vertex.getMetadata) vertex.getMetadata.getOrDefault("primaryCategory", "").asInstanceOf[String] else "" + val objectCategoryDefinition: ObjectCategoryDefinition = getObjectCategoryDefinition(primaryCategory, schemaName, request.getContext.getOrDefault("channel", "all").asInstanceOf[String]) + val categoryDefinition = DefinitionFactory.getDefinition(graphId, schemaName, version, objectCategoryDefinition) + val edgeKey = categoryDefinition.getEdgeKey() + if (null != edgeKey && !edgeKey.isEmpty) { + val metadata = vertex.getMetadata + val cacheKey = "edge_" + request.getObjectType.toLowerCase() + val data = metadata.containsKey(edgeKey) match { + case true => List[String](metadata.get(edgeKey).asInstanceOf[String]) + case _ => List[String]() + } + if (!data.isEmpty) { + metadata.get("status") match { + case "Live" => RedisCache.addToList(cacheKey, data) + case "Retired" => RedisCache.removeFromList(cacheKey, data) + } + } + } + vertex + } + private def setRelationship(dbNode: Node, inputNode: Node, dbRels: util.Map[String, util.List[Relation]]): Unit = { var addRels: util.List[Relation] = new util.ArrayList[Relation]() var delRels: util.List[Relation] = new util.ArrayList[Relation]() @@ -175,6 +266,35 @@ object DefinitionNode { dbNode.setDeletedRelations(delRels) } + private def setEdges(dbVertex: Vertex, inputVertex: Vertex, dbRels: util.Map[String, util.List[Edges]]): Unit = { + var addRels: util.List[Edges] = new util.ArrayList[Edges]() + var delRels: util.List[Edges] = new util.ArrayList[Edges]() + val inRel: util.List[Edges] = dbVertex.getInEdges + val outRel: util.List[Edges] = dbVertex.getOutEdges + val inRelReq: util.List[Edges] = if (CollectionUtils.isNotEmpty(inputVertex.getInEdges)) new util.ArrayList[Edges](inputVertex.getInEdges) else new util.ArrayList[Edges]() + val outRelReq: util.List[Edges] = if (CollectionUtils.isNotEmpty(inputVertex.getOutEdges)) new util.ArrayList[Edges](inputVertex.getOutEdges) else new util.ArrayList[Edges]() + if (CollectionUtils.isNotEmpty(inRelReq)) { + if (CollectionUtils.isNotEmpty(dbRels.get("in"))) { + inRelReq.addAll(dbRels.get("in")) + inputVertex.setInEdges(inRelReq) + } + getNewEdgesList(inRel, inRelReq, addRels, delRels) + } + if (CollectionUtils.isNotEmpty(outRelReq)) { + if (CollectionUtils.isNotEmpty(dbRels.get("out"))) { + outRelReq.addAll(dbRels.get("out")) + inputVertex.setOutEdges(outRelReq) + } + getNewEdgesList(outRel, outRelReq, addRels, delRels) + } + if (CollectionUtils.isNotEmpty(addRels)) { + dbVertex.setAddedEdges(addRels) + updateEdgeMetadata(dbVertex) + } + if (CollectionUtils.isNotEmpty(delRels)) + dbVertex.setDeletedEdges(delRels) + } + private def getNewRelationsList(dbRelations: util.List[Relation], newRelations: util.List[Relation], addRels: util.List[Relation], delRels: util.List[Relation]): Unit = { val relList = new util.ArrayList[String] for (rel <- newRelations) { @@ -190,6 +310,21 @@ object DefinitionNode { } } + private def getNewEdgesList(dbEdges: util.List[Edges], newEdges: util.List[Edges], addEdges: util.List[Edges], delEdges: util.List[Edges]): Unit = { + val edgeList = new util.ArrayList[String] + for (edge <- newEdges) { + addEdges.add(edge) + val relKey = edge.getStartVertexId + edge.getEdgeType + edge.getEndVertexId + if (!edgeList.contains(relKey)) edgeList.add(relKey) + } + if (null != dbEdges && !dbEdges.isEmpty) { + for (rel <- dbEdges) { + val relKey = rel.getStartVertexId + rel.getEdgeType + rel.getEndVertexId + if (!edgeList.contains(relKey)) delEdges.add(rel) + } + } + } + def updateRelationMetadata(node: Node): Unit = { var relOcr = new util.HashMap[String, Integer]() val rels = node.getAddedRelations @@ -206,6 +341,22 @@ object DefinitionNode { node.setAddedRelations(rels) } + def updateEdgeMetadata(vertex: Vertex): Unit = { + var relOcr = new util.HashMap[String, Integer]() + val rels = vertex.getAddedEdges + for (rel <- rels) { + val relKey = rel.getStartVertexObjectType + rel.getEdgeType + rel.getEndVertexObjectType + if (relOcr.containsKey(relKey)) + relOcr.put(relKey, relOcr.get(relKey) + 1) + else relOcr.put(relKey, 1) + if (relKey.contains("hasSequenceMember")) { + val index = if (rel.getMetadata.containsKey("index")) rel.getMetadata.get("index").asInstanceOf[Integer] else relOcr.get(relKey) + rel.setMetadata(Map[String, AnyRef]("IL_SEQUENCE_INDEX" -> index).asJava) + } else rel.setMetadata(new util.HashMap[String, AnyRef]()) + } + vertex.setAddedEdges(rels) + } + def resetJsonProperties(node: Node, graphId: String, version: String, schemaName: String, ocd: ObjectCategoryDefinition = ObjectCategoryDefinition())(implicit ec: ExecutionContext, oec: OntologyEngineContext): Node = { val jsonPropList = fetchJsonProps(graphId, version, schemaName, ocd) if (!jsonPropList.isEmpty) { @@ -221,6 +372,21 @@ object DefinitionNode { node } + def resetVertexJsonProperties(vertex: Vertex, graphId: String, version: String, schemaName: String, ocd: ObjectCategoryDefinition = ObjectCategoryDefinition())(implicit ec: ExecutionContext, oec: OntologyEngineContext): Vertex = { + val jsonPropList = fetchJsonProps(graphId, version, schemaName, ocd) + if (!jsonPropList.isEmpty) { + vertex.getMetadata.entrySet().map(entry => { + if (jsonPropList.contains(entry.getKey)) { + entry.getValue match { + case value: String => entry.setValue(JsonUtils.deserialize(value.asInstanceOf[String], classOf[Object])) + case _ => entry + } + } + }) + } + vertex + } + def getDBRelations(graphId: String, schemaName: String, version: String, request: util.Map[String, AnyRef], dbNode: Node, ocd: ObjectCategoryDefinition = ObjectCategoryDefinition())(implicit ec: ExecutionContext, oec: OntologyEngineContext): util.Map[String, util.List[Relation]] = { val inRelations = new util.ArrayList[Relation]() val outRelations = new util.ArrayList[Relation]() @@ -257,6 +423,43 @@ object DefinitionNode { } } + def getDBEdges(graphId: String, schemaName: String, version: String, request: util.Map[String, AnyRef], dbVertex: Vertex, ocd: ObjectCategoryDefinition = ObjectCategoryDefinition())(implicit ec: ExecutionContext, oec: OntologyEngineContext): util.Map[String, util.List[Edges]] = { + val inEdges = new util.ArrayList[Edges]() + val outEdges = new util.ArrayList[Edges]() + val relDefMap = getRelationDefinitionMap(graphId, version, schemaName, ocd); + if (null != dbVertex) { + if (CollectionUtils.isNotEmpty(dbVertex.getInEdges)) { + for (inRel <- dbVertex.getInEdges) { + val key = inRel.getEdgeType + "_in_" + inRel.getStartVertexObjectType + if (relDefMap.containsKey(key)) { + val value = relDefMap.get(key).get + if (!request.containsKey(value)) { + inEdges.add(inRel) + } + } + } + } + if (CollectionUtils.isNotEmpty(dbVertex.getOutEdges)) { + for (outRel <- dbVertex.getOutEdges) { + val key = outRel.getEdgeType + "_out_" + outRel.getEndVertexObjectType + if (relDefMap.containsKey(key)) { + val value = relDefMap.get(key).get + if (!request.containsKey(value)) { + outEdges.add(outRel) + } + } + } + } + } + new util.HashMap[String, util.List[Edges]]() { + { + put("in", inEdges) + put("out", outEdges) + } + } + } + + def validateContentNodes(nodes: List[Node], graphId: String, schemaName: String, version: String)(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[List[Node]] = { val futures = nodes.map(node => { val ocd = ObjectCategoryDefinition(node.getMetadata.getOrDefault("primaryCategory", "").asInstanceOf[String], node.getObjectType, node.getMetadata.getOrDefault("channel", "all").asInstanceOf[String]) diff --git a/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/schema/IDefinition.scala b/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/schema/IDefinition.scala index b173f0fe2..df36fa398 100644 --- a/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/schema/IDefinition.scala +++ b/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/schema/IDefinition.scala @@ -1,7 +1,7 @@ package org.sunbird.graph.schema import org.sunbird.graph.OntologyEngineContext -import org.sunbird.graph.dac.model.Node +import org.sunbird.graph.dac.model.{Node, Vertex} import org.sunbird.schema.{ISchemaValidator, SchemaValidatorFactory} import scala.concurrent.{ExecutionContext, Future} @@ -12,12 +12,16 @@ abstract class IDefinition(graphId: String, schemaName: String, version: String def getNode(input: java.util.Map[String, AnyRef]): Node + def getVertex(input: java.util.Map[String, AnyRef]): Vertex @throws[Exception] def validate(node: Node, operation: String = "update", setDefaultValue: Boolean = true)(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Node] - + @throws[Exception] + def validateVertex(vertex: Vertex, operation: String = "update", setDefaultValue: Boolean = true)(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Vertex] @throws[Exception] def getNode(identifier: String, operation: String = "read", mode: String, versioning: Option[String] = None, disableCache: Option[Boolean] = None)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Node] + def getVertex(identifier: String, operation: String = "read", mode: String, versioning: Option[String] = None, disableCache: Option[Boolean] = None)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Vertex] + def getSchemaName(): String ={ schemaName } diff --git a/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/schema/validator/BaseDefinitionNode.scala b/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/schema/validator/BaseDefinitionNode.scala index 10a11200d..8f1561025 100644 --- a/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/schema/validator/BaseDefinitionNode.scala +++ b/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/schema/validator/BaseDefinitionNode.scala @@ -1,14 +1,13 @@ package org.sunbird.graph.schema.validator import java.util - import org.apache.commons.collections4.{CollectionUtils, MapUtils} import org.apache.commons.lang3.StringUtils import org.sunbird.common.dto.Request import org.sunbird.graph.OntologyEngineContext import org.sunbird.graph.common.Identifier import org.sunbird.graph.dac.enums.SystemNodeTypes -import org.sunbird.graph.dac.model.{Node, Relation} +import org.sunbird.graph.dac.model.{Edges, Node, Relation, Vertex} import org.sunbird.graph.schema.{IDefinition, ObjectCategoryDefinition} import scala.collection.JavaConverters._ @@ -53,17 +52,45 @@ class BaseDefinitionNode(graphId: String, schemaName: String, version: String = node } + override def getVertex(input: java.util.Map[String, Object]): Vertex = { + val result = schemaValidator.getStructuredData(input) + val vertex = new Vertex(graphId, result.getMetadata) + val objectType = schemaValidator.getConfig.getString("objectType") + vertex.setVertexType(SystemNodeTypes.DATA_NODE.name) + vertex.setObjectType(objectType) + vertex.setIdentifier(input.getOrDefault("identifier", Identifier.getIdentifier(graphId, Identifier.getUniqueIdFromTimestamp)).asInstanceOf[String]) + input.remove("identifier") + setEdges(vertex, result.getRelations) + if (CollectionUtils.isNotEmpty(vertex.getInEdges)) vertex.setAddedEdges(vertex.getInEdges) + if (CollectionUtils.isNotEmpty(vertex.getOutEdges)) vertex.setAddedEdges(vertex.getOutEdges) + vertex.setExternalData(result.getExternalData) + vertex + } + @throws[Exception] override def validate(node: Node, operation: String, setDefaultValue: Boolean)(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Node] = { Future{node} } + @throws[Exception] + override def validateVertex(vertex: Vertex, operation: String, setDefaultValue: Boolean)(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Vertex] = { + Future { + vertex + } + } + override def getNode(identifier: String, operation: String, mode: String, versioning: Option[String] = None, disableCache: Option[Boolean] = None)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Node] = { val request: Request = new Request() val node: Future[Node] = oec.graphService.getNodeByUniqueId(graphId, identifier, false, request) node } + override def getVertex(identifier: String, operation: String, mode: String, versioning: Option[String] = None, disableCache: Option[Boolean] = None)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Vertex] = { + val request: Request = new Request() + val vertex: Future[Vertex] = oec.janusGraphService.getNodeByUniqueId(graphId, identifier, false, request) + vertex + } + protected def setRelations(node: Node, relations: java.util.Map[String, AnyRef]): Unit = { if (MapUtils.isNotEmpty(relations)) { @@ -92,4 +119,33 @@ class BaseDefinitionNode(graphId: String, schemaName: String, version: String = node.setOutRelations(outRelations) } } + + protected def setEdges(vertex: Vertex, edges: java.util.Map[String, AnyRef]): Unit = { + if (MapUtils.isNotEmpty(edges)) { + def getEdges(schema: Map[String, AnyRef], direction: String): List[Edges] = { + edges.asScala.filterKeys(key => schema.keySet.contains(key)) + .flatten(entry => { + val relSchema = schema.get(entry._1).get.asInstanceOf[java.util.Map[String, AnyRef]].asScala + val relData = entry._2.asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]] + relData.asScala.map(r => { + val relation = { + if (StringUtils.equalsAnyIgnoreCase("out", direction)) { + new Edges(vertex.getIdentifier, relSchema.get("type").get.asInstanceOf[String], r.get("identifier").asInstanceOf[String]) + .updateMetadata((r.asScala - "identifier").asJava) + } else { + new Edges(r.get("identifier").asInstanceOf[String], relSchema.get("type").get.asInstanceOf[String], vertex.getIdentifier) + .updateMetadata((r.asScala - "identifier").asJava) + } + } + relation + }) + }).toList + } + + val inRelations = getEdges(inRelationsSchema, "in").asJava + vertex.setInEdges(inRelations) + val outRelations = getEdges(outRelationsSchema, "out").asJava + vertex.setOutEdges(outRelations) + } + } } diff --git a/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/schema/validator/SchemaValidator.scala b/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/schema/validator/SchemaValidator.scala index 6eab97c08..e803f286a 100644 --- a/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/schema/validator/SchemaValidator.scala +++ b/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/schema/validator/SchemaValidator.scala @@ -1,7 +1,7 @@ package org.sunbird.graph.schema.validator import org.sunbird.graph.OntologyEngineContext -import org.sunbird.graph.dac.model.Node +import org.sunbird.graph.dac.model.{Node, Vertex} import org.sunbird.graph.schema.IDefinition import scala.concurrent.{ExecutionContext, Future} @@ -19,4 +19,17 @@ trait SchemaValidator extends IDefinition { super.validate(node, operation) } + + @throws[Exception] + abstract override def validateVertex(vertex: Vertex, operation: String, setDefaultValue: Boolean)(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Vertex] = { + if (setDefaultValue) { + val result = schemaValidator.validate(vertex.getMetadata) + if (setDefaultValue && operation.equalsIgnoreCase("create")) { + vertex.setMetadata(result.getMetadata) + } + } + + super.validateVertex(vertex, operation) + } + } diff --git a/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/schema/validator/VersioningNode.scala b/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/schema/validator/VersioningNode.scala index 0e81b68f5..c378517f2 100644 --- a/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/schema/validator/VersioningNode.scala +++ b/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/schema/validator/VersioningNode.scala @@ -2,14 +2,13 @@ package org.sunbird.graph.schema.validator import java.util import java.util.concurrent.CompletionException - import org.sunbird.cache.impl.RedisCache import org.sunbird.common.{DateUtils, JsonUtils, Platform} import org.sunbird.common.dto.{Request, ResponseHandler} import org.sunbird.common.exception.ResourceNotFoundException import org.sunbird.graph.OntologyEngineContext import org.sunbird.graph.common.enums.AuditProperties -import org.sunbird.graph.dac.model.Node +import org.sunbird.graph.dac.model.{Node, Vertex} import org.sunbird.graph.exception.GraphErrorCodes import org.sunbird.graph.external.ExternalPropsManager import org.sunbird.graph.schema.{DefinitionFactory, IDefinition} @@ -36,6 +35,14 @@ trait VersioningNode extends IDefinition { } } + abstract override def getVertex(identifier: String, operation: String, mode: String = "read", versioning: Option[String] = None, disableCache: Option[Boolean] = None)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Vertex] = { + operation match { + case "update" => getVertexToUpdate(identifier, versioning); + case "read" => getVertexToRead(identifier, mode, disableCache) + case _ => getVertexToRead(identifier, mode, disableCache) + } + } + private def getNodeToUpdate(identifier: String, versioning: Option[String] = None)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Node] = { val nodeFuture: Future[Node] = super.getNode(identifier , "update", null) nodeFuture.map(node => { @@ -49,6 +56,23 @@ trait VersioningNode extends IDefinition { }).flatMap(f => f) } + private def getVertexToUpdate(identifier: String, versioning: Option[String] = None)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Vertex] = { + val vertexFuture: Future[Vertex] = super.getVertex(identifier, "update", null) + vertexFuture.map(vertex => { + val versioningEnable = versioning.getOrElse({ + if (schemaValidator.getConfig.hasPath("version")) schemaValidator.getConfig.getString("version") else "disable" + }) + if (null == vertex) + throw new ResourceNotFoundException(GraphErrorCodes.ERR_INVALID_NODE.toString, "Node Not Found With Identifier : " + identifier) + else if ("enable".equalsIgnoreCase(versioningEnable)) + getEditableVertex(identifier, vertex) + else + Future { + vertex + } + }).flatMap(f => f) + } + private def getNodeToRead(identifier: String, mode: String, disableCache: Option[Boolean])(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Node] = { if ("edit".equalsIgnoreCase(mode)) { val imageNode = super.getNode(identifier + IMAGE_SUFFIX, "read", mode) @@ -71,11 +95,39 @@ trait VersioningNode extends IDefinition { } } } + private def getVertexToRead(identifier: String, mode: String, disableCache: Option[Boolean])(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Vertex] = { + println("IN getVertexToRead") + if ("edit".equalsIgnoreCase(mode)) { + val imageVertex = super.getVertex(identifier + IMAGE_SUFFIX, "read", mode) + imageVertex recoverWith { + case e: CompletionException => { + if (e.getCause.isInstanceOf[ResourceNotFoundException]) + super.getVertex(identifier, "read", mode) + else + throw e.getCause + } + } + } else { + if (disableCache.nonEmpty) { + if (disableCache.get) super.getVertex(identifier, "read", mode) + else getVertexFromCache(identifier) + } else { + val cacheKey = getSchemaName().toLowerCase() + ".cache.enable" + if (Platform.getBoolean(cacheKey, false)) getVertexFromCache(identifier) + else super.getVertex(identifier, "read", mode) + } + } + } private def getNodeFromCache(identifier: String)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Node]= { val ttl: Integer = if (Platform.config.hasPath(getSchemaName().toLowerCase() + ".cache.ttl")) Platform.config.getInt(getSchemaName().toLowerCase() + ".cache.ttl") else 86400 getCachedNode(identifier, ttl) } + private def getVertexFromCache(identifier: String)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Vertex] = { + val ttl: Integer = if (Platform.config.hasPath(getSchemaName().toLowerCase() + ".cache.ttl")) Platform.config.getInt(getSchemaName().toLowerCase() + ".cache.ttl") else 86400 + getCachedVertex(identifier, ttl) + } + private def getEditableNode(identifier: String, node: Node)(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Node] = { val status = node.getMetadata.get("status").asInstanceOf[String] if(statusList.contains(status)) { @@ -110,6 +162,42 @@ trait VersioningNode extends IDefinition { Future{node} } + private def getEditableVertex(identifier: String, vertex: Vertex)(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Vertex] = { + val status = vertex.getMetadata.get("status").asInstanceOf[String] + if (statusList.contains(status)) { + val imageId = vertex.getIdentifier + IMAGE_SUFFIX + try { + val imageVertex = oec.janusGraphService.getNodeByUniqueId(vertex.getGraphId, imageId, false, new Request()) + imageVertex recoverWith { + case e: CompletionException => { + TelemetryManager.error("Exception occurred while fetching image node, may not be found", e.getCause) + if (e.getCause.isInstanceOf[ResourceNotFoundException]) { + vertex.setIdentifier(imageId) + vertex.setObjectType(vertex.getObjectType + IMAGE_OBJECT_SUFFIX) + vertex.getMetadata.put("status", "Draft") + vertex.getMetadata.put("prevStatus", status) + vertex.getMetadata.put(AuditProperties.lastStatusChangedOn.name, DateUtils.formatCurrentDate()) + oec.janusGraphService.addVertex(vertex.getGraphId, vertex).map(imgVertex => { + imgVertex.getMetadata.put("isImageNodeCreated", "yes"); + copyExternalProps(identifier, vertex.getGraphId, imgVertex.getObjectType.toLowerCase().replace("image", "")).map(response => { + if (!ResponseHandler.checkError(response)) { + if (null != response.getResult && !response.getResult.isEmpty) + imgVertex.setExternalData(response.getResult) + } + imgVertex + }) + }).flatMap(f => f) + } else + throw e.getCause + } + } + } + } else + Future { + vertex + } + } + private def copyExternalProps(identifier: String, graphId: String, schemaName: String)(implicit ec: ExecutionContext, oec: OntologyEngineContext) = { val request = new Request() request.setContext(new util.HashMap[String, AnyRef](){{ @@ -144,6 +232,22 @@ trait VersioningNode extends IDefinition { }).flatMap(f => f) } + def getCachedVertex(identifier: String, ttl: Integer)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Vertex] = { + val nodeStringFuture: Future[String] = RedisCache.getAsync(identifier, nodeCacheAsyncHandler, ttl) + nodeStringFuture.map(nodeString => { + if (null != nodeString && !nodeString.asInstanceOf[String].isEmpty) { + val nodeMap: util.Map[String, AnyRef] = JsonUtils.deserialize(nodeString.asInstanceOf[String], classOf[java.util.Map[String, AnyRef]]) + val vertex: Vertex = NodeUtil.deserializeVertex(nodeMap, getSchemaName(), schemaValidator.getConfig + .getAnyRef("relations").asInstanceOf[java.util.Map[String, AnyRef]]) + Future { + vertex + } + } else { + super.getVertex(identifier, "read", null) + } + }).flatMap(f => f) + } + private def nodeCacheAsyncHandler(objKey: String)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[String] = { super.getNode(objKey, "read", null).map(node => { if (List("Live", "Unlisted").contains(node.getMetadata.get("status").asInstanceOf[String])) { diff --git a/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/utils/NodeUtil.scala b/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/utils/NodeUtil.scala index dc0d53869..866bb3c86 100644 --- a/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/utils/NodeUtil.scala +++ b/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/utils/NodeUtil.scala @@ -1,7 +1,6 @@ package org.sunbird.graph.utils import java.util - import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.scala.DefaultScalaModule import org.apache.commons.collections4.{CollectionUtils, MapUtils} @@ -9,7 +8,7 @@ import org.apache.commons.lang3.StringUtils import org.sunbird.common.{JsonUtils, Platform} import org.sunbird.graph.OntologyEngineContext import org.sunbird.graph.common.enums.SystemProperties -import org.sunbird.graph.dac.model.{Node, Relation} +import org.sunbird.graph.dac.model.{Edges, Node, Relation, Vertex} import org.sunbird.graph.schema.{DefinitionNode, ObjectCategoryDefinition, ObjectCategoryDefinitionMap} import scala.collection.JavaConverters @@ -40,6 +39,26 @@ object NodeUtil { finalMetadata } + def serializeVertex(vertex: Vertex, fields: util.List[String], schemaName: String, schemaVersion: String, withoutRelations: Boolean = false)(implicit oec: OntologyEngineContext, ec: ExecutionContext): util.Map[String, AnyRef] = { + val metadataMap = vertex.getMetadata + val objectCategoryDefinition: ObjectCategoryDefinition = DefinitionNode.getObjectCategoryDefinition(vertex.getMetadata.getOrDefault("primaryCategory", "").asInstanceOf[String], vertex.getObjectType.toLowerCase().replace("image", ""), vertex.getMetadata.getOrDefault("channel", "all").asInstanceOf[String]) + val jsonProps = DefinitionNode.fetchJsonProps(vertex.getGraphId, schemaVersion, vertex.getObjectType.toLowerCase().replace("image", ""), objectCategoryDefinition) + val updatedMetadataMap: util.Map[String, AnyRef] = metadataMap.entrySet().asScala.filter(entry => null != entry.getValue).map((entry: util.Map.Entry[String, AnyRef]) => handleKeyNames(entry, fields) -> convertJsonProperties(entry, jsonProps)).toMap.asJava + val definitionMap = DefinitionNode.getRelationDefinitionMap(vertex.getGraphId, schemaVersion, vertex.getObjectType.toLowerCase().replace("image", ""), objectCategoryDefinition).asJava + val finalMetadata = new util.HashMap[String, AnyRef]() + finalMetadata.put("objectType", vertex.getObjectType) + finalMetadata.putAll(updatedMetadataMap) + if (!withoutRelations) { + val relMap: util.Map[String, util.List[util.Map[String, AnyRef]]] = getRelationMap(vertex, updatedMetadataMap, definitionMap) + finalMetadata.putAll(relMap) + } + if (CollectionUtils.isNotEmpty(fields)) + finalMetadata.keySet.retainAll(fields) + finalMetadata.put("identifier", vertex.getIdentifier) + finalMetadata.put("languageCode", getLanguageCodes(vertex)) + finalMetadata + } + def setRelation(node: Node, nodeMap: util.Map[String, AnyRef], relationMap: util.Map[String, AnyRef]) = { val inRelations: util.List[Relation] = new util.ArrayList[Relation]() @@ -93,6 +112,66 @@ object NodeUtil { node.setOutRelations(outRelations) } + def setEdges(vertex: Vertex, nodeMap: util.Map[String, AnyRef], relationMap: util.Map[String, AnyRef]) = { + val inRelations: util.List[Edges] = new util.ArrayList[Edges]() + val outRelations: util.List[Edges] = new util.ArrayList[Edges]() + relationMap.asScala.foreach(entry => { + if (nodeMap.containsKey(entry._1) && null != nodeMap.get(entry._1) && !nodeMap.get(entry._1).asInstanceOf[util.List[util.Map[String, AnyRef]]].isEmpty) { + nodeMap.get(entry._1).asInstanceOf[util.List[util.Map[String, AnyRef]]].asScala.map(relMap => { + if ("in".equalsIgnoreCase(entry._2.asInstanceOf[util.Map[String, AnyRef]].get("direction").asInstanceOf[String])) { + val rel: Edges = new Edges(relMap.get("identifier").asInstanceOf[String], entry._2.asInstanceOf[util.Map[String, AnyRef]].get("type").asInstanceOf[String], vertex.getIdentifier) + rel.setStartVertexObjectType(relMap.get("objectType").asInstanceOf[String]) + rel.setEndVertexObjectType(vertex.getObjectType) + rel.setStartVertexName(relMap.get("name").asInstanceOf[String]) + rel.setStartVertexMetadata(new util.HashMap[String, AnyRef]() { + { + put("description", relMap.get("description")) + put("status", relMap.get("status")) + } + }) + if (null != relMap.get("index") && 0 < relMap.get("index").asInstanceOf[Integer]) { + rel.setMetadata(new util.HashMap[String, AnyRef]() { + { + put(SystemProperties.IL_SEQUENCE_INDEX.name(), relMap.get("index")) + } + }) + } + inRelations.add(rel) + } else { + val rel: Edges = new Edges(vertex.getIdentifier, entry._2.asInstanceOf[util.Map[String, AnyRef]].get("type").asInstanceOf[String], relMap.get("identifier").asInstanceOf[String]) + rel.setStartVertexObjectType(vertex.getObjectType) + rel.setEndVertexObjectType(relMap.get("objectType").asInstanceOf[String]) + rel.setStartVertexName(relMap.get("name").asInstanceOf[String]) + rel.setStartVertexMetadata(new util.HashMap[String, AnyRef]() { + { + put("description", relMap.get("description")) + put("status", relMap.get("status")) + } + }) + val index: Integer = { + if (null != relMap.get("index")) { + if (relMap.get("index").isInstanceOf[String]) { + Integer.parseInt(relMap.get("index").asInstanceOf[String]) + } else relMap.get("index").asInstanceOf[Number].intValue() + } else + null + } + if (null != index && 0 < index) { + rel.setMetadata(new util.HashMap[String, AnyRef]() { + { + put(SystemProperties.IL_SEQUENCE_INDEX.name(), relMap.get("index")) + } + }) + } + outRelations.add(rel) + } + }) + } + }) + vertex.setInEdges(inRelations) + vertex.setOutEdges(outRelations) + } + def deserialize(nodeMap: util.Map[String, AnyRef], schemaName: String, relationMap:util.Map[String, AnyRef]): Node = { val node: Node = new Node() if(MapUtils.isNotEmpty(nodeMap)) { @@ -109,6 +188,22 @@ object NodeUtil { node } + def deserializeVertex(nodeMap: util.Map[String, AnyRef], schemaName: String, relationMap: util.Map[String, AnyRef]): Vertex = { + val vertex: Vertex = new Vertex() + if (MapUtils.isNotEmpty(nodeMap)) { + vertex.setIdentifier(nodeMap.get("identifier").asInstanceOf[String]) + vertex.setObjectType(nodeMap.get("objectType").asInstanceOf[String]) + val filteredMetadata: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef](JavaConverters.mapAsJavaMapConverter(nodeMap.asScala.filterNot(entry => relationMap.containsKey(entry._1)).toMap).asJava) + vertex.setMetadata(filteredMetadata) + setEdges(vertex, nodeMap, relationMap) + } + vertex.getMetadata.asScala.map(entry => { + if (entry._2.isInstanceOf[::[AnyRef]]) (entry._1 -> entry._2.asInstanceOf[::[AnyRef]].toArray.toList) + else entry + }) + vertex + } + def handleKeyNames(entry: util.Map.Entry[String, AnyRef], fields: util.List[String]) = { if(CollectionUtils.isEmpty(fields)) { @@ -142,6 +237,39 @@ object NodeUtil { } relMap } + + def getRelationMap(vertex: Vertex, updatedMetadataMap: util.Map[String, AnyRef], relationMap: util.Map[String, AnyRef]): util.Map[String, util.List[util.Map[String, AnyRef]]] = { + val inRelations: util.List[Edges] = { + if (CollectionUtils.isEmpty(vertex.getInEdges)) new util.ArrayList[Edges] else vertex.getInEdges + } + val outRelations: util.List[Edges] = { + if (CollectionUtils.isEmpty(vertex.getOutEdges)) new util.ArrayList[Edges] else vertex.getOutEdges + } + val relMap = new util.HashMap[String, util.List[util.Map[String, AnyRef]]] + for (rel <- inRelations.asScala) { + val relKey: String = rel.getEdgeType + "_in_" + rel.getStartVertexObjectType + if (relMap.containsKey(relationMap.get(relKey))) relMap.get(relationMap.get(relKey)).add(populateRelationMaps(rel, "in")) + else { + if (null != relationMap.get(relKey)) { + relMap.put(relationMap.get(relKey).asInstanceOf[String], new util.ArrayList[util.Map[String, AnyRef]]() { + add(populateRelationMaps(rel, "in")) + }) + } + } + } + for (rel <- outRelations.asScala) { + val relKey: String = rel.getEdgeType + "_out_" + rel.getEndVertexObjectType + if (relMap.containsKey(relationMap.get(relKey))) relMap.get(relationMap.get(relKey)).add(populateRelationMaps(rel, "out")) + else { + if (null != relationMap.get(relKey)) { + relMap.put(relationMap.get(relKey).asInstanceOf[String], new util.ArrayList[util.Map[String, AnyRef]]() { + add(populateRelationMaps(rel, "out")) + }) + } + } + } + relMap + } def convertJsonProperties(entry: util.Map.Entry[String, AnyRef], jsonProps: scala.List[String]) = { if(jsonProps.contains(entry.getKey)) { @@ -176,6 +304,26 @@ object NodeUtil { } } + def populateRelationMaps(rel: Edges, direction: String): util.Map[String, AnyRef] = { + if ("out".equalsIgnoreCase(direction)) { + val objectType = rel.getEndVertexObjectType.replace("Image", "") + val relData = Map("identifier" -> rel.getEndVertexId.replace(".img", ""), + "name" -> rel.getEndVertexName, + "objectType" -> objectType, + "relation" -> rel.getEdgeType) ++ relationObjectAttributes(objectType).map(key => (key -> rel.getEndVertexMetadata.get(key))).toMap + val indexMap = if (rel.getEdgeType.equals("hasSequenceMember")) Map("index" -> rel.getMetadata.getOrDefault("IL_SEQUENCE_INDEX", 1.asInstanceOf[Number]).asInstanceOf[Number]) else Map() + val completeRelData = relData ++ indexMap + mapAsJavaMap(completeRelData) + } else { + val objectType = rel.getStartVertexObjectType.replace("Image", "") + val relData = Map("identifier" -> rel.getStartVertexId.replace(".img", ""), + "name" -> rel.getStartVertexName, + "objectType" -> objectType, + "relation" -> rel.getEdgeType) ++ relationObjectAttributes(objectType).map(key => (key -> rel.getStartVertexMetadata.get(key))).toMap + mapAsJavaMap(relData) + } + } + def getLanguageCodes(node: Node): util.List[String] = { val value = node.getMetadata.get("language") val languages:util.List[String] = value match { @@ -191,6 +339,21 @@ object NodeUtil { } } + def getLanguageCodes(vertex: Vertex): util.List[String] = { + val value = vertex.getMetadata.get("language") + val languages: util.List[String] = value match { + case value: String => List(value).asJava + case value: util.List[String] => value + case value: Array[String] => value.filter((lng: String) => StringUtils.isNotBlank(lng)).toList.asJava + case _ => new util.ArrayList[String]() + } + if (CollectionUtils.isNotEmpty(languages)) { + JavaConverters.bufferAsJavaListConverter(languages.asScala.map(lang => if (Platform.config.hasPath("languageCode." + lang.toLowerCase)) Platform.config.getString("languageCode." + lang.toLowerCase) else "")).asJava + } else { + languages + } + } + def isRetired(node: Node): Boolean = StringUtils.equalsIgnoreCase(node.getMetadata.get("status").asInstanceOf[String], "Retired") } diff --git a/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/vertex/DataVertex.scala b/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/vertex/DataVertex.scala new file mode 100644 index 000000000..d7c99efa1 --- /dev/null +++ b/ontology-engine/graph-engine_2.12/src/main/scala/org/sunbird/graph/vertex/DataVertex.scala @@ -0,0 +1,269 @@ +package org.sunbird.graph.vertex + +import org.apache.commons.collections4.{CollectionUtils, MapUtils} +import org.apache.commons.lang3.StringUtils +import org.sunbird.common.dto.{Request, Response} +import org.sunbird.common.exception.{ClientException, ErrorCodes, ResponseCode} +import org.sunbird.graph.OntologyEngineContext +import org.sunbird.graph.dac.model.{Edges, Vertex} +import org.sunbird.graph.nodes.DataNode.{saveExternalProperties, updateExternalProperties} +import org.sunbird.graph.schema.{DefinitionDTO, DefinitionFactory, DefinitionNode} +import org.sunbird.parseq.Task + +import java.util +import java.util.Optional +import java.util.concurrent.CompletionException +import scala.collection.JavaConverters._ +import scala.collection.convert.ImplicitConversions._ +import scala.concurrent.{ExecutionContext, Future} +object DataVertex { + + private val SYSTEM_UPDATE_ALLOWED_CONTENT_STATUS = List("Live", "Unlisted") + + @throws[Exception] + def create(request: Request, dataModifier: (Vertex) => Vertex = defaultVertexDataModifier)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Vertex] = { + DefinitionNode.validates(request).map(vertex => { + val response = oec.janusGraphService.addVertex(request.graphId, dataModifier(vertex)) + response.map(vertex => DefinitionNode.postProcessor(request, vertex)).map(result => { + val futureList = Task.parallel[Response]( + saveExternalProperties(vertex.getIdentifier, vertex.getExternalData, request.getContext, request.getObjectType), + createEdges(request.graphId, vertex, request.getContext)) + futureList.map(list => result) + }).flatMap(f => f) recoverWith { case e: CompletionException => throw e.getCause } + }).flatMap(f => f) + } + + @throws[Exception] + def read(request: Request)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Vertex] = { + DefinitionNode.getVertex(request).map(vertex => { + val schema = vertex.getObjectType.toLowerCase.replace("image", "") + val objectType: String = request.getContext.get("objectType").asInstanceOf[String] + request.getContext.put("schemaName", schema) + val fields: List[String] = Optional.ofNullable(request.get("fields").asInstanceOf[util.List[String]]).orElse(new util.ArrayList[String]()).toList + val version: String = if (null != vertex && null != vertex.getMetadata) { + val schemaVersion: String = vertex.getMetadata.getOrDefault("schemaVersion", "0.0").asInstanceOf[String] + val scVer = if (StringUtils.isNotBlank(schemaVersion) && schemaVersion.toDouble != 0.0) schemaVersion else request.getContext.get("version").asInstanceOf[String] + scVer + } else request.getContext.get("version").asInstanceOf[String] + val extPropNameList = DefinitionNode.getExternalProps(request.getContext.get("graph_id").asInstanceOf[String], version, schema) + if (CollectionUtils.isNotEmpty(extPropNameList) && null != fields && fields.exists(field => extPropNameList.contains(field))) + populateExternalProperties(fields, vertex, request, extPropNameList) + else + Future(vertex) + }).flatMap(f => f) recoverWith { + case e: CompletionException => throw e.getCause + } + } + + def deleteVertex(request: Request)(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[java.lang.Boolean] = { + val identifier: String = request.getRequest.getOrDefault("identifier", "").asInstanceOf[String] + oec.janusGraphService.deleteNode(request.graphId, identifier, request) + } + + def update(request: Request, dataModifier: (Vertex) => Vertex = defaultVertexDataModifier)(implicit oec: OntologyEngineContext, ec: ExecutionContext): Future[Vertex] = { + val identifier: String = request.getContext.get("identifier").asInstanceOf[String] + DefinitionNode.validates(identifier, request).map(vertex => { + request.getContext.put("schemaName", vertex.getObjectType.toLowerCase.replace("image", "")) + val response = oec.janusGraphService.upsertVertex(request.graphId, dataModifier(vertex), request) + response.map(vertex => DefinitionNode.postProcessor(request, vertex)) + .map(result => { + val futureList = Task.parallel[Response]( + updateExternalProperties(vertex.getIdentifier, vertex.getExternalData, request.getContext, request.getObjectType, request), + updateEdges(request.graphId, vertex, request.getContext) + ) + futureList.map(list => result) + }).flatMap(f => f) recoverWith { case e: CompletionException => throw e.getCause } + }).flatMap(f => f) recoverWith { case e: CompletionException => throw e.getCause } + } + + def bulkUpdate(request: Request)(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[util.Map[String, Vertex]] = { + val identifiers: util.List[String] = request.get("identifiers").asInstanceOf[util.List[String]] + val metadata: util.Map[String, AnyRef] = request.get("metadata").asInstanceOf[util.Map[String, AnyRef]] + oec.janusGraphService.updateVertexes(request.graphId, identifiers, metadata) + } +/* private def updateEdges(vertex: Vertex, graphId: String, context: util.Map[String, AnyRef])(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Response] = { + val request = new Request // Assuming Request is required for JanusGraph + request.setContext(context) + + if (vertex.getAddedEdges.isEmpty && vertex.getDeletedEdges.isEmpty) { + Future(new Response) // No changes, return empty response + } else { + val futures = Seq( + // Delete edges if any + if (vertex.getDeletedEdges.nonEmpty) { + oec.janusGraphService.removeEdges(graphId, getEdgesMap(vertex.getDeletedEdges)) + } else Future.successful(Unit), + // Add edges if any + if (vertex.getAddedEdges.nonEmpty) { + oec.janusGraphService.createEdges(graphId, getEdgesMap(vertex.getAddedEdges)) + } else Future.successful(Unit) + ) + Future.sequence(futures) // Combine deletion and addition calls into one future + .map(_ => new Response) // Wrap the combined future with a new Response + } + + }*/ + + private def createEdges(graphId: String, vertex: Vertex, context: util.Map[String, AnyRef])(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Response] = { + val edges: util.List[Edges] = vertex.getAddedEdges + if (CollectionUtils.isNotEmpty(edges)) { + oec.janusGraphService.createEdges(graphId, getEdgesMap(edges)) + } else { + Future(new Response) + } + } + + def updateEdges(graphId: String, vertex: Vertex, context: util.Map[String, AnyRef])(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Response] = { + if (CollectionUtils.isEmpty(vertex.getAddedEdges) && CollectionUtils.isEmpty(vertex.getDeletedEdges)) { + Future(new Response) + } else { + if (CollectionUtils.isNotEmpty(vertex.getDeletedEdges)) { + oec.janusGraphService.removeEdges(graphId, getEdgesMap(vertex.getDeletedEdges)) + } + if (CollectionUtils.isNotEmpty(vertex.getAddedEdges)) { + oec.janusGraphService.createEdges(graphId, getEdgesMap(vertex.getAddedEdges)) + } + Future(new Response) + } + } + + private def getEdgesMap(edges: util.List[Edges]): java.util.List[util.Map[String, AnyRef]] = { + val list = new util.ArrayList[util.Map[String, AnyRef]] + for (edge <- edges) { + if ((StringUtils.isNotBlank(edge.getStartVertexId) && StringUtils.isNotBlank(edge.getEndVertexId)) && StringUtils.isNotBlank(edge.getEdgeType)) { + val map = new util.HashMap[String, AnyRef] + map.put("startNodeId", edge.getStartVertexId) + map.put("endNodeId", edge.getEndVertexId) + map.put("relation", edge.getEdgeType) + if (MapUtils.isNotEmpty(edge.getMetadata)) map.put("relMetadata", edge.getMetadata) + else map.put("relMetadata", new util.HashMap[String, AnyRef]()) + list.add(map) + } + else throw new ClientException("ERR_INVALID_RELATION_OBJECT", "Invalid Relation Object Found.") + } + list + } + + private def defaultVertexDataModifier(vertex: Vertex) = { + vertex + } + + private def defaultDataModifier(vertex: Vertex) = { + vertex + } + +// def systemUpdate(request: Request, vertexList: util.List[Vertex], hierarchyKey: String, hierarchyFunc: Option[Request => Future[Response]] = None)(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Vertex] = { +// val data: util.Map[String, AnyRef] = request.getRequest +// +// // validate nodes +// validateVertex(vertexList, request) +// +// // get definition for the object and filter relations +// val definition = getDefinition(request) +// val metadata = filterRelations(definition, data) +// +// // get status +// val status = getStatus(request, vertexList) +// // Generate request for new metadata +// val newRequest = new Request(request) +// newRequest.putAll(metadata) +// newRequest.getContext.put("versioning", "disabled") +// // Enrich Hierarchy and Update the nodes +// vertexList.map(vertex => { +// enrichHierarchyAndUpdate(newRequest, vertex, status, hierarchyKey, hierarchyFunc) +// }).head +// +// } +// +// @throws[Exception] +// private def enrichHierarchyAndUpdate(request: Request, vertex: Vertex, status: String, hierarchyKey: String, hierarchyFunc: Option[Request => Future[Response]] = None)(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Vertex] = { +// val metadata: util.Map[String, AnyRef] = request.getRequest +// val identifier = vertex.getIdentifier +// // Image node cannot be made Live or Unlisted using system call +// if (identifier.endsWith(".img") && +// SYSTEM_UPDATE_ALLOWED_CONTENT_STATUS.contains(status)) metadata.remove("status") +// if (metadata.isEmpty) throw new ClientException(ErrorCodes.ERR_BAD_REQUEST.name(), s"Invalid Request. Cannot update status of Image Node to $status.") +// +// // Update previous status and status update Timestamp +// if (metadata.containsKey("status")) { +// metadata.put("prevStatus", vertex.getMetadata.get("status")) +// metadata.put("lastStatusChangedOn", DateUtils.formatCurrentDate) +// } +// // Generate new request object for Each request +// val newRequest = new Request(request) +// newRequest.putAll(metadata) +// newRequest.getContext.put("identifier", identifier) +// // Enrich Hierarchy and Update with the new request +// enrichHierarchy(newRequest, metadata, status, hierarchyKey: String, hierarchyFunc) +// .flatMap(req => update(req)) recoverWith { case e: CompletionException => throw e.getCause } +// } + + private def enrichHierarchy(request: Request, metadata: util.Map[String, AnyRef], status: String, hierarchyKey: String, hierarchyFunc: Option[Request => Future[Response]] = None)(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Request] = { + val identifier = request.getContext.get("identifier").asInstanceOf[String] + // Check if hierarchy could be enriched + if (!identifier.endsWith(".img") && SYSTEM_UPDATE_ALLOWED_CONTENT_STATUS.contains(status)) { + hierarchyFunc match { + case Some(hierarchyFunc) => { + // Get current Hierarchy + val hierarchyRequest = new Request(request) + hierarchyRequest.put("rootId", identifier) + hierarchyFunc(hierarchyRequest).map(response => { + // Add metadata to the hierarchy + if (response.get(hierarchyKey) != null) { + val hierarchy = response.get(hierarchyKey).asInstanceOf[util.Map[String, AnyRef]] + val hierarchyMetadata = new util.HashMap[String, AnyRef]() + hierarchyMetadata.putAll(hierarchy) + hierarchyMetadata.putAll(metadata) + // add hierarchy to the request object + request.put("hierarchy", hierarchyMetadata) + request + } else request + }) + } + case _ => Future(request) + } + } else Future(request) + } + + def validateVertex(vertexs: java.util.List[Vertex], request: Request): Unit = { + if (vertexs.isEmpty) + throw new ClientException(ResponseCode.RESOURCE_NOT_FOUND.name(), s"Error! Node(s) doesn't Exists with identifier : ${request.getContext.get("identifier")}.") + + val objectType = request.getContext.get("objectType").asInstanceOf[String] + vertexs.foreach(vertex => { + if (vertex.getMetadata == null && !objectType.equalsIgnoreCase(vertex.getObjectType) && vertex.getMetadata.get("status").asInstanceOf[String].equalsIgnoreCase("failed")) + throw new ClientException(ErrorCodes.ERR_BAD_REQUEST.name(), s"Cannot update content with FAILED status for id : ${vertex.getIdentifier}.") + }) + } + + private def getStatus(request: Request, vertexList: util.List[Vertex]): String = { + val vertex = vertexList.filter(node => !node.getIdentifier.endsWith(".img")).headOption.getOrElse(vertexList.head) + request.getOrDefault("status", vertex.getMetadata.get("status")).asInstanceOf[String] + } + + private def getDefinition(request: Request)(implicit ec: ExecutionContext, oec: OntologyEngineContext): DefinitionDTO = { + val schemaName: String = request.getContext.get("schemaName").asInstanceOf[String] + val version = request.getContext.get("version").asInstanceOf[String] + DefinitionFactory.getDefinition(request.graphId, schemaName, version) + } + + private def filterRelations(definition: DefinitionDTO, data: util.Map[String, AnyRef]): util.Map[String, AnyRef] = { + val relations = definition.getRelationsMap().keySet() + data.filter(item => { + !relations.contains(item._1) + }) + } + + private def populateExternalProperties(fields: List[String], vertex: Vertex, request: Request, externalProps: List[String])(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Vertex] = { + if (StringUtils.equalsIgnoreCase(request.get("mode").asInstanceOf[String], "edit")) + request.put("identifier", vertex.getIdentifier) + val externalPropsResponse = oec.graphService.readExternalProps(request, externalProps.filter(prop => fields.contains(prop))) + externalPropsResponse.map(response => { + vertex.getMetadata.putAll(response.getResult) + Future { + vertex + } + }).flatMap(f => f) + } + +} diff --git a/ontology-engine/pom.xml b/ontology-engine/pom.xml index 3dc8826f6..59e194dd1 100644 --- a/ontology-engine/pom.xml +++ b/ontology-engine/pom.xml @@ -13,6 +13,7 @@ graph-common graph-dac-api + graph-dac graph-core_2.12 graph-engine_2.12 parseq diff --git a/taxonomy-api/taxonomy-actors/src/main/scala/org/sunbird/actors/CategoryActor.scala b/taxonomy-api/taxonomy-actors/src/main/scala/org/sunbird/actors/CategoryActor.scala index 49d9a5e84..34ead38c5 100644 --- a/taxonomy-api/taxonomy-actors/src/main/scala/org/sunbird/actors/CategoryActor.scala +++ b/taxonomy-api/taxonomy-actors/src/main/scala/org/sunbird/actors/CategoryActor.scala @@ -13,6 +13,7 @@ import org.sunbird.graph.nodes.DataNode import org.sunbird.utils.{Constants, RequestUtil} import org.sunbird.mangers.FrameworkManager import org.sunbird.cache.impl.RedisCache +import org.sunbird.graph.vertex.DataVertex import scala.concurrent.{ExecutionContext, Future} @@ -37,14 +38,14 @@ class CategoryActor @Inject()(implicit oec: OntologyEngineContext) extends BaseA request.getRequest.put(Constants.IDENTIFIER, code) RedisCache.delete("masterCategories") FrameworkManager.validateTranslationMap(request) - DataNode.create(request).map(node => { + DataVertex.create(request).map(node => { ResponseHandler.OK.put(Constants.IDENTIFIER, node.getIdentifier).put(Constants.NODE_ID, node.getIdentifier) }) } private def read(request: Request): Future[Response] = { - DataNode.read(request).map(node => { - val metadata: util.Map[String, AnyRef] = NodeUtil.serialize(node, null, request.getContext.get("schemaName").asInstanceOf[String], request.getContext.get("version").asInstanceOf[String]) + DataVertex.read(request).map(node => { + val metadata: util.Map[String, AnyRef] = NodeUtil.serializeVertex(node, null, request.getContext.get("schemaName").asInstanceOf[String], request.getContext.get("version").asInstanceOf[String]) ResponseHandler.OK.put("category", metadata) }) } diff --git a/taxonomy-api/taxonomy-actors/src/main/scala/org/sunbird/actors/CategoryInstanceActor.scala b/taxonomy-api/taxonomy-actors/src/main/scala/org/sunbird/actors/CategoryInstanceActor.scala index 6833c37f7..3d1bc4194 100644 --- a/taxonomy-api/taxonomy-actors/src/main/scala/org/sunbird/actors/CategoryInstanceActor.scala +++ b/taxonomy-api/taxonomy-actors/src/main/scala/org/sunbird/actors/CategoryInstanceActor.scala @@ -8,7 +8,7 @@ import org.sunbird.common.dto.{Request, Response, ResponseHandler} import org.sunbird.common.exception.ClientException import org.sunbird.graph.OntologyEngineContext import org.sunbird.graph.dac.enums.RelationTypes -import org.sunbird.graph.dac.model.Node +import org.sunbird.graph.dac.model.{Node, Vertex} import org.sunbird.graph.nodes.DataNode import org.sunbird.graph.utils.NodeUtil import org.sunbird.utils.{Constants, RequestUtil} @@ -73,6 +73,14 @@ class CategoryInstanceActor @Inject()(implicit oec: OntologyEngineContext) exten if (indexList.nonEmpty) indexList.max + 1 else 1 } + private def getCategoryIndex(node: Vertex): Integer = { + val indexList = (node.getOutEdges.asScala ++ node.getInEdges.asScala).filter(r => (StringUtils.equals(r.getEdgeType, RelationTypes.SEQUENCE_MEMBERSHIP.relationName()) && StringUtils.equals(r.getStartVertexId, node.getIdentifier))) + .map(relation => { + relation.getMetadata.getOrDefault("IL_SEQUENCE_INDEX", 1.asInstanceOf[Number]).toString.toInt.intValue() + }) + if (indexList.nonEmpty) indexList.max + 1 else 1 + } + private def read(request: Request): Future[Response] = { validateCategoryInstanceObject(request).map(node => { val metadata: util.Map[String, AnyRef] = NodeUtil.serialize(node, null, request.getContext.get("schemaName").asInstanceOf[String], request.getContext.get("version").asInstanceOf[String]) diff --git a/taxonomy-api/taxonomy-actors/src/main/scala/org/sunbird/actors/ObjectCategoryActor.scala b/taxonomy-api/taxonomy-actors/src/main/scala/org/sunbird/actors/ObjectCategoryActor.scala index 4b91c9df9..544880f75 100644 --- a/taxonomy-api/taxonomy-actors/src/main/scala/org/sunbird/actors/ObjectCategoryActor.scala +++ b/taxonomy-api/taxonomy-actors/src/main/scala/org/sunbird/actors/ObjectCategoryActor.scala @@ -1,7 +1,6 @@ package org.sunbird.actors import java.util - import javax.inject.Inject import org.apache.commons.lang3.StringUtils import org.sunbird.actor.core.BaseActor @@ -11,6 +10,7 @@ import org.sunbird.common.exception.ClientException import org.sunbird.graph.OntologyEngineContext import org.sunbird.graph.nodes.DataNode import org.sunbird.graph.utils.NodeUtil +import org.sunbird.graph.vertex.DataVertex import org.sunbird.utils.{Constants, RequestUtil} import scala.collection.JavaConverters @@ -35,7 +35,7 @@ class ObjectCategoryActor @Inject()(implicit oec: OntologyEngineContext) extends RequestUtil.restrictProperties(request) if (!request.getRequest.containsKey(Constants.NAME)) throw new ClientException("ERR_NAME_SET_AS_IDENTIFIER", "name will be set as identifier") request.getRequest.put(Constants.IDENTIFIER, Constants.CATEGORY_PREFIX + Slug.makeSlug(request.getRequest.get(Constants.NAME).asInstanceOf[String])) - DataNode.create(request).map(node => { + DataVertex.create(request).map(node => { ResponseHandler.OK.put(Constants.IDENTIFIER, node.getIdentifier) }) } @@ -44,8 +44,8 @@ class ObjectCategoryActor @Inject()(implicit oec: OntologyEngineContext) extends private def read(request: Request): Future[Response] = { val fields: util.List[String] = JavaConverters.seqAsJavaListConverter(request.get(Constants.FIELDS).asInstanceOf[String].split(",").filter(field => StringUtils.isNotBlank(field) && !StringUtils.equalsIgnoreCase(field, "null"))).asJava request.getRequest.put(Constants.FIELDS, fields) - DataNode.read(request).map(node => { - val metadata: util.Map[String, AnyRef] = NodeUtil.serialize(node, fields, request.getContext.get(Constants.SCHEMA_NAME).asInstanceOf[String], request.getContext.get(Constants.VERSION).asInstanceOf[String]) + DataVertex.read(request).map(node => { + val metadata: util.Map[String, AnyRef] = NodeUtil.serializeVertex(node, fields, request.getContext.get(Constants.SCHEMA_NAME).asInstanceOf[String], request.getContext.get(Constants.VERSION).asInstanceOf[String]) ResponseHandler.OK.put(Constants.OBJECT_CATEGORY, metadata) }) } @@ -53,7 +53,7 @@ class ObjectCategoryActor @Inject()(implicit oec: OntologyEngineContext) extends @throws[Exception] private def update(request: Request): Future[Response] = { RequestUtil.restrictProperties(request) - DataNode.update(request).map(node => { + DataVertex.update(request).map(node => { ResponseHandler.OK.put(Constants.IDENTIFIER, node.getIdentifier) }) } diff --git a/taxonomy-api/taxonomy-actors/src/main/scala/org/sunbird/actors/TermActor.scala b/taxonomy-api/taxonomy-actors/src/main/scala/org/sunbird/actors/TermActor.scala index 32f892d3f..e7ae664bc 100644 --- a/taxonomy-api/taxonomy-actors/src/main/scala/org/sunbird/actors/TermActor.scala +++ b/taxonomy-api/taxonomy-actors/src/main/scala/org/sunbird/actors/TermActor.scala @@ -7,7 +7,7 @@ import org.sunbird.common.dto.{Request, Response, ResponseHandler} import org.sunbird.common.exception.{ClientException, ResponseCode } import org.sunbird.graph.OntologyEngineContext import org.sunbird.graph.dac.enums.RelationTypes -import org.sunbird.graph.dac.model.Node +import org.sunbird.graph.dac.model.{Node, Vertex} import org.sunbird.graph.nodes.DataNode import org.sunbird.graph.utils.NodeUtil import org.sunbird.utils.{Constants, RequestUtil} @@ -105,6 +105,14 @@ class TermActor @Inject()(implicit oec: OntologyEngineContext) extends BaseActor if (indexList.nonEmpty) indexList.max + 1 else 1 } + private def getIndex(node: Vertex): Integer = { + val indexList = (node.getOutEdges.asScala ++ node.getInEdges.asScala).filter(r => (StringUtils.equals(r.getEdgeType, RelationTypes.SEQUENCE_MEMBERSHIP.relationName()) && StringUtils.equals(r.getStartVertexId, node.getIdentifier))) + .map(relation => { + relation.getMetadata.getOrDefault("IL_SEQUENCE_INDEX", 1.asInstanceOf[Number]).toString.toInt.intValue() + }) + if (indexList.nonEmpty) indexList.max + 1 else 1 + } + private def read(request: Request): Future[Response] = { validateCategoryInstance(request) validateTerm(request).map(node => { diff --git a/taxonomy-api/taxonomy-actors/src/main/scala/org/sunbird/mangers/FrameworkManager.scala b/taxonomy-api/taxonomy-actors/src/main/scala/org/sunbird/mangers/FrameworkManager.scala index dd6773a85..b9e83432f 100644 --- a/taxonomy-api/taxonomy-actors/src/main/scala/org/sunbird/mangers/FrameworkManager.scala +++ b/taxonomy-api/taxonomy-actors/src/main/scala/org/sunbird/mangers/FrameworkManager.scala @@ -6,10 +6,9 @@ import org.sunbird.cache.impl.RedisCache import org.sunbird.common.{JsonUtils, Platform} import org.sunbird.common.dto.{Request, Response, ResponseHandler} import org.sunbird.common.exception.{ClientException, ServerException} -import org.sunbird.graph.OntologyEngineContext -import org.sunbird.graph.dac.model.{Relation, SubGraph} +import org.sunbird.graph.{JanusGraphService, OntologyEngineContext} +import org.sunbird.graph.dac.model.{Edges, Relation, SubGraph, VertexSubGraph} import org.sunbird.graph.nodes.DataNode - import org.sunbird.graph.schema.{DefinitionNode, ObjectCategoryDefinition} import org.sunbird.graph.utils.NodeUtil import org.sunbird.graph.utils.NodeUtil.{convertJsonProperties, handleKeyNames} @@ -113,6 +112,48 @@ object FrameworkManager { } } + def getCompleteMetadata(id: String, subGraph: VertexSubGraph, includeRelations: Boolean)(implicit oec: OntologyEngineContext, ec: ExecutionContext): util.Map[String, AnyRef] = { + val nodes = subGraph.getVertexs + val relations = subGraph.getEdges + val node = nodes.get(id) + val metadata = node.getMetadata + val objectType = node.getObjectType.toLowerCase().replace("image", "") + val channel = node.getMetadata.getOrDefault("channel", "all").asInstanceOf[String] + val definition: ObjectCategoryDefinition = DefinitionNode.getObjectCategoryDefinition("", objectType, channel) + val jsonProps = DefinitionNode.fetchJsonProps(node.getGraphId, schemaVersion, objectType, definition) + val updatedMetadata: util.Map[String, AnyRef] = metadata.entrySet().asScala.filter(entry => null != entry.getValue) + .map((entry: util.Map.Entry[String, AnyRef]) => handleKeyNames(entry, null) -> convertJsonProperties(entry, jsonProps)).toMap ++ + Map("objectType" -> node.getObjectType, "identifier" -> node.getIdentifier, "languageCode" -> NodeUtil.getLanguageCodes(node)) + + val fields = DefinitionNode.getMetadataFields(node.getGraphId, schemaVersion, objectType, definition) + val filteredData: util.Map[String, AnyRef] = if (fields.nonEmpty) updatedMetadata.filterKeys(key => fields.contains(key)) else updatedMetadata + + val relationDef = DefinitionNode.getRelationDefinitionMap(node.getGraphId, schemaVersion, objectType, definition) + val outRelations = relations.filter((rel: Edges) => { + StringUtils.equals(rel.getStartVertexId.toString(), node.getIdentifier) + }).sortBy((rel: Edges) => rel.getMetadata.getOrDefault("IL_SEQUENCE_INDEX", "1").toString.toInt)(Ordering.Int).toList + + if (includeRelations) { + val relMetadata = getEdgesAsMetadata(relationDef, outRelations, "out") + val childHierarchy = relMetadata.map(x => (x._1, x._2.map(a => { + val identifier = a.getOrElse("identifier", "") + val childNode = nodes.get(identifier) + val index = a.getOrElse("index", 1).asInstanceOf[Number] + val metaData = (childNode.getMetadata ++ Map("index" -> index)).asJava + childNode.setMetadata(metaData) + if ("associations".equalsIgnoreCase(x._1)) { + getCompleteMetadata(childNode.getIdentifier, subGraph, false) + } else { + getCompleteMetadata(childNode.getIdentifier, subGraph, true) + } + }).toList.asJava)) + val data = (filteredData ++ childHierarchy).asJava + data + } else { + filteredData + } + } + def getRelationAsMetadata(definitionMap: Map[String, AnyRef], relationMap: util.List[Relation], direction: String) = { relationMap.asScala.map(rel => { @@ -136,6 +177,30 @@ object FrameworkManager { })).distinct.asJava )) } + def getEdgesAsMetadata(definitionMap: Map[String, AnyRef], relationMap: util.List[Edges], direction: String) = { + relationMap.asScala.map(rel => { + val endObjectType = rel.getEndVertexObjectType.replace("Image", "") + val relKey: String = rel.getEdgeType + "_" + direction + "_" + endObjectType + if (definitionMap.containsKey(relKey)) { + val relData = Map[String, Object]("identifier" -> rel.getEndVertexId.replace(".img", ""), + "name" -> rel.getEndVertexName, + "objectType" -> endObjectType, + "relation" -> rel.getEdgeType, + "KEY" -> definitionMap.getOrDefault(relKey, "").asInstanceOf[String] + ) ++ rel.getMetadata.asScala + val indexMap = if (rel.getEdgeType.equals("hasSequenceMember")) Map("index" -> rel.getMetadata.getOrDefault("IL_SEQUENCE_INDEX", 1.asInstanceOf[Number]).toString.toInt) else Map() + relData ++ indexMap + } else { + Map[String, Object]() + } + }).filter(x => x.nonEmpty) + .groupBy(x => x.getOrDefault("KEY", "").asInstanceOf[String]) + .map(x => (x._1, (x._2.toList.map(x => { + x.-("KEY") + x.-("IL_SEQUENCE_INDEX") + })).distinct.asJava)) + } + def getFrameworkHierarchy(request: Request)(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Map[String, AnyRef]] = { val req = new Request(request) req.put("identifier", request.get("identifier"))