PPL: Add json_extract function

core/build.gradle

@@ -54,6 +54,7 @@ dependencies {
     api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
     api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
     api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
+    api group: 'com.jayway.jsonpath', name: 'json-path', version: '2.9.0'
     api group: 'com.google.code.gson', name: 'gson', version: '2.8.9'
     api group: 'com.tdunning', name: 't-digest', version: '3.3'
     api project(':common')

core/src/main/java/org/opensearch/sql/ast/expression/Cast.java

@@ -13,6 +13,7 @@
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_FLOAT;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_INT;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_IP;
+import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_JSON;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_LONG;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_SHORT;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_STRING;
@@ -56,6 +57,7 @@ public class Cast extends UnresolvedExpression {
           .put("timestamp", CAST_TO_TIMESTAMP.getName())
           .put("datetime", CAST_TO_DATETIME.getName())
           .put("ip", CAST_TO_IP.getName())
+          .put("json", CAST_TO_JSON.getName())
           .build();
 
   /** The source expression cast from. */

core/src/main/java/org/opensearch/sql/expression/DSL.java

@@ -687,6 +687,14 @@ public static FunctionExpression jsonValid(Expression... expressions) {
     return compile(FunctionProperties.None, BuiltinFunctionName.JSON_VALID, expressions);
   }
 
+  public static FunctionExpression jsonExtract(Expression... expressions) {
+    return compile(FunctionProperties.None, BuiltinFunctionName.JSON_EXTRACT, expressions);
+  }
+
+  public static FunctionExpression stringToJson(Expression value) {
+    return compile(FunctionProperties.None, BuiltinFunctionName.JSON, value);
+  }
+
   public static Aggregator avg(Expression... expressions) {
     return aggregate(BuiltinFunctionName.AVG, expressions);
   }
@@ -843,6 +851,10 @@ public static FunctionExpression castIp(Expression value) {
     return compile(FunctionProperties.None, BuiltinFunctionName.CAST_TO_IP, value);
   }
 
+  public static FunctionExpression castJson(Expression value) {
+    return compile(FunctionProperties.None, BuiltinFunctionName.CAST_TO_JSON, value);
+  }
+
   public static FunctionExpression typeof(Expression value) {
     return compile(FunctionProperties.None, BuiltinFunctionName.TYPEOF, value);
   }

core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java

@@ -206,6 +206,8 @@ public enum BuiltinFunctionName {
 
   /** Json Functions. */
   JSON_VALID(FunctionName.of("json_valid")),
+  JSON(FunctionName.of("json")),
+  JSON_EXTRACT(FunctionName.of("json_extract")),
 
   /** GEOSPATIAL Functions. */
   GEOIP(FunctionName.of("geoip")),
@@ -238,6 +240,7 @@ public enum BuiltinFunctionName {
   CAST_TO_TIMESTAMP(FunctionName.of("cast_to_timestamp")),
   CAST_TO_DATETIME(FunctionName.of("cast_to_datetime")),
   CAST_TO_IP(FunctionName.of("cast_to_ip")),
+  CAST_TO_JSON(FunctionName.of("cast_to_json")),
   TYPEOF(FunctionName.of("typeof")),
 
   /** Relevance Function. */

core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java

@@ -7,8 +7,10 @@
 
 import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN;
 import static org.opensearch.sql.data.type.ExprCoreType.STRING;
+import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED;
 import static org.opensearch.sql.expression.function.FunctionDSL.define;
 import static org.opensearch.sql.expression.function.FunctionDSL.impl;
+import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling;
 
 import lombok.experimental.UtilityClass;
 import org.opensearch.sql.expression.function.BuiltinFunctionName;
@@ -20,10 +22,24 @@
 public class JsonFunctions {
   public void register(BuiltinFunctionRepository repository) {
     repository.register(jsonValid());
+    repository.register(jsonFunction());
+    repository.register(jsonExtract());
   }
 
   private DefaultFunctionResolver jsonValid() {
     return define(
         BuiltinFunctionName.JSON_VALID.getName(), impl(JsonUtils::isValidJson, BOOLEAN, STRING));
   }
+
+  private DefaultFunctionResolver jsonFunction() {
+    return define(
+        BuiltinFunctionName.JSON.getName(),
+        impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING));
+  }
+
+  private DefaultFunctionResolver jsonExtract() {
+    return define(
+        BuiltinFunctionName.JSON_EXTRACT.getName(),
+        impl(JsonUtils::extractJson, UNDEFINED, STRING, STRING));
+  }
 }

core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java

@@ -17,6 +17,7 @@
 import static org.opensearch.sql.data.type.ExprCoreType.STRING;
 import static org.opensearch.sql.data.type.ExprCoreType.TIME;
 import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP;
+import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED;
 import static org.opensearch.sql.expression.function.FunctionDSL.impl;
 import static org.opensearch.sql.expression.function.FunctionDSL.implWithProperties;
 import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling;
@@ -42,6 +43,7 @@
 import org.opensearch.sql.expression.function.BuiltinFunctionRepository;
 import org.opensearch.sql.expression.function.DefaultFunctionResolver;
 import org.opensearch.sql.expression.function.FunctionDSL;
+import org.opensearch.sql.utils.JsonUtils;
 
 @UtilityClass
 public class TypeCastOperators {
@@ -57,6 +59,7 @@ public static void register(BuiltinFunctionRepository repository) {
     repository.register(castToDouble());
     repository.register(castToBoolean());
     repository.register(castToIp());
+    repository.register(castToJson());
     repository.register(castToDate());
     repository.register(castToTime());
     repository.register(castToTimestamp());
@@ -105,7 +108,8 @@ private static DefaultFunctionResolver castToShort() {
         impl(
             nullMissingHandling((v) -> new ExprShortValue(v.booleanValue() ? 1 : 0)),
             SHORT,
-            BOOLEAN));
+            BOOLEAN),
+        impl(nullMissingHandling((v) -> v), SHORT, UNDEFINED));
   }
 
   private static DefaultFunctionResolver castToInt() {
@@ -119,7 +123,8 @@ private static DefaultFunctionResolver castToInt() {
         impl(
             nullMissingHandling((v) -> new ExprIntegerValue(v.booleanValue() ? 1 : 0)),
             INTEGER,
-            BOOLEAN));
+            BOOLEAN),
+        impl(nullMissingHandling((v) -> v), INTEGER, UNDEFINED));
   }
 
   private static DefaultFunctionResolver castToLong() {
@@ -133,7 +138,8 @@ private static DefaultFunctionResolver castToLong() {
         impl(
             nullMissingHandling((v) -> new ExprLongValue(v.booleanValue() ? 1L : 0L)),
             LONG,
-            BOOLEAN));
+            BOOLEAN),
+        impl(nullMissingHandling((v) -> v), LONG, UNDEFINED));
   }
 
   private static DefaultFunctionResolver castToFloat() {
@@ -147,7 +153,8 @@ private static DefaultFunctionResolver castToFloat() {
         impl(
             nullMissingHandling((v) -> new ExprFloatValue(v.booleanValue() ? 1f : 0f)),
             FLOAT,
-            BOOLEAN));
+            BOOLEAN),
+        impl(nullMissingHandling((v) -> v), FLOAT, UNDEFINED));
   }
 
   private static DefaultFunctionResolver castToDouble() {
@@ -161,7 +168,8 @@ private static DefaultFunctionResolver castToDouble() {
         impl(
             nullMissingHandling((v) -> new ExprDoubleValue(v.booleanValue() ? 1D : 0D)),
             DOUBLE,
-            BOOLEAN));
+            BOOLEAN),
+        impl(nullMissingHandling((v) -> v), DOUBLE, UNDEFINED));
   }
 
   private static DefaultFunctionResolver castToBoolean() {
@@ -173,7 +181,8 @@ private static DefaultFunctionResolver castToBoolean() {
             STRING),
         impl(
             nullMissingHandling((v) -> ExprBooleanValue.of(v.doubleValue() != 0)), BOOLEAN, DOUBLE),
-        impl(nullMissingHandling((v) -> v), BOOLEAN, BOOLEAN));
+        impl(nullMissingHandling((v) -> v), BOOLEAN, BOOLEAN),
+        impl(nullMissingHandling((v) -> v), BOOLEAN, UNDEFINED));
   }
 
   private static DefaultFunctionResolver castToIp() {
@@ -183,6 +192,12 @@ private static DefaultFunctionResolver castToIp() {
         impl(nullMissingHandling((v) -> v), IP, IP));
   }
 
+  private static DefaultFunctionResolver castToJson() {
+    return FunctionDSL.define(
+        BuiltinFunctionName.CAST_TO_JSON.getName(),
+        impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING));
+  }
+
   private static DefaultFunctionResolver castToDate() {
     return FunctionDSL.define(
         BuiltinFunctionName.CAST_TO_DATE.getName(),

core/src/main/java/org/opensearch/sql/utils/JsonUtils.java

@@ -1,18 +1,42 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
 package org.opensearch.sql.utils;
 
+import static org.opensearch.sql.data.model.ExprValueUtils.*;
+
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.jayway.jsonpath.InvalidJsonException;
+import com.jayway.jsonpath.InvalidPathException;
+import com.jayway.jsonpath.JsonPath;
+import com.jayway.jsonpath.PathNotFoundException;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
 import lombok.experimental.UtilityClass;
+import org.opensearch.sql.data.model.ExprBooleanValue;
+import org.opensearch.sql.data.model.ExprCollectionValue;
+import org.opensearch.sql.data.model.ExprDoubleValue;
+import org.opensearch.sql.data.model.ExprIntegerValue;
+import org.opensearch.sql.data.model.ExprNullValue;
+import org.opensearch.sql.data.model.ExprStringValue;
+import org.opensearch.sql.data.model.ExprTupleValue;
 import org.opensearch.sql.data.model.ExprValue;
 import org.opensearch.sql.data.model.ExprValueUtils;
+import org.opensearch.sql.exception.SemanticCheckException;
 
 @UtilityClass
 public class JsonUtils {
   /**
    * Checks if given JSON string can be parsed as valid JSON.
    *
    * @param jsonExprValue JSON string (e.g. "{\"hello\": \"world\"}").
-   * @return true if the string can be parsed as valid JSON, else false.
+   * @return true if the string can be parsed as valid JSON, else false (including null or missing).
    */
   public static ExprValue isValidJson(ExprValue jsonExprValue) {
     ObjectMapper objectMapper = new ObjectMapper();
@@ -23,9 +47,101 @@ public static ExprValue isValidJson(ExprValue jsonExprValue) {
 
     try {
       objectMapper.readTree(jsonExprValue.stringValue());
-      return ExprValueUtils.LITERAL_TRUE;
+      return LITERAL_TRUE;
     } catch (JsonProcessingException e) {
-      return ExprValueUtils.LITERAL_FALSE;
+      return LITERAL_FALSE;
+    }
+  }
+
+  /**
+   * Converts a JSON encoded string to a {@link ExprValue}. Expression type will be UNDEFINED.
+   *
+   * @param json JSON string (e.g. "{\"hello\": \"world\"}").
+   * @return ExprValue returns an expression that best represents the provided JSON-encoded string.
+   *     <ol>
+   *       <li>{@link ExprTupleValue} if the JSON is an object
+   *       <li>{@link ExprCollectionValue} if the JSON is an array
+   *       <li>{@link ExprDoubleValue} if the JSON is a floating-point number scalar
+   *       <li>{@link ExprIntegerValue} if the JSON is an integral number scalar
+   *       <li>{@link ExprStringValue} if the JSON is a string scalar
+   *       <li>{@link ExprBooleanValue} if the JSON is a boolean scalar
+   *       <li>{@link ExprNullValue} if the JSON is null, empty, or invalid
+   *     </ol>
+   */
+  public static ExprValue castJson(ExprValue json) {
+    ObjectMapper objectMapper = new ObjectMapper();
+    JsonNode jsonNode;
+    try {
+      jsonNode = objectMapper.readTree(json.stringValue());
+    } catch (JsonProcessingException e) {
+      final String errorFormat = "JSON string '%s' is not valid. Error details: %s";
+      throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e);
+    }
+
+    return processJsonNode(jsonNode);
+  }
+
+  /**
+   * Extract value of JSON string at given JSON path.
+   *
+   * @param json JSON string (e.g. "{\"hello\": \"world\"}").
+   * @param path JSON path (e.g. "$.hello")
+   * @return ExprValue of value at given path of json string.
+   */
+  public static ExprValue extractJson(ExprValue json, ExprValue path) {
+    if (json == LITERAL_NULL || json == LITERAL_MISSING) {
+      return json;
+    }
+
+    String jsonString = json.stringValue();
+    String jsonPath = path.stringValue();
+
+    if (jsonString.isEmpty()) {
+      return LITERAL_NULL;
+    }
+
+    try {
+      Object results = JsonPath.parse(jsonString).read(jsonPath);
+      return ExprValueUtils.fromObjectValue(results);
+    } catch (PathNotFoundException e) {
+      return LITERAL_NULL;
+    } catch (InvalidPathException e) {
+      final String errorFormat = "JSON path '%s' is not valid. Error details: %s";
+      throw new SemanticCheckException(String.format(errorFormat, path, e.getMessage()), e);
+    } catch (InvalidJsonException e) {
+      final String errorFormat = "JSON string '%s' is not valid. Error details: %s";
+      throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e);
+    }
+  }
+
+  private static ExprValue processJsonNode(JsonNode jsonNode) {
+    switch (jsonNode.getNodeType()) {
+      case ARRAY:
+        List<ExprValue> elements = new LinkedList<>();
+        for (var iter = jsonNode.iterator(); iter.hasNext(); ) {
+          jsonNode = iter.next();
+          elements.add(processJsonNode(jsonNode));
+        }
+        return new ExprCollectionValue(elements);
+      case OBJECT:
+        Map<String, ExprValue> values = new LinkedHashMap<>();
+        for (var iter = jsonNode.fields(); iter.hasNext(); ) {
+          Map.Entry<String, JsonNode> entry = iter.next();
+          values.put(entry.getKey(), processJsonNode(entry.getValue()));
+        }
+        return ExprTupleValue.fromExprValueMap(values);
+      case STRING:
+        return new ExprStringValue(jsonNode.asText());
+      case NUMBER:
+        if (jsonNode.isFloatingPointNumber()) {
+          return new ExprDoubleValue(jsonNode.asDouble());
+        }
+        return new ExprIntegerValue(jsonNode.asLong());
+      case BOOLEAN:
+        return jsonNode.asBoolean() ? LITERAL_TRUE : LITERAL_FALSE;
+      default:
+        // in all other cases, return null
+        return LITERAL_NULL;
     }
   }
 }