From 70152eb653c2ea2e09428d602af68e55e63af290 Mon Sep 17 00:00:00 2001 From: Kenrick Yap <14yapkc1@gmail.com> Date: Fri, 3 Jan 2025 10:43:05 -0800 Subject: [PATCH 01/70] added implementation Signed-off-by: Kenrick Yap <14yapkc1@gmail.com> --- .../org/opensearch/sql/expression/DSL.java | 4 ++ .../function/BuiltinFunctionName.java | 3 ++ .../function/BuiltinFunctionRepository.java | 2 + .../sql/expression/json/JsonFunctions.java | 46 +++++++++++++++++++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 3 ++ ppl/src/main/antlr/OpenSearchPPLParser.g4 | 1 + 6 files changed, 59 insertions(+) create mode 100644 core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index 44ecc2bc86..fc97f35d11 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -683,6 +683,10 @@ public static FunctionExpression notLike(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.NOT_LIKE, expressions); } + public static FunctionExpression jsonValid(Expression... expressions){ + return compile(FunctionProperties.None, BuiltinFunctionName.JSON_VALID, expressions); + } + public static Aggregator avg(Expression... expressions) { return aggregate(BuiltinFunctionName.AVG, expressions); } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index f8e9cf7c5f..43fdbf2eb7 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -204,6 +204,9 @@ public enum BuiltinFunctionName { TRIM(FunctionName.of("trim")), UPPER(FunctionName.of("upper")), + /** Json Functions. */ + JSON_VALID(FunctionName.of("json_valid")), + /** NULL Test. */ IS_NULL(FunctionName.of("is null")), IS_NOT_NULL(FunctionName.of("is not null")), diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java index 79ea58b860..72d637fd2b 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java @@ -28,6 +28,7 @@ import org.opensearch.sql.expression.datetime.DateTimeFunctions; import org.opensearch.sql.expression.datetime.IntervalClause; import org.opensearch.sql.expression.ip.IPFunctions; +import org.opensearch.sql.expression.json.JsonFunctions; import org.opensearch.sql.expression.operator.arthmetic.ArithmeticFunctions; import org.opensearch.sql.expression.operator.arthmetic.MathematicalFunctions; import org.opensearch.sql.expression.operator.convert.TypeCastOperators; @@ -83,6 +84,7 @@ public static synchronized BuiltinFunctionRepository getInstance() { SystemFunctions.register(instance); OpenSearchFunctions.register(instance); IPFunctions.register(instance); + JsonFunctions.register(instance); } return instance; } diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java new file mode 100644 index 0000000000..e4ba7af499 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -0,0 +1,46 @@ +package org.opensearch.sql.expression.json; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.experimental.UtilityClass; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.BuiltinFunctionRepository; +import org.opensearch.sql.expression.function.DefaultFunctionResolver; + +import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.expression.function.FunctionDSL.define; +import static org.opensearch.sql.expression.function.FunctionDSL.impl; +import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; + +@UtilityClass +public class JsonFunctions { + public void register(BuiltinFunctionRepository repository) { + + repository.register(jsonValid()); + } + + private DefaultFunctionResolver jsonValid() { + return define( + BuiltinFunctionName.JSON_VALID.getName(), + impl(nullMissingHandling(JsonFunctions::isValidJson), BOOLEAN, STRING)); + } + + /** + * Checks if given JSON string can be parsed as valid JSON. + * + * @param jsonExprValue JSON string (e.g. "198.51.100.14" or "2001:0db8::ff00:42:8329"). + * @return true if the string can be parsed as valid JSON, else false. + */ + private ExprValue isValidJson(ExprValue jsonExprValue) { + ObjectMapper objectMapper = new ObjectMapper(); + try { + objectMapper.readTree(jsonExprValue.stringValue()); + return ExprValueUtils.LITERAL_TRUE; + } catch (Exception e) { + return ExprValueUtils.LITERAL_FALSE; + } + } +} diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 053ec530db..5b6b9e41b8 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -332,6 +332,9 @@ ISNULL: 'ISNULL'; ISNOTNULL: 'ISNOTNULL'; CIDRMATCH: 'CIDRMATCH'; +// JSON FUNCTIONS +JSON_VALID: 'JSON_VALID'; + // FLOWCONTROL FUNCTIONS IFNULL: 'IFNULL'; NULLIF: 'NULLIF'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 27f7e4014b..999c5d9c87 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -662,6 +662,7 @@ conditionFunctionName | ISNULL | ISNOTNULL | CIDRMATCH + | JSON_VALID ; // flow control function return non-boolean value From 76c3995ddd8765ce26672dd4948abf3f0270ba4b Mon Sep 17 00:00:00 2001 From: Kenrick Yap <14yapkc1@gmail.com> Date: Mon, 6 Jan 2025 15:14:25 -0800 Subject: [PATCH 02/70] added doctest, integ-tests, and unit tests Signed-off-by: Kenrick Yap <14yapkc1@gmail.com> --- .../expression/json/JsonFunctionsTest.java | 59 +++++++++++++++++ docs/category.json | 1 + docs/user/dql/metadata.rst | 3 +- docs/user/ppl/functions/json.rst | 34 ++++++++++ doctest/test_data/json_test.json | 5 ++ doctest/test_docs.py | 4 +- .../sql/legacy/SQLIntegTestCase.java | 8 ++- .../org/opensearch/sql/legacy/TestUtils.java | 5 ++ .../opensearch/sql/legacy/TestsConstants.java | 1 + .../opensearch/sql/ppl/JsonFunctionIT.java | 65 +++++++++++++++++++ .../json_test_index_mappping.json | 12 ++++ integ-test/src/test/resources/json_test.json | 10 +++ 12 files changed, 204 insertions(+), 3 deletions(-) create mode 100644 core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java create mode 100644 docs/user/ppl/functions/json.rst create mode 100644 doctest/test_data/json_test.json create mode 100644 integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java create mode 100644 integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json create mode 100644 integ-test/src/test/resources/json_test.json diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java new file mode 100644 index 0000000000..ee817dc71a --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -0,0 +1,59 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.json; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.env.Environment; + +@ExtendWith(MockitoExtension.class) +public class JsonFunctionsTest { + + private static final ExprValue JsonObject = ExprValueUtils.stringValue("{\"a\":\"1\",\"b\":\"2\"}"); + private static final ExprValue JsonArray = ExprValueUtils.stringValue("[1, 2, 3, 4]"); + private static final ExprValue JsonScalarString = ExprValueUtils.stringValue("\"abc\""); + private static final ExprValue JsonEmptyString = ExprValueUtils.stringValue(""); + private static final ExprValue JsonInvalidObject = ExprValueUtils.stringValue("{\"invalid\":\"json\", \"string\"}"); + private static final ExprValue JsonInvalidScalar = ExprValueUtils.stringValue("abc"); + + @Mock private Environment env; + + @Test + public void json_valid_invalid_json_string() { + assertEquals(LITERAL_FALSE, execute(JsonInvalidObject)); + assertEquals(LITERAL_FALSE, execute(JsonInvalidScalar)); + } + + @Test + public void json_valid_valid_json_string() { + assertEquals(LITERAL_TRUE, JsonObject); + assertEquals(LITERAL_TRUE, JsonArray); + assertEquals(LITERAL_TRUE, JsonScalarString); + assertEquals(LITERAL_TRUE, JsonEmptyString); + } + + private ExprValue execute(ExprValue jsonString) { + final String fieldName = "json_string"; + FunctionExpression exp = DSL.jsonValid(DSL.literal(jsonString)); + + when(DSL.ref(fieldName, STRING).valueOf(env)).thenReturn(jsonString); + + return exp.valueOf(env); + } +} diff --git a/docs/category.json b/docs/category.json index 32f56cfb46..efbb57d6e6 100644 --- a/docs/category.json +++ b/docs/category.json @@ -34,6 +34,7 @@ "user/ppl/functions/datetime.rst", "user/ppl/functions/expressions.rst", "user/ppl/functions/ip.rst", + "user/ppl/functions/json.rst", "user/ppl/functions/math.rst", "user/ppl/functions/relevance.rst", "user/ppl/functions/string.rst" diff --git a/docs/user/dql/metadata.rst b/docs/user/dql/metadata.rst index aba4eb0c75..b059c0cded 100644 --- a/docs/user/dql/metadata.rst +++ b/docs/user/dql/metadata.rst @@ -35,7 +35,7 @@ Example 1: Show All Indices Information SQL query:: os> SHOW TABLES LIKE '%' - fetched rows / total rows = 10/10 + fetched rows / total rows = 11/11 +----------------+-------------+-----------------+------------+---------+----------+------------+-----------+---------------------------+----------------+ | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | TABLE_TYPE | REMARKS | TYPE_CAT | TYPE_SCHEM | TYPE_NAME | SELF_REFERENCING_COL_NAME | REF_GENERATION | |----------------+-------------+-----------------+------------+---------+----------+------------+-----------+---------------------------+----------------| @@ -44,6 +44,7 @@ SQL query:: | docTestCluster | null | accounts | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | apache | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | books | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | json_test | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | nested | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | nyc_taxi | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | people | BASE TABLE | null | null | null | null | null | null | diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst new file mode 100644 index 0000000000..8f986ace6b --- /dev/null +++ b/docs/user/ppl/functions/json.rst @@ -0,0 +1,34 @@ +==================== +IP Address Functions +==================== + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 1 + +JSON_VALID +---------- + +Description +>>>>>>>>>>> + +Usage: `json_valid(json_string)` checks if `json_string` is a valid STRING string. + +Argument type: STRING + +Return type: BOOLEAN + +Example:: + + > source=json_test | where json_valid(json_string) | fields test_name, json_string + fetched rows / total rows = 4/4 + +--------------------+--------------------+ + | test_name | json_string | + |--------------------|--------------------| + | json object | {"a":"1","b":"2"} | + | json array | [1, 2, 3, 4] | + | json scalar string | [1, 2, 3, 4] | + | json empty string | [1, 2, 3, 4] | + +--------------------+--------------------+ diff --git a/doctest/test_data/json_test.json b/doctest/test_data/json_test.json new file mode 100644 index 0000000000..2da491675e --- /dev/null +++ b/doctest/test_data/json_test.json @@ -0,0 +1,5 @@ +{"test_name":"json object", "json_string":"{\"a\":\"1\",\"b\":\"2\"}"} +{"test_name":"json array", "json_string":"[1, 2, 3, 4]"} +{"test_name":"json scalar string", "json_string":"\"abc\""} +{"test_name":"json empty string","json_string":""} +{"test_name":"json invalid object", "json_string":"{\"invalid\":\"json\", \"string\"}"} diff --git a/doctest/test_docs.py b/doctest/test_docs.py index 1d46766c6d..906bbd65b5 100644 --- a/doctest/test_docs.py +++ b/doctest/test_docs.py @@ -30,6 +30,7 @@ NESTED = "nested" DATASOURCES = ".ql-datasources" WEBLOGS = "weblogs" +JSON_TEST = "json_test" class DocTestConnection(OpenSearchConnection): @@ -123,6 +124,7 @@ def set_up_test_indices(test): load_file("nested_objects.json", index_name=NESTED) load_file("datasources.json", index_name=DATASOURCES) load_file("weblogs.json", index_name=WEBLOGS) + load_file("json_test.json", index_name=JSON_TEST) def load_file(filename, index_name): @@ -151,7 +153,7 @@ def set_up(test): def tear_down(test): # drop leftover tables after each test - test_data_client.indices.delete(index=[ACCOUNTS, EMPLOYEES, PEOPLE, ACCOUNT2, NYC_TAXI, BOOKS, APACHE, WILDCARD, NESTED, WEBLOGS], ignore_unavailable=True) + test_data_client.indices.delete(index=[ACCOUNTS, EMPLOYEES, PEOPLE, ACCOUNT2, NYC_TAXI, BOOKS, APACHE, WILDCARD, NESTED, WEBLOGS, JSON_TEST], ignore_unavailable=True) docsuite = partial(doctest.DocFileSuite, diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index 1728be74e6..d4f7213736 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -22,6 +22,7 @@ import static org.opensearch.sql.legacy.TestUtils.getGameOfThronesIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getGeopointIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getJoinTypeIndexMapping; +import static org.opensearch.sql.legacy.TestUtils.getJsonTestIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getLocationIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getMappingFile; import static org.opensearch.sql.legacy.TestUtils.getNestedSimpleIndexMapping; @@ -745,7 +746,12 @@ public enum Index { TestsConstants.TEST_INDEX_GEOPOINT, "dates", getGeopointIndexMapping(), - "src/test/resources/geopoints.json"); + "src/test/resources/geopoints.json"), + JSON_TEST( + TestsConstants.TEST_INDEX_JSON_TEST, + "json", + getJsonTestIndexMapping(), + "src/test/resources/json_test.json"); private final String name; private final String type; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index 195dda0cbd..610ad1366a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -250,6 +250,11 @@ public static String getGeopointIndexMapping() { return getMappingFile(mappingFile); } + public static String getJsonTestIndexMapping() { + String mappingFile = "json_test_index_mapping.json"; + return getMappingFile(mappingFile); + } + public static void loadBulk(Client client, String jsonPath, String defaultIndex) throws Exception { System.out.println(String.format("Loading file %s into opensearch cluster", jsonPath)); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index 1e336f544e..387054ac7e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -58,6 +58,7 @@ public class TestsConstants { public static final String TEST_INDEX_MULTI_NESTED_TYPE = TEST_INDEX + "_multi_nested"; public static final String TEST_INDEX_NESTED_WITH_NULLS = TEST_INDEX + "_nested_with_nulls"; public static final String TEST_INDEX_GEOPOINT = TEST_INDEX + "_geopoint"; + public static final String TEST_INDEX_JSON_TEST = TEST_INDEX + "_json_test"; public static final String DATASOURCES = ".ql-datasources"; public static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java new file mode 100644 index 0000000000..62e7868b41 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import org.json.JSONObject; + +import javax.json.Json; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_JSON_TEST; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +public class JsonFunctionIT extends PPLIntegTestCase { + @Override + public void init() throws IOException { + loadIndex(Index.JSON_TEST); + } + + @Test + public void test_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST + ) + ); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json object"), + rows("json array"), + rows("json scalar string"), + rows("json empty string") + ); + } + + @Test + public void test_not_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where not json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST + ) + ); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json invalid object") + ); + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json new file mode 100644 index 0000000000..b825254b11 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json @@ -0,0 +1,12 @@ +{ + "mappings": { + "properties": { + "test_name": { + "type": "text" + }, + "json_string": { + "type": "text" + } + } + } +} diff --git a/integ-test/src/test/resources/json_test.json b/integ-test/src/test/resources/json_test.json new file mode 100644 index 0000000000..e198eb7c43 --- /dev/null +++ b/integ-test/src/test/resources/json_test.json @@ -0,0 +1,10 @@ +{"index":{"_id":"1"}} +{"test_name":"json object", "json_string":"{\"a\":\"1\",\"b\":\"2\"}"} +{"index":{"_id":"2"}} +{"test_name":"json array", "json_string":"[1, 2, 3, 4]"} +{"index":{"_id":"3"}} +{"test_name":"json scalar string", "json_string":"\"abc\""} +{"index":{"_id":"4"}} +{"test_name":"json empty string","json_string":""} +{"index":{"_id":"5"}} +{"test_name":"json invalid object", "json_string":"{\"invalid\":\"json\", \"string\"}"} From ce2c551351eaf58f3055cd280f54a7bb3f0ac95a Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Mon, 6 Jan 2025 15:27:23 -0800 Subject: [PATCH 03/70] addressed PR comments Signed-off-by: Kenrick Yap --- .../org/opensearch/sql/expression/json/JsonFunctions.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index e4ba7af499..f0745af4b7 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.expression.json; import com.fasterxml.jackson.databind.ObjectMapper; From ad1bde30c80a9eaf5305af1bb22c8d776a2cefbe Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Mon, 6 Jan 2025 17:03:25 -0800 Subject: [PATCH 04/70] fixed unit tests Signed-off-by: Kenrick Yap --- .../sql/expression/json/JsonFunctions.java | 1 - .../expression/json/JsonFunctionsTest.java | 20 +++++-------------- docs/user/ppl/functions/json.rst | 19 +++++++++--------- 3 files changed, 15 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index f0745af4b7..7ef90cf73c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -23,7 +23,6 @@ @UtilityClass public class JsonFunctions { public void register(BuiltinFunctionRepository repository) { - repository.register(jsonValid()); } diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index ee817dc71a..56b6363167 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -13,18 +13,14 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.expression.DSL; -import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.FunctionExpression; -import org.opensearch.sql.expression.env.Environment; @ExtendWith(MockitoExtension.class) public class JsonFunctionsTest { - private static final ExprValue JsonObject = ExprValueUtils.stringValue("{\"a\":\"1\",\"b\":\"2\"}"); private static final ExprValue JsonArray = ExprValueUtils.stringValue("[1, 2, 3, 4]"); private static final ExprValue JsonScalarString = ExprValueUtils.stringValue("\"abc\""); @@ -32,8 +28,6 @@ public class JsonFunctionsTest { private static final ExprValue JsonInvalidObject = ExprValueUtils.stringValue("{\"invalid\":\"json\", \"string\"}"); private static final ExprValue JsonInvalidScalar = ExprValueUtils.stringValue("abc"); - @Mock private Environment env; - @Test public void json_valid_invalid_json_string() { assertEquals(LITERAL_FALSE, execute(JsonInvalidObject)); @@ -42,18 +36,14 @@ public void json_valid_invalid_json_string() { @Test public void json_valid_valid_json_string() { - assertEquals(LITERAL_TRUE, JsonObject); - assertEquals(LITERAL_TRUE, JsonArray); - assertEquals(LITERAL_TRUE, JsonScalarString); - assertEquals(LITERAL_TRUE, JsonEmptyString); + assertEquals(LITERAL_TRUE, execute(JsonObject)); + assertEquals(LITERAL_TRUE, execute(JsonArray)); + assertEquals(LITERAL_TRUE, execute(JsonScalarString)); + assertEquals(LITERAL_TRUE, execute(JsonEmptyString)); } private ExprValue execute(ExprValue jsonString) { - final String fieldName = "json_string"; FunctionExpression exp = DSL.jsonValid(DSL.literal(jsonString)); - - when(DSL.ref(fieldName, STRING).valueOf(env)).thenReturn(jsonString); - - return exp.valueOf(env); + return exp.valueOf(); } } diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 8f986ace6b..bf5bd46b7a 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -22,13 +22,14 @@ Return type: BOOLEAN Example:: - > source=json_test | where json_valid(json_string) | fields test_name, json_string + > source=json_test | eval is_valid = json_valid(json_string) | fields test_name, json_string, is_valid fetched rows / total rows = 4/4 - +--------------------+--------------------+ - | test_name | json_string | - |--------------------|--------------------| - | json object | {"a":"1","b":"2"} | - | json array | [1, 2, 3, 4] | - | json scalar string | [1, 2, 3, 4] | - | json empty string | [1, 2, 3, 4] | - +--------------------+--------------------+ + +---------------------+------------------------------+----------+ + | test_name | json_string | is_valid | + |---------------------|------------------------------|----------| + | json object | {"a":"1","b":"2"} | True | + | json array | [1, 2, 3, 4] | True | + | json scalar string | "abc" | True | + | json empty string | | True | + | json invalid object | {"invalid":"json", "string"} | True | + +---------------------+------------------------------+----------+ From ccf47a277b420ccf2ec9ab94d23c05c33b0408d1 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 7 Jan 2025 09:20:59 -0800 Subject: [PATCH 05/70] addressed pr comments Signed-off-by: Kenrick Yap --- .../org/opensearch/sql/expression/json/JsonFunctions.java | 3 ++- .../org/opensearch/sql/expression/json/JsonFunctionsTest.java | 4 ++-- .../resources/indexDefinitions/json_test_index_mappping.json | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 7ef90cf73c..6cdc14807e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -5,6 +5,7 @@ package org.opensearch.sql.expression.json; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.experimental.UtilityClass; import org.opensearch.sql.data.model.ExprValue; @@ -43,7 +44,7 @@ private ExprValue isValidJson(ExprValue jsonExprValue) { try { objectMapper.readTree(jsonExprValue.stringValue()); return ExprValueUtils.LITERAL_TRUE; - } catch (Exception e) { + } catch (JsonProcessingException e) { return ExprValueUtils.LITERAL_FALSE; } } diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 56b6363167..5f2f51bcb9 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -29,13 +29,13 @@ public class JsonFunctionsTest { private static final ExprValue JsonInvalidScalar = ExprValueUtils.stringValue("abc"); @Test - public void json_valid_invalid_json_string() { + public void json_valid_returns_false() { assertEquals(LITERAL_FALSE, execute(JsonInvalidObject)); assertEquals(LITERAL_FALSE, execute(JsonInvalidScalar)); } @Test - public void json_valid_valid_json_string() { + public void json_valid_returns_true() { assertEquals(LITERAL_TRUE, execute(JsonObject)); assertEquals(LITERAL_TRUE, execute(JsonArray)); assertEquals(LITERAL_TRUE, execute(JsonScalarString)); diff --git a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json index b825254b11..fb97836d5e 100644 --- a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json +++ b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json @@ -2,7 +2,7 @@ "mappings": { "properties": { "test_name": { - "type": "text" + "type": "keyword" }, "json_string": { "type": "text" From acc76a027792e7f66d6a6dd2c3c8771080fb01ff Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 7 Jan 2025 13:40:28 -0800 Subject: [PATCH 06/70] addressed PR comments Signed-off-by: Kenrick Yap --- .../sql/expression/json/JsonFunctions.java | 19 ++------------ .../org/opensearch/sql/utils/JsonUtils.java | 26 +++++++++++++++++++ .../datetime/DateTimeFunctionTest.java | 4 +-- .../sql/expression/datetime/ExtractTest.java | 3 +++ .../sql/expression/datetime/YearweekTest.java | 5 ++-- 5 files changed, 36 insertions(+), 21 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/utils/JsonUtils.java diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 6cdc14807e..4e8a3bba69 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -14,6 +14,7 @@ import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.DefaultFunctionResolver; +import org.opensearch.sql.utils.JsonUtils; import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.STRING; @@ -30,22 +31,6 @@ public void register(BuiltinFunctionRepository repository) { private DefaultFunctionResolver jsonValid() { return define( BuiltinFunctionName.JSON_VALID.getName(), - impl(nullMissingHandling(JsonFunctions::isValidJson), BOOLEAN, STRING)); - } - - /** - * Checks if given JSON string can be parsed as valid JSON. - * - * @param jsonExprValue JSON string (e.g. "198.51.100.14" or "2001:0db8::ff00:42:8329"). - * @return true if the string can be parsed as valid JSON, else false. - */ - private ExprValue isValidJson(ExprValue jsonExprValue) { - ObjectMapper objectMapper = new ObjectMapper(); - try { - objectMapper.readTree(jsonExprValue.stringValue()); - return ExprValueUtils.LITERAL_TRUE; - } catch (JsonProcessingException e) { - return ExprValueUtils.LITERAL_FALSE; - } + impl(nullMissingHandling(JsonUtils::isValidJson), BOOLEAN, STRING)); } } diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java new file mode 100644 index 0000000000..393f83256a --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -0,0 +1,26 @@ +package org.opensearch.sql.utils; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.experimental.UtilityClass; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; + +@UtilityClass +public class JsonUtils { + /** + * Checks if given JSON string can be parsed as valid JSON. + * + * @param jsonExprValue JSON string (e.g. "198.51.100.14" or "2001:0db8::ff00:42:8329"). + * @return true if the string can be parsed as valid JSON, else false. + */ + public static ExprValue isValidJson(ExprValue jsonExprValue) { + ObjectMapper objectMapper = new ObjectMapper(); + try { + objectMapper.readTree(jsonExprValue.stringValue()); + return ExprValueUtils.LITERAL_TRUE; + } catch (JsonProcessingException e) { + return ExprValueUtils.LITERAL_FALSE; + } + } +} diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index c820c97196..dbc75c45ae 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.stream.Stream; import lombok.AllArgsConstructor; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -1228,9 +1229,8 @@ public void testWeekFormats( expectedInteger); } - // subtracting 1 as a temporary fix for year 2024. - // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test + @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testWeekOfYearWithTimeType() { assertAll( () -> diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index 02d50d0b59..fd87c144da 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -11,6 +11,8 @@ import java.time.LocalDate; import java.util.stream.Stream; + +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -91,6 +93,7 @@ private void datePartWithTimeArgQuery(String part, String time, long expected) { } @Test + @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testExtractDatePartWithTimeType() { datePartWithTimeArgQuery( "DAY", timeInput, LocalDate.now(functionProperties.getQueryStartClock()).getDayOfMonth()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index 47225ac601..87e2f05d85 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -14,6 +14,8 @@ import java.time.LocalDate; import java.util.stream.Stream; + +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -97,9 +99,8 @@ public void testYearweekWithoutMode() { assertEquals(eval(expression), eval(expressionWithoutMode)); } - // subtracting 1 as a temporary fix for year 2024. - // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test + @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testYearweekWithTimeType() { int week = LocalDate.now(functionProperties.getQueryStartClock()).get(ALIGNED_WEEK_OF_YEAR) - 1; int year = LocalDate.now(functionProperties.getQueryStartClock()).getYear(); From 519c6f21aaa8c45dd22d525c414f220db64a704d Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 7 Jan 2025 13:40:47 -0800 Subject: [PATCH 07/70] removed unused dependencies Signed-off-by: Kenrick Yap --- .../org/opensearch/sql/expression/json/JsonFunctions.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 4e8a3bba69..dc7bde32c2 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -5,12 +5,7 @@ package org.opensearch.sql.expression.json; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import lombok.experimental.UtilityClass; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.model.ExprValueUtils; -import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.DefaultFunctionResolver; From 2e319fea1e26bcb1b324413cc2ebc9983eb11453 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 7 Jan 2025 14:28:45 -0800 Subject: [PATCH 08/70] linting Signed-off-by: Kenrick Yap --- .../org/opensearch/sql/expression/DSL.java | 2 +- .../sql/expression/json/JsonFunctions.java | 28 +++--- .../org/opensearch/sql/utils/JsonUtils.java | 28 +++--- .../datetime/DateTimeFunctionTest.java | 3 +- .../sql/expression/datetime/ExtractTest.java | 4 +- .../sql/expression/datetime/YearweekTest.java | 4 +- .../expression/json/JsonFunctionsTest.java | 48 +++++----- .../opensearch/sql/ppl/JsonFunctionIT.java | 89 ++++++++----------- 8 files changed, 98 insertions(+), 108 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index fc97f35d11..dc819c8163 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -683,7 +683,7 @@ public static FunctionExpression notLike(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.NOT_LIKE, expressions); } - public static FunctionExpression jsonValid(Expression... expressions){ + public static FunctionExpression jsonValid(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.JSON_VALID, expressions); } diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index dc7bde32c2..49541e5d59 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -5,27 +5,27 @@ package org.opensearch.sql.expression.json; -import lombok.experimental.UtilityClass; -import org.opensearch.sql.expression.function.BuiltinFunctionName; -import org.opensearch.sql.expression.function.BuiltinFunctionRepository; -import org.opensearch.sql.expression.function.DefaultFunctionResolver; -import org.opensearch.sql.utils.JsonUtils; - import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; +import lombok.experimental.UtilityClass; +import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.BuiltinFunctionRepository; +import org.opensearch.sql.expression.function.DefaultFunctionResolver; +import org.opensearch.sql.utils.JsonUtils; + @UtilityClass public class JsonFunctions { - public void register(BuiltinFunctionRepository repository) { - repository.register(jsonValid()); - } + public void register(BuiltinFunctionRepository repository) { + repository.register(jsonValid()); + } - private DefaultFunctionResolver jsonValid() { - return define( - BuiltinFunctionName.JSON_VALID.getName(), - impl(nullMissingHandling(JsonUtils::isValidJson), BOOLEAN, STRING)); - } + private DefaultFunctionResolver jsonValid() { + return define( + BuiltinFunctionName.JSON_VALID.getName(), + impl(nullMissingHandling(JsonUtils::isValidJson), BOOLEAN, STRING)); + } } diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 393f83256a..af02d26ef7 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -8,19 +8,19 @@ @UtilityClass public class JsonUtils { - /** - * Checks if given JSON string can be parsed as valid JSON. - * - * @param jsonExprValue JSON string (e.g. "198.51.100.14" or "2001:0db8::ff00:42:8329"). - * @return true if the string can be parsed as valid JSON, else false. - */ - public static ExprValue isValidJson(ExprValue jsonExprValue) { - ObjectMapper objectMapper = new ObjectMapper(); - try { - objectMapper.readTree(jsonExprValue.stringValue()); - return ExprValueUtils.LITERAL_TRUE; - } catch (JsonProcessingException e) { - return ExprValueUtils.LITERAL_FALSE; - } + /** + * Checks if given JSON string can be parsed as valid JSON. + * + * @param jsonExprValue JSON string (e.g. "198.51.100.14" or "2001:0db8::ff00:42:8329"). + * @return true if the string can be parsed as valid JSON, else false. + */ + public static ExprValue isValidJson(ExprValue jsonExprValue) { + ObjectMapper objectMapper = new ObjectMapper(); + try { + objectMapper.readTree(jsonExprValue.stringValue()); + return ExprValueUtils.LITERAL_TRUE; + } catch (JsonProcessingException e) { + return ExprValueUtils.LITERAL_FALSE; } + } } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index dbc75c45ae..042491b2fc 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -1230,7 +1230,8 @@ public void testWeekFormats( } @Test - @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") + @Disabled( + "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testWeekOfYearWithTimeType() { assertAll( () -> diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index fd87c144da..feb809af57 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -11,7 +11,6 @@ import java.time.LocalDate; import java.util.stream.Stream; - import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -93,7 +92,8 @@ private void datePartWithTimeArgQuery(String part, String time, long expected) { } @Test - @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") + @Disabled( + "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testExtractDatePartWithTimeType() { datePartWithTimeArgQuery( "DAY", timeInput, LocalDate.now(functionProperties.getQueryStartClock()).getDayOfMonth()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index 87e2f05d85..9e7b8c04c1 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -14,7 +14,6 @@ import java.time.LocalDate; import java.util.stream.Stream; - import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -100,7 +99,8 @@ public void testYearweekWithoutMode() { } @Test - @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") + @Disabled( + "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testYearweekWithTimeType() { int week = LocalDate.now(functionProperties.getQueryStartClock()).get(ALIGNED_WEEK_OF_YEAR) - 1; int year = LocalDate.now(functionProperties.getQueryStartClock()).getYear(); diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 5f2f51bcb9..2e8ece2817 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -6,10 +6,8 @@ package org.opensearch.sql.expression.json; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.when; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; -import static org.opensearch.sql.data.type.ExprCoreType.STRING; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -21,29 +19,31 @@ @ExtendWith(MockitoExtension.class) public class JsonFunctionsTest { - private static final ExprValue JsonObject = ExprValueUtils.stringValue("{\"a\":\"1\",\"b\":\"2\"}"); - private static final ExprValue JsonArray = ExprValueUtils.stringValue("[1, 2, 3, 4]"); - private static final ExprValue JsonScalarString = ExprValueUtils.stringValue("\"abc\""); - private static final ExprValue JsonEmptyString = ExprValueUtils.stringValue(""); - private static final ExprValue JsonInvalidObject = ExprValueUtils.stringValue("{\"invalid\":\"json\", \"string\"}"); - private static final ExprValue JsonInvalidScalar = ExprValueUtils.stringValue("abc"); + private static final ExprValue JsonObject = + ExprValueUtils.stringValue("{\"a\":\"1\",\"b\":\"2\"}"); + private static final ExprValue JsonArray = ExprValueUtils.stringValue("[1, 2, 3, 4]"); + private static final ExprValue JsonScalarString = ExprValueUtils.stringValue("\"abc\""); + private static final ExprValue JsonEmptyString = ExprValueUtils.stringValue(""); + private static final ExprValue JsonInvalidObject = + ExprValueUtils.stringValue("{\"invalid\":\"json\", \"string\"}"); + private static final ExprValue JsonInvalidScalar = ExprValueUtils.stringValue("abc"); - @Test - public void json_valid_returns_false() { - assertEquals(LITERAL_FALSE, execute(JsonInvalidObject)); - assertEquals(LITERAL_FALSE, execute(JsonInvalidScalar)); - } + @Test + public void json_valid_returns_false() { + assertEquals(LITERAL_FALSE, execute(JsonInvalidObject)); + assertEquals(LITERAL_FALSE, execute(JsonInvalidScalar)); + } - @Test - public void json_valid_returns_true() { - assertEquals(LITERAL_TRUE, execute(JsonObject)); - assertEquals(LITERAL_TRUE, execute(JsonArray)); - assertEquals(LITERAL_TRUE, execute(JsonScalarString)); - assertEquals(LITERAL_TRUE, execute(JsonEmptyString)); - } + @Test + public void json_valid_returns_true() { + assertEquals(LITERAL_TRUE, execute(JsonObject)); + assertEquals(LITERAL_TRUE, execute(JsonArray)); + assertEquals(LITERAL_TRUE, execute(JsonScalarString)); + assertEquals(LITERAL_TRUE, execute(JsonEmptyString)); + } - private ExprValue execute(ExprValue jsonString) { - FunctionExpression exp = DSL.jsonValid(DSL.literal(jsonString)); - return exp.valueOf(); - } + private ExprValue execute(ExprValue jsonString) { + FunctionExpression exp = DSL.jsonValid(DSL.literal(jsonString)); + return exp.valueOf(); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java index 62e7868b41..f02750147d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java @@ -5,61 +5,50 @@ package org.opensearch.sql.ppl; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import org.json.JSONObject; - -import javax.json.Json; - import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_JSON_TEST; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifySchema; -public class JsonFunctionIT extends PPLIntegTestCase { - @Override - public void init() throws IOException { - loadIndex(Index.JSON_TEST); - } - - @Test - public void test_json_valid() throws IOException { - JSONObject result; - - result = - executeQuery( - String.format( - "source=%s | where json_valid(json_string) | fields test_name", - TEST_INDEX_JSON_TEST - ) - ); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows( - result, - rows("json object"), - rows("json array"), - rows("json scalar string"), - rows("json empty string") - ); - } - - @Test - public void test_not_json_valid() throws IOException { - JSONObject result; +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; - result = - executeQuery( - String.format( - "source=%s | where not json_valid(json_string) | fields test_name", - TEST_INDEX_JSON_TEST - ) - ); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows( - result, - rows("json invalid object") - ); - } +public class JsonFunctionIT extends PPLIntegTestCase { + @Override + public void init() throws IOException { + loadIndex(Index.JSON_TEST); + } + + @Test + public void test_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json object"), + rows("json array"), + rows("json scalar string"), + rows("json empty string")); + } + + @Test + public void test_not_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where not json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows(result, rows("json invalid object")); + } } From ee0820df0b2edb2e44753822831305e493e8c43a Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Wed, 8 Jan 2025 14:16:31 -0800 Subject: [PATCH 09/70] addressed pr comment and rolling back disabled test case Signed-off-by: Kenrick Yap --- core/src/main/java/org/opensearch/sql/utils/JsonUtils.java | 2 +- .../sql/expression/datetime/DateTimeFunctionTest.java | 4 ++-- .../org/opensearch/sql/expression/datetime/ExtractTest.java | 2 -- .../org/opensearch/sql/expression/datetime/YearweekTest.java | 4 ++-- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index af02d26ef7..d7f37b4197 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -11,7 +11,7 @@ public class JsonUtils { /** * Checks if given JSON string can be parsed as valid JSON. * - * @param jsonExprValue JSON string (e.g. "198.51.100.14" or "2001:0db8::ff00:42:8329"). + * @param jsonExprValue JSON string (e.g. "{\"hello\": \"world\"}"). * @return true if the string can be parsed as valid JSON, else false. */ public static ExprValue isValidJson(ExprValue jsonExprValue) { diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index 042491b2fc..4b287319ba 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -1229,9 +1229,9 @@ public void testWeekFormats( expectedInteger); } + // subtracting 1 as a temporary fix for year 2024. + // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test - @Disabled( - "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testWeekOfYearWithTimeType() { assertAll( () -> diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index feb809af57..a9ae6274b1 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -92,8 +92,6 @@ private void datePartWithTimeArgQuery(String part, String time, long expected) { } @Test - @Disabled( - "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testExtractDatePartWithTimeType() { datePartWithTimeArgQuery( "DAY", timeInput, LocalDate.now(functionProperties.getQueryStartClock()).getDayOfMonth()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index 9e7b8c04c1..e273ee9ed4 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -98,9 +98,9 @@ public void testYearweekWithoutMode() { assertEquals(eval(expression), eval(expressionWithoutMode)); } + // subtracting 1 as a temporary fix for year 2024. + // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test - @Disabled( - "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testYearweekWithTimeType() { int week = LocalDate.now(functionProperties.getQueryStartClock()).get(ALIGNED_WEEK_OF_YEAR) - 1; int year = LocalDate.now(functionProperties.getQueryStartClock()).getYear(); From 3407d4a4d92473b6c2be0154c325b49b6f26751a Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 9 Jan 2025 10:58:21 -0800 Subject: [PATCH 10/70] removed disabled import Signed-off-by: Kenrick Yap --- .../opensearch/sql/expression/datetime/DateTimeFunctionTest.java | 1 - .../java/org/opensearch/sql/expression/datetime/ExtractTest.java | 1 - .../org/opensearch/sql/expression/datetime/YearweekTest.java | 1 - 3 files changed, 3 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index 4b287319ba..c820c97196 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -27,7 +27,6 @@ import java.util.List; import java.util.stream.Stream; import lombok.AllArgsConstructor; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index 1dfa3908e6..d7635de610 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -11,7 +11,6 @@ import java.time.LocalDate; import java.util.stream.Stream; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index e273ee9ed4..47225ac601 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -14,7 +14,6 @@ import java.time.LocalDate; import java.util.stream.Stream; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; From 7ef6cc95f9dec0515a1316efafb3b48181433358 Mon Sep 17 00:00:00 2001 From: kenrickyap <121634635+kenrickyap@users.noreply.github.com> Date: Thu, 9 Jan 2025 13:01:48 -0800 Subject: [PATCH 11/70] Update docs/user/ppl/functions/json.rst Co-authored-by: Andrew Carbonetto Signed-off-by: kenrickyap <121634635+kenrickyap@users.noreply.github.com> --- docs/user/ppl/functions/json.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index bf5bd46b7a..a69101300b 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -31,5 +31,5 @@ Example:: | json array | [1, 2, 3, 4] | True | | json scalar string | "abc" | True | | json empty string | | True | - | json invalid object | {"invalid":"json", "string"} | True | + | json invalid object | {"invalid":"json", "string"} | False | +---------------------+------------------------------+----------+ From e5e90acc98d2b71209fc3d450eb91899d00cb058 Mon Sep 17 00:00:00 2001 From: kenrickyap <121634635+kenrickyap@users.noreply.github.com> Date: Thu, 9 Jan 2025 13:02:10 -0800 Subject: [PATCH 12/70] Update integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java Co-authored-by: Andrew Carbonetto Signed-off-by: kenrickyap <121634635+kenrickyap@users.noreply.github.com> --- .../src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java index f02750147d..501ef9448e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java @@ -15,7 +15,7 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; -public class JsonFunctionIT extends PPLIntegTestCase { +public class JsonFunctionsIT extends PPLIntegTestCase { @Override public void init() throws IOException { loadIndex(Index.JSON_TEST); From 2187a5ac0f053b6815c1900459e4ba295ca7d060 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 9 Jan 2025 13:05:12 -0800 Subject: [PATCH 13/70] nit Signed-off-by: Kenrick Yap --- ...json_test_index_mappping.json => json_test_index_mapping.json} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename integ-test/src/test/resources/indexDefinitions/{json_test_index_mappping.json => json_test_index_mapping.json} (100%) diff --git a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json b/integ-test/src/test/resources/indexDefinitions/json_test_index_mapping.json similarity index 100% rename from integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json rename to integ-test/src/test/resources/indexDefinitions/json_test_index_mapping.json From 3512b335980f8f8fcbc6cfa10accb1b6d622f877 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 9 Jan 2025 13:16:09 -0800 Subject: [PATCH 14/70] fixed integ test Signed-off-by: Kenrick Yap --- .../sql/ppl/{JsonFunctionIT.java => JsonFunctionsIT.java} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename integ-test/src/test/java/org/opensearch/sql/ppl/{JsonFunctionIT.java => JsonFunctionsIT.java} (100%) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java similarity index 100% rename from integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java rename to integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java From 9fea6060ba12f95f174041439422787b59cdc3d4 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 9 Jan 2025 13:29:03 -0800 Subject: [PATCH 15/70] change text type to keyword Signed-off-by: Kenrick Yap --- .../resources/indexDefinitions/json_test_index_mapping.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/resources/indexDefinitions/json_test_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/json_test_index_mapping.json index fb97836d5e..86bd0c6e94 100644 --- a/integ-test/src/test/resources/indexDefinitions/json_test_index_mapping.json +++ b/integ-test/src/test/resources/indexDefinitions/json_test_index_mapping.json @@ -5,7 +5,7 @@ "type": "keyword" }, "json_string": { - "type": "text" + "type": "keyword" } } } From fbc54bc59f1e54b89d14dff96b2578772f2647c5 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Fri, 10 Jan 2025 10:37:08 -0800 Subject: [PATCH 16/70] addressed PR comments Signed-off-by: Kenrick Yap --- .../expression/json/JsonFunctionsTest.java | 3 +++ docs/user/ppl/functions/json.rst | 21 ++++++++++--------- doctest/test_data/json_test.json | 1 + .../opensearch/sql/ppl/JsonFunctionsIT.java | 1 + integ-test/src/test/resources/json_test.json | 2 ++ 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 2e8ece2817..e374841e7f 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -19,6 +19,8 @@ @ExtendWith(MockitoExtension.class) public class JsonFunctionsTest { + private static final ExprValue JsonNestedObject = + ExprValueUtils.stringValue("{\"a\":\"1\",\"b\":{\"c\":\"2\",\"d\":\"3\"}}"); private static final ExprValue JsonObject = ExprValueUtils.stringValue("{\"a\":\"1\",\"b\":\"2\"}"); private static final ExprValue JsonArray = ExprValueUtils.stringValue("[1, 2, 3, 4]"); @@ -36,6 +38,7 @@ public void json_valid_returns_false() { @Test public void json_valid_returns_true() { + assertEquals(LITERAL_TRUE, execute(JsonNestedObject)); assertEquals(LITERAL_TRUE, execute(JsonObject)); assertEquals(LITERAL_TRUE, execute(JsonArray)); assertEquals(LITERAL_TRUE, execute(JsonScalarString)); diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index a69101300b..ce3d1a4c76 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -1,5 +1,5 @@ ==================== -IP Address Functions +JSON Functions ==================== .. rubric:: Table of contents @@ -24,12 +24,13 @@ Example:: > source=json_test | eval is_valid = json_valid(json_string) | fields test_name, json_string, is_valid fetched rows / total rows = 4/4 - +---------------------+------------------------------+----------+ - | test_name | json_string | is_valid | - |---------------------|------------------------------|----------| - | json object | {"a":"1","b":"2"} | True | - | json array | [1, 2, 3, 4] | True | - | json scalar string | "abc" | True | - | json empty string | | True | - | json invalid object | {"invalid":"json", "string"} | False | - +---------------------+------------------------------+----------+ + +---------------------+---------------------------------+----------+ + | test_name | json_string | is_valid | + |---------------------|---------------------------------|----------| + | json nested object | {"a":"1","b":{"c":"2","d":"3"}} | True | + | json object | {"a":"1","b":"2"} | True | + | json array | [1, 2, 3, 4] | True | + | json scalar string | "abc" | True | + | json empty string | | True | + | json invalid object | {"invalid":"json", "string"} | False | + +---------------------+---------------------------------+----------+ diff --git a/doctest/test_data/json_test.json b/doctest/test_data/json_test.json index 2da491675e..7494fc4aa9 100644 --- a/doctest/test_data/json_test.json +++ b/doctest/test_data/json_test.json @@ -1,3 +1,4 @@ +{"test_name":"json nested object", "json_string":"{\"a\":\"1\",\"b\":{\"c\":\"2\",\"d\":\"3\"}}"} {"test_name":"json object", "json_string":"{\"a\":\"1\",\"b\":\"2\"}"} {"test_name":"json array", "json_string":"[1, 2, 3, 4]"} {"test_name":"json scalar string", "json_string":"\"abc\""} diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 501ef9448e..f852a97d48 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -33,6 +33,7 @@ public void test_json_valid() throws IOException { verifySchema(result, schema("test_name", null, "string")); verifyDataRows( result, + rows("json nested object"), rows("json object"), rows("json array"), rows("json scalar string"), diff --git a/integ-test/src/test/resources/json_test.json b/integ-test/src/test/resources/json_test.json index e198eb7c43..badb4f4f6e 100644 --- a/integ-test/src/test/resources/json_test.json +++ b/integ-test/src/test/resources/json_test.json @@ -1,3 +1,5 @@ +{"index":{"_id":"0"}} +{"test_name":"json nested object", "json_string":"{\"a\":\"1\",\"b\":{\"c\":\"2\",\"d\":\"3\"}}"} {"index":{"_id":"1"}} {"test_name":"json object", "json_string":"{\"a\":\"1\",\"b\":\"2\"}"} {"index":{"_id":"2"}} From 31ad2a4286c7a1d4876eb9e42eb4a41777b9c826 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Fri, 10 Jan 2025 16:07:52 -0800 Subject: [PATCH 17/70] fix doc-test Signed-off-by: Kenrick Yap --- docs/user/ppl/functions/json.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index ce3d1a4c76..74c173be13 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -23,7 +23,7 @@ Return type: BOOLEAN Example:: > source=json_test | eval is_valid = json_valid(json_string) | fields test_name, json_string, is_valid - fetched rows / total rows = 4/4 + fetched rows / total rows = 6/6 +---------------------+---------------------------------+----------+ | test_name | json_string | is_valid | |---------------------|---------------------------------|----------| From 2b2a8f3a4143ead02a96373b58e0482388066c21 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 14 Jan 2025 09:10:06 -0800 Subject: [PATCH 18/70] added null test Signed-off-by: Kenrick Yap --- core/src/main/java/org/opensearch/sql/utils/JsonUtils.java | 5 +++++ .../test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java | 2 +- integ-test/src/test/resources/json_test.json | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index d7f37b4197..37c374286e 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -16,6 +16,11 @@ public class JsonUtils { */ public static ExprValue isValidJson(ExprValue jsonExprValue) { ObjectMapper objectMapper = new ObjectMapper(); + + if (jsonExprValue.isNull() || jsonExprValue.isMissing()) { + return ExprValueUtils.LITERAL_FALSE; + } + try { objectMapper.readTree(jsonExprValue.stringValue()); return ExprValueUtils.LITERAL_TRUE; diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index f852a97d48..9e5ac041fb 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -50,6 +50,6 @@ public void test_not_json_valid() throws IOException { "source=%s | where not json_valid(json_string) | fields test_name", TEST_INDEX_JSON_TEST)); verifySchema(result, schema("test_name", null, "string")); - verifyDataRows(result, rows("json invalid object")); + verifyDataRows(result, rows("json invalid object"), rows("json null")); } } diff --git a/integ-test/src/test/resources/json_test.json b/integ-test/src/test/resources/json_test.json index badb4f4f6e..e393bfeb8e 100644 --- a/integ-test/src/test/resources/json_test.json +++ b/integ-test/src/test/resources/json_test.json @@ -10,3 +10,5 @@ {"test_name":"json empty string","json_string":""} {"index":{"_id":"5"}} {"test_name":"json invalid object", "json_string":"{\"invalid\":\"json\", \"string\"}"} +{"index":{"_id":"6"}} +{"test_name":"json null", "json_string":null} From 1913bfe755abf3bfe3607bcb8f3e421efd1d9a17 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Tue, 14 Jan 2025 22:24:54 -0800 Subject: [PATCH 19/70] SQL: adding error case unit tests for json_valid Signed-off-by: Andrew Carbonetto --- .../sql/expression/json/JsonFunctionsTest.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index e374841e7f..c889a237d4 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -6,7 +6,9 @@ package org.opensearch.sql.expression.json; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; import org.junit.jupiter.api.Test; @@ -14,6 +16,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.FunctionExpression; @@ -34,6 +37,15 @@ public class JsonFunctionsTest { public void json_valid_returns_false() { assertEquals(LITERAL_FALSE, execute(JsonInvalidObject)); assertEquals(LITERAL_FALSE, execute(JsonInvalidScalar)); + + // caught by nullMissingHandling and returns null + assertEquals(LITERAL_NULL, execute(LITERAL_NULL)); + } + + @Test + public void json_valid_throws_ExpressionEvaluationException() { + assertThrows( + ExpressionEvaluationException.class, () -> execute(ExprValueUtils.booleanValue(true))); } @Test From 67d979d2f61f0e6b94d394014b82950bceb00f57 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 15 Jan 2025 09:12:53 -0800 Subject: [PATCH 20/70] json_valid: null and missing should return false Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/expression/json/JsonFunctions.java | 4 +--- .../opensearch/sql/expression/json/JsonFunctionsTest.java | 6 +++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 49541e5d59..acc0c4c064 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -9,7 +9,6 @@ import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; -import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import lombok.experimental.UtilityClass; import org.opensearch.sql.expression.function.BuiltinFunctionName; @@ -25,7 +24,6 @@ public void register(BuiltinFunctionRepository repository) { private DefaultFunctionResolver jsonValid() { return define( - BuiltinFunctionName.JSON_VALID.getName(), - impl(nullMissingHandling(JsonUtils::isValidJson), BOOLEAN, STRING)); + BuiltinFunctionName.JSON_VALID.getName(), impl(JsonUtils::isValidJson, BOOLEAN, STRING)); } } diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index c889a237d4..3228a565c2 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -8,6 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; @@ -37,9 +38,8 @@ public class JsonFunctionsTest { public void json_valid_returns_false() { assertEquals(LITERAL_FALSE, execute(JsonInvalidObject)); assertEquals(LITERAL_FALSE, execute(JsonInvalidScalar)); - - // caught by nullMissingHandling and returns null - assertEquals(LITERAL_NULL, execute(LITERAL_NULL)); + assertEquals(LITERAL_FALSE, execute(LITERAL_NULL)); + assertEquals(LITERAL_FALSE, execute(LITERAL_MISSING)); } @Test From aa6b72354d45fd9a423cd0837b7dee213e7df8f8 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 8 Jan 2025 10:49:03 -0800 Subject: [PATCH 21/70] PPL: Add json and cast to json functions Signed-off-by: Andrew Carbonetto --- .../opensearch/sql/ast/expression/Cast.java | 2 + .../org/opensearch/sql/expression/DSL.java | 8 ++ .../function/BuiltinFunctionName.java | 6 +- .../operator/convert/TypeCastOperators.java | 9 ++ .../org/opensearch/sql/utils/JsonUtils.java | 68 ++++++++- .../expression/json/JsonFunctionsTest.java | 132 ++++++++++++++++++ .../convert/TypeCastOperatorTest.java | 103 ++++++++++++++ docs/user/ppl/functions/json.rst | 25 ++++ .../opensearch/sql/ppl/JsonFunctionsIT.java | 28 ++++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 8 ++ 11 files changed, 386 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java b/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java index 541dbedead..854ba0ed69 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java @@ -13,6 +13,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_FLOAT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_INT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_IP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_JSON; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_LONG; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_SHORT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_STRING; @@ -56,6 +57,7 @@ public class Cast extends UnresolvedExpression { .put("timestamp", CAST_TO_TIMESTAMP.getName()) .put("datetime", CAST_TO_DATETIME.getName()) .put("ip", CAST_TO_IP.getName()) + .put("json", CAST_TO_JSON.getName()) .build(); /** The source expression cast from. */ diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index dc819c8163..a50e3d1470 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -843,6 +843,10 @@ public static FunctionExpression castIp(Expression value) { return compile(FunctionProperties.None, BuiltinFunctionName.CAST_TO_IP, value); } + public static FunctionExpression castJson(Expression value) { + return compile(FunctionProperties.None, BuiltinFunctionName.CAST_TO_JSON, value); + } + public static FunctionExpression typeof(Expression value) { return compile(FunctionProperties.None, BuiltinFunctionName.TYPEOF, value); } @@ -973,6 +977,10 @@ public static FunctionExpression utc_timestamp( return compile(functionProperties, BuiltinFunctionName.UTC_TIMESTAMP, args); } + public static FunctionExpression json_function(Expression value) { + return compile(FunctionProperties.None, BuiltinFunctionName.JSON, value); + } + @SuppressWarnings("unchecked") private static T compile( FunctionProperties functionProperties, BuiltinFunctionName bfn, Expression... args) { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 43fdbf2eb7..ef9a872974 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -235,6 +235,7 @@ public enum BuiltinFunctionName { CAST_TO_TIMESTAMP(FunctionName.of("cast_to_timestamp")), CAST_TO_DATETIME(FunctionName.of("cast_to_datetime")), CAST_TO_IP(FunctionName.of("cast_to_ip")), + CAST_TO_JSON(FunctionName.of("cast_to_json")), TYPEOF(FunctionName.of("typeof")), /** Relevance Function. */ @@ -259,7 +260,10 @@ public enum BuiltinFunctionName { MULTIMATCH(FunctionName.of("multimatch")), MULTIMATCHQUERY(FunctionName.of("multimatchquery")), WILDCARDQUERY(FunctionName.of("wildcardquery")), - WILDCARD_QUERY(FunctionName.of("wildcard_query")); + WILDCARD_QUERY(FunctionName.of("wildcard_query")), + + /* Json Functions. */ + JSON(FunctionName.of("json")); private final FunctionName name; diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index b388f7d89a..d6518e47d9 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -17,10 +17,12 @@ import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; +import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import static org.opensearch.sql.expression.function.FunctionDSL.impl; import static org.opensearch.sql.expression.function.FunctionDSL.implWithProperties; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandlingWithProperties; +import static org.opensearch.sql.utils.JsonUtils.castJson; import java.util.Arrays; import java.util.stream.Collectors; @@ -57,6 +59,7 @@ public static void register(BuiltinFunctionRepository repository) { repository.register(castToDouble()); repository.register(castToBoolean()); repository.register(castToIp()); + repository.register(castToJson()); repository.register(castToDate()); repository.register(castToTime()); repository.register(castToTimestamp()); @@ -183,6 +186,12 @@ private static DefaultFunctionResolver castToIp() { impl(nullMissingHandling((v) -> v), IP, IP)); } + private static DefaultFunctionResolver castToJson() { + return FunctionDSL.define( + BuiltinFunctionName.CAST_TO_JSON.getName(), + impl(nullMissingHandling((v) -> castJson(v.stringValue())), UNDEFINED, STRING)); + } + private static DefaultFunctionResolver castToDate() { return FunctionDSL.define( BuiltinFunctionName.CAST_TO_DATE.getName(), diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 37c374286e..9835db9193 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -1,10 +1,24 @@ package org.opensearch.sql.utils; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; + import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import lombok.experimental.UtilityClass; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.exception.SemanticCheckException; @UtilityClass public class JsonUtils { @@ -23,9 +37,57 @@ public static ExprValue isValidJson(ExprValue jsonExprValue) { try { objectMapper.readTree(jsonExprValue.stringValue()); - return ExprValueUtils.LITERAL_TRUE; + return LITERAL_TRUE; } catch (JsonProcessingException e) { - return ExprValueUtils.LITERAL_FALSE; + return LITERAL_FALSE; + } + } + + /** Converts a JSON encoded string to an Expression object. */ + public static ExprValue castJson(String json) { + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode jsonNode; + try { + jsonNode = objectMapper.readTree(json); + } catch (JsonProcessingException e) { + final String errorFormat = "JSON string '%s' is not valid. Error details: %s"; + throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e); + } + + return processJsonNode(jsonNode); + } + + private static ExprValue processJsonNode(JsonNode jsonNode) { + if (jsonNode.isFloatingPointNumber()) { + return new ExprDoubleValue(jsonNode.asDouble()); + } + if (jsonNode.isIntegralNumber()) { + return new ExprIntegerValue(jsonNode.asLong()); + } + if (jsonNode.isBoolean()) { + return jsonNode.asBoolean() ? LITERAL_TRUE : LITERAL_FALSE; } + if (jsonNode.isTextual()) { + return new ExprStringValue(jsonNode.asText()); + } + if (jsonNode.isArray()) { + List elements = new LinkedList<>(); + for (var iter = jsonNode.iterator(); iter.hasNext(); ) { + jsonNode = iter.next(); + elements.add(processJsonNode(jsonNode)); + } + return new ExprCollectionValue(elements); + } + if (jsonNode.isObject()) { + Map values = new LinkedHashMap<>(); + for (var iter = jsonNode.fields(); iter.hasNext(); ) { + Map.Entry entry = iter.next(); + values.put(entry.getKey(), processJsonNode(entry.getValue())); + } + return ExprTupleValue.fromExprValueMap(values); + } + + // in all other cases, return null + return LITERAL_NULL; } } diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 3228a565c2..e0fcb9ab81 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -7,17 +7,60 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprBooleanValue; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprLongValue; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprBooleanValue; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprLongValue; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprBooleanValue; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprLongValue; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.ExpressionEvaluationException; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.FunctionExpression; @@ -61,4 +104,93 @@ private ExprValue execute(ExprValue jsonString) { FunctionExpression exp = DSL.jsonValid(DSL.literal(jsonString)); return exp.valueOf(); } + + @Test + void json_returnsJsonObject() { + FunctionExpression exp; + + // Setup + final String objectJson = + "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " + + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; + + LinkedHashMap objectMap = new LinkedHashMap<>(); + objectMap.put("foo", new ExprStringValue("foo")); + objectMap.put("fuzz", ExprBooleanValue.of(true)); + objectMap.put("bar", new ExprLongValue(1234)); + objectMap.put("bar2", new ExprDoubleValue(12.34)); + objectMap.put("baz", ExprNullValue.of()); + objectMap.put( + "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); + objectMap.put( + "arr", + new ExprCollectionValue( + List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = DSL.json_function(DSL.literal(objectJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + } + + @Test + void json_returnsJsonArray() { + FunctionExpression exp; + + // Setup + final String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; + ExprValue expectedArrayExpr = + new ExprCollectionValue( + List.of( + new ExprStringValue("foo"), + new ExprStringValue("fuzz"), + LITERAL_TRUE, + new ExprStringValue("bar"), + new ExprIntegerValue(1234), + new ExprDoubleValue(12.34), + LITERAL_NULL)); + + // exercise + exp = DSL.json_function(DSL.literal(arrayJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprCollectionValue); + assertEquals(expectedArrayExpr, value); + } + + @Test + void json_returnsScalar() { + assertEquals( + new ExprStringValue("foobar"), DSL.json_function(DSL.literal("\"foobar\"")).valueOf()); + + assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); + + assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); + + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); + + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); + + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), DSL.json_function(DSL.literal("{}")).valueOf()); + } + + @Test + void json_returnsSemanticCheckException() { + // invalid type + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); + + // missing bracket + assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); + + // mnissing quote + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); + } } diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java index fd579dfb47..27bd806c11 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java @@ -8,6 +8,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.BYTE; import static org.opensearch.sql.data.type.ExprCoreType.DATE; @@ -21,12 +23,16 @@ import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import java.util.stream.Stream; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.data.model.ExprByteValue; +import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDateValue; import org.opensearch.sql.data.model.ExprDoubleValue; import org.opensearch.sql.data.model.ExprFloatValue; @@ -39,6 +45,7 @@ import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTimeValue; import org.opensearch.sql.data.model.ExprTimestampValue; +import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; @@ -389,4 +396,100 @@ void castToIp() { assertEquals(IP, exp.type()); assertTrue(exp.valueOf().isMissing()); } + + @Test + void castJson_returnsJsonObject() { + FunctionExpression exp; + + // Setup + String objectJson = + "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " + + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; + + LinkedHashMap objectMap = new LinkedHashMap<>(); + objectMap.put("foo", new ExprStringValue("foo")); + objectMap.put("fuzz", ExprBooleanValue.of(true)); + objectMap.put("bar", new ExprLongValue(1234)); + objectMap.put("bar2", new ExprDoubleValue(12.34)); + objectMap.put("baz", ExprNullValue.of()); + objectMap.put( + "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); + objectMap.put( + "arr", + new ExprCollectionValue( + List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = DSL.castJson(DSL.literal(objectJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + } + + @Test + void castJson_returnsJsonArray() { + FunctionExpression exp; + + // Setup + String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; + ExprValue expectedArrayExpr = + new ExprCollectionValue( + List.of( + new ExprStringValue("foo"), + new ExprStringValue("fuzz"), + LITERAL_TRUE, + new ExprStringValue("bar"), + new ExprIntegerValue(1234), + new ExprDoubleValue(12.34), + LITERAL_NULL)); + + // exercise + exp = DSL.castJson(DSL.literal(arrayJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprCollectionValue); + assertEquals(expectedArrayExpr, value); + } + + @Test + void castJson_returnsScalar() { + String scalarStringJson = "\"foobar\""; + assertEquals( + new ExprStringValue("foobar"), DSL.castJson(DSL.literal(scalarStringJson)).valueOf()); + + String scalarNumberJson = "1234"; + assertEquals(new ExprIntegerValue(1234), DSL.castJson(DSL.literal(scalarNumberJson)).valueOf()); + + String scalarBooleanJson = "true"; + assertEquals(LITERAL_TRUE, DSL.castJson(DSL.literal(scalarBooleanJson)).valueOf()); + + String scalarNullJson = "null"; + assertEquals(LITERAL_NULL, DSL.castJson(DSL.literal(scalarNullJson)).valueOf()); + + String empty = ""; + assertEquals(LITERAL_NULL, DSL.castJson(DSL.literal(empty)).valueOf()); + + String emptyObject = "{}"; + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), + DSL.castJson(DSL.literal(emptyObject)).valueOf()); + } + + @Test + void castJson_returnsSemanticCheckException() { + // invalid type + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); + + // missing bracket + assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); + + // mnissing quote + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); + } } diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 74c173be13..82c0da23a6 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -34,3 +34,28 @@ Example:: | json empty string | | True | | json invalid object | {"invalid":"json", "string"} | False | +---------------------+---------------------------------+----------+ + +JSON +---------- + +Description +>>>>>>>>>>> + +Usage: `json(value)` Evaluates whether a string can be parsed as a json-encoded string and casted as an expression. Returns the JSON value if valid, null otherwise. + +Argument type: STRING + +Return type: BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY + +Example:: + + > source=json_test | where json_valid(json_string) | eval json=json(json_string) | fields test_name, json_string, json + fetched rows / total rows = 4/4 + +---------------------+------------------------------+---------------+ + | test_name | json_string | json | + |---------------------|------------------------------|---------------| + | json object | {"a":"1","b":"2"} | {a:"1",b:"2"} | + | json array | [1, 2, 3, 4] | [1,2,3,4] | + | json scalar string | "abc" | "abc" | + | json empty string | | null | + +---------------------+------------------------------+---------------+ diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 9e5ac041fb..33c54ff60b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -52,4 +52,32 @@ public void test_not_json_valid() throws IOException { verifySchema(result, schema("test_name", null, "string")); verifyDataRows(result, rows("json invalid object"), rows("json null")); } + + @Test + public void test_cast_json() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | eval json=cast(json_string to json) | fields json", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json object"), + rows("json array"), + rows("json scalar string"), + rows("json empty string")); + } + + @Test + public void test_json() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | eval json=json(json_string) | fields json", TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows(result, rows("json invalid object")); + } } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index ea7e060d9c..9d6707a872 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -143,6 +143,7 @@ FLOAT: 'FLOAT'; STRING: 'STRING'; BOOLEAN: 'BOOLEAN'; IP: 'IP'; +JSON: 'JSON'; // SPECIAL CHARACTERS AND OPERATORS PIPE: '|'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 999c5d9c87..74b05dc28b 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -409,6 +409,7 @@ convertedDataType | typeName = STRING | typeName = BOOLEAN | typeName = IP + | typeName = JSON ; evalFunctionName @@ -419,6 +420,7 @@ evalFunctionName | flowControlFunctionName | systemFunctionName | positionFunctionName + | jsonFunctionName ; functionArgs @@ -700,6 +702,10 @@ positionFunctionName : POSITION ; +jsonFunctionName + : JSON + ; + // operators comparisonOperator : EQUAL @@ -963,4 +969,6 @@ keywordsCanBeId | SPARKLINE | C | DC + // JSON + | JSON ; From 4c992359992b557c481ac8ef4f6df3880525e53c Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 8 Jan 2025 10:56:50 -0800 Subject: [PATCH 22/70] PPL: Update json cast for review Signed-off-by: Andrew Carbonetto --- .../sql/expression/function/BuiltinFunctionName.java | 6 ++---- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 +- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 3 +-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index ef9a872974..cd309a712d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -206,6 +206,7 @@ public enum BuiltinFunctionName { /** Json Functions. */ JSON_VALID(FunctionName.of("json_valid")), + JSON(FunctionName.of("json")), /** NULL Test. */ IS_NULL(FunctionName.of("is null")), @@ -260,10 +261,7 @@ public enum BuiltinFunctionName { MULTIMATCH(FunctionName.of("multimatch")), MULTIMATCHQUERY(FunctionName.of("multimatchquery")), WILDCARDQUERY(FunctionName.of("wildcardquery")), - WILDCARD_QUERY(FunctionName.of("wildcard_query")), - - /* Json Functions. */ - JSON(FunctionName.of("json")); + WILDCARD_QUERY(FunctionName.of("wildcard_query")); private final FunctionName name; diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 9d6707a872..31a668be00 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -143,7 +143,6 @@ FLOAT: 'FLOAT'; STRING: 'STRING'; BOOLEAN: 'BOOLEAN'; IP: 'IP'; -JSON: 'JSON'; // SPECIAL CHARACTERS AND OPERATORS PIPE: '|'; @@ -335,6 +334,7 @@ CIDRMATCH: 'CIDRMATCH'; // JSON FUNCTIONS JSON_VALID: 'JSON_VALID'; +JSON: 'JSON'; // FLOWCONTROL FUNCTIONS IFNULL: 'IFNULL'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 74b05dc28b..5d1f29614d 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -870,6 +870,7 @@ keywordsCanBeId | mathematicalFunctionName | positionFunctionName | conditionFunctionName + | jsonFunctionName // commands | SEARCH | DESCRIBE @@ -969,6 +970,4 @@ keywordsCanBeId | SPARKLINE | C | DC - // JSON - | JSON ; From 9ccde7f3da431f223cf9d596bb6f61b606b1df44 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 8 Jan 2025 17:17:51 -0800 Subject: [PATCH 23/70] Fix testes Signed-off-by: Andrew Carbonetto --- .../sql/expression/json/JsonFunctions.java | 8 ++++++ .../operator/convert/TypeCastOperators.java | 4 +-- .../org/opensearch/sql/utils/JsonUtils.java | 4 +-- .../opensearch/sql/ppl/JsonFunctionsIT.java | 25 ++++++++++++------- 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index acc0c4c064..f5bcc505a6 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; @@ -20,10 +21,17 @@ public class JsonFunctions { public void register(BuiltinFunctionRepository repository) { repository.register(jsonValid()); + repository.register(jsonFunction()); } private DefaultFunctionResolver jsonValid() { return define( BuiltinFunctionName.JSON_VALID.getName(), impl(JsonUtils::isValidJson, BOOLEAN, STRING)); } + + private DefaultFunctionResolver jsonFunction() { + return define( + BuiltinFunctionName.JSON.getName(), + impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING)); + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index d6518e47d9..cfd570ab89 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -22,7 +22,6 @@ import static org.opensearch.sql.expression.function.FunctionDSL.implWithProperties; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandlingWithProperties; -import static org.opensearch.sql.utils.JsonUtils.castJson; import java.util.Arrays; import java.util.stream.Collectors; @@ -44,6 +43,7 @@ import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.DefaultFunctionResolver; import org.opensearch.sql.expression.function.FunctionDSL; +import org.opensearch.sql.utils.JsonUtils; @UtilityClass public class TypeCastOperators { @@ -189,7 +189,7 @@ private static DefaultFunctionResolver castToIp() { private static DefaultFunctionResolver castToJson() { return FunctionDSL.define( BuiltinFunctionName.CAST_TO_JSON.getName(), - impl(nullMissingHandling((v) -> castJson(v.stringValue())), UNDEFINED, STRING)); + impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING)); } private static DefaultFunctionResolver castToDate() { diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 9835db9193..0579eda933 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -44,11 +44,11 @@ public static ExprValue isValidJson(ExprValue jsonExprValue) { } /** Converts a JSON encoded string to an Expression object. */ - public static ExprValue castJson(String json) { + public static ExprValue castJson(ExprValue json) { ObjectMapper objectMapper = new ObjectMapper(); JsonNode jsonNode; try { - jsonNode = objectMapper.readTree(json); + jsonNode = objectMapper.readTree(json.stringValue()); } catch (JsonProcessingException e) { final String errorFormat = "JSON string '%s' is not valid. Error details: %s"; throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 33c54ff60b..465443c096 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -12,6 +12,8 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; +import java.util.List; +import java.util.Map; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -58,15 +60,15 @@ public void test_cast_json() throws IOException { JSONObject result = executeQuery( String.format( - "source=%s | eval json=cast(json_string to json) | fields json", + "source=%s | where json_valid(json_string) | eval casted=cast(json_string as json) | fields test_name, casted", TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string")); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, - rows("json object"), - rows("json array"), - rows("json scalar string"), - rows("json empty string")); + rows("json object", Map.of("a", "1", "b", "2")), + rows("json array", List.of(1,2,3,4)), + rows("json scalar string", "abc"), + rows("json empty string", null)); } @Test @@ -76,8 +78,13 @@ public void test_json() throws IOException { result = executeQuery( String.format( - "source=%s | eval json=json(json_string) | fields json", TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows(result, rows("json invalid object")); + "source=%s | where json_valid(json_string) | eval casted=json(json_string) | fields test_name, casted", TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); + verifyDataRows( + result, + rows("json object", Map.of("a", "1", "b", "2")), + rows("json array", List.of(1,2,3,4)), + rows("json scalar string", "abc"), + rows("json empty string", null)); } } From 4306bf374bce01404ca8067ec6966dd9c0ac2b89 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Thu, 9 Jan 2025 09:33:34 -0800 Subject: [PATCH 24/70] spotless Signed-off-by: Andrew Carbonetto --- .../java/org/opensearch/sql/ppl/JsonFunctionsIT.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 465443c096..015333cdea 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -60,13 +60,14 @@ public void test_cast_json() throws IOException { JSONObject result = executeQuery( String.format( - "source=%s | where json_valid(json_string) | eval casted=cast(json_string as json) | fields test_name, casted", + "source=%s | where json_valid(json_string) | eval casted=cast(json_string as json)" + + " | fields test_name, casted", TEST_INDEX_JSON_TEST)); verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, rows("json object", Map.of("a", "1", "b", "2")), - rows("json array", List.of(1,2,3,4)), + rows("json array", List.of(1, 2, 3, 4)), rows("json scalar string", "abc"), rows("json empty string", null)); } @@ -78,12 +79,14 @@ public void test_json() throws IOException { result = executeQuery( String.format( - "source=%s | where json_valid(json_string) | eval casted=json(json_string) | fields test_name, casted", TEST_INDEX_JSON_TEST)); + "source=%s | where json_valid(json_string) | eval casted=json(json_string) | fields" + + " test_name, casted", + TEST_INDEX_JSON_TEST)); verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, rows("json object", Map.of("a", "1", "b", "2")), - rows("json array", List.of(1,2,3,4)), + rows("json array", List.of(1, 2, 3, 4)), rows("json scalar string", "abc"), rows("json empty string", null)); } From 613137bdc229169641f9dd483990f91f9709b48d Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Mon, 13 Jan 2025 16:13:01 -0800 Subject: [PATCH 25/70] Fix tests Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/ppl/JsonFunctionsIT.java | 13 +++++++++---- integ-test/src/test/resources/json_test.json | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 015333cdea..a636273107 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -12,8 +12,10 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -66,8 +68,9 @@ public void test_cast_json() throws IOException { verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, - rows("json object", Map.of("a", "1", "b", "2")), - rows("json array", List.of(1, 2, 3, 4)), + rows("json nested object", new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), + rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), + rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), rows("json empty string", null)); } @@ -83,10 +86,12 @@ public void test_json() throws IOException { + " test_name, casted", TEST_INDEX_JSON_TEST)); verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); + JSONObject firstRow = new JSONObject(Map.of("c", 2)); verifyDataRows( result, - rows("json object", Map.of("a", "1", "b", "2")), - rows("json array", List.of(1, 2, 3, 4)), + rows("json nested object", new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), + rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), + rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), rows("json empty string", null)); } diff --git a/integ-test/src/test/resources/json_test.json b/integ-test/src/test/resources/json_test.json index e393bfeb8e..dae01ea4ce 100644 --- a/integ-test/src/test/resources/json_test.json +++ b/integ-test/src/test/resources/json_test.json @@ -1,5 +1,5 @@ {"index":{"_id":"0"}} -{"test_name":"json nested object", "json_string":"{\"a\":\"1\",\"b\":{\"c\":\"2\",\"d\":\"3\"}}"} +{"test_name":"json nested object", "json_string":"{\"a\":\"1\", \"b\": {\"c\": \"3\"}, \"d\": [1, 2, 3]}"} {"index":{"_id":"1"}} {"test_name":"json object", "json_string":"{\"a\":\"1\",\"b\":\"2\"}"} {"index":{"_id":"2"}} From ab288728fa9263ce1a79d1baff7ea25ac85966c3 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Mon, 13 Jan 2025 16:20:17 -0800 Subject: [PATCH 26/70] SPOTLESS Signed-off-by: Andrew Carbonetto --- .../expression/json/JsonFunctionsTest.java | 176 +++++++++--------- .../opensearch/sql/ppl/JsonFunctionsIT.java | 9 +- 2 files changed, 94 insertions(+), 91 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index e0fcb9ab81..a1f18f62da 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -105,92 +105,92 @@ private ExprValue execute(ExprValue jsonString) { return exp.valueOf(); } - @Test - void json_returnsJsonObject() { - FunctionExpression exp; - - // Setup - final String objectJson = - "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " - + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; - - LinkedHashMap objectMap = new LinkedHashMap<>(); - objectMap.put("foo", new ExprStringValue("foo")); - objectMap.put("fuzz", ExprBooleanValue.of(true)); - objectMap.put("bar", new ExprLongValue(1234)); - objectMap.put("bar2", new ExprDoubleValue(12.34)); - objectMap.put("baz", ExprNullValue.of()); - objectMap.put( - "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); - objectMap.put( - "arr", - new ExprCollectionValue( - List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); - ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); - - // exercise - exp = DSL.json_function(DSL.literal(objectJson)); - - // Verify - var value = exp.valueOf(); - assertTrue(value instanceof ExprTupleValue); - assertEquals(expectedTupleExpr, value); - } - - @Test - void json_returnsJsonArray() { - FunctionExpression exp; - - // Setup - final String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; - ExprValue expectedArrayExpr = - new ExprCollectionValue( - List.of( - new ExprStringValue("foo"), - new ExprStringValue("fuzz"), - LITERAL_TRUE, - new ExprStringValue("bar"), - new ExprIntegerValue(1234), - new ExprDoubleValue(12.34), - LITERAL_NULL)); - - // exercise - exp = DSL.json_function(DSL.literal(arrayJson)); - - // Verify - var value = exp.valueOf(); - assertTrue(value instanceof ExprCollectionValue); - assertEquals(expectedArrayExpr, value); - } - - @Test - void json_returnsScalar() { - assertEquals( - new ExprStringValue("foobar"), DSL.json_function(DSL.literal("\"foobar\"")).valueOf()); - - assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); - - assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); - - assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); - - assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); - - assertEquals( - ExprTupleValue.fromExprValueMap(Map.of()), DSL.json_function(DSL.literal("{}")).valueOf()); - } - - @Test - void json_returnsSemanticCheckException() { - // invalid type - assertThrows( - SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); - - // missing bracket - assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); - - // mnissing quote - assertThrows( - SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); - } + @Test + void json_returnsJsonObject() { + FunctionExpression exp; + + // Setup + final String objectJson = + "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " + + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; + + LinkedHashMap objectMap = new LinkedHashMap<>(); + objectMap.put("foo", new ExprStringValue("foo")); + objectMap.put("fuzz", ExprBooleanValue.of(true)); + objectMap.put("bar", new ExprLongValue(1234)); + objectMap.put("bar2", new ExprDoubleValue(12.34)); + objectMap.put("baz", ExprNullValue.of()); + objectMap.put( + "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); + objectMap.put( + "arr", + new ExprCollectionValue( + List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = DSL.json_function(DSL.literal(objectJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + } + + @Test + void json_returnsJsonArray() { + FunctionExpression exp; + + // Setup + final String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; + ExprValue expectedArrayExpr = + new ExprCollectionValue( + List.of( + new ExprStringValue("foo"), + new ExprStringValue("fuzz"), + LITERAL_TRUE, + new ExprStringValue("bar"), + new ExprIntegerValue(1234), + new ExprDoubleValue(12.34), + LITERAL_NULL)); + + // exercise + exp = DSL.json_function(DSL.literal(arrayJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprCollectionValue); + assertEquals(expectedArrayExpr, value); + } + + @Test + void json_returnsScalar() { + assertEquals( + new ExprStringValue("foobar"), DSL.json_function(DSL.literal("\"foobar\"")).valueOf()); + + assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); + + assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); + + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); + + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); + + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), DSL.json_function(DSL.literal("{}")).valueOf()); + } + + @Test + void json_returnsSemanticCheckException() { + // invalid type + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); + + // missing bracket + assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); + + // mnissing quote + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index a636273107..eecf7fb338 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -12,7 +12,6 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import org.json.JSONArray; @@ -68,7 +67,9 @@ public void test_cast_json() throws IOException { verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, - rows("json nested object", new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), + rows( + "json nested object", + new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), @@ -89,7 +90,9 @@ public void test_json() throws IOException { JSONObject firstRow = new JSONObject(Map.of("c", 2)); verifyDataRows( result, - rows("json nested object", new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), + rows( + "json nested object", + new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), From 3ec16e0a4fe7c856b5990ce7d4d32c74e9010609 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 15 Jan 2025 10:33:11 -0800 Subject: [PATCH 27/70] Clean up for merge Signed-off-by: Andrew Carbonetto --- .../sql/expression/json/JsonFunctions.java | 1 + .../org/opensearch/sql/utils/JsonUtils.java | 1 + .../expression/json/JsonFunctionsTest.java | 30 ------------------- 3 files changed, 2 insertions(+), 30 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index f5bcc505a6..75f134aa4e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -10,6 +10,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; +import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import lombok.experimental.UtilityClass; import org.opensearch.sql.expression.function.BuiltinFunctionName; diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 0579eda933..c5f7031b13 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -18,6 +18,7 @@ import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.SemanticCheckException; @UtilityClass diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index a1f18f62da..89ea57c2e4 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -7,24 +7,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -39,28 +27,10 @@ import org.opensearch.sql.data.model.ExprNullValue; import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTupleValue; -import org.opensearch.sql.data.model.ExprBooleanValue; -import org.opensearch.sql.data.model.ExprCollectionValue; -import org.opensearch.sql.data.model.ExprDoubleValue; -import org.opensearch.sql.data.model.ExprIntegerValue; -import org.opensearch.sql.data.model.ExprLongValue; -import org.opensearch.sql.data.model.ExprNullValue; -import org.opensearch.sql.data.model.ExprStringValue; -import org.opensearch.sql.data.model.ExprTupleValue; -import org.opensearch.sql.data.model.ExprBooleanValue; -import org.opensearch.sql.data.model.ExprCollectionValue; -import org.opensearch.sql.data.model.ExprDoubleValue; -import org.opensearch.sql.data.model.ExprIntegerValue; -import org.opensearch.sql.data.model.ExprLongValue; -import org.opensearch.sql.data.model.ExprNullValue; -import org.opensearch.sql.data.model.ExprStringValue; -import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; -import org.opensearch.sql.exception.SemanticCheckException; -import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.FunctionExpression; From 6dbf37bb19fd3e49dc94a6c237ca940bfbe068b7 Mon Sep 17 00:00:00 2001 From: Kenrick Yap <14yapkc1@gmail.com> Date: Fri, 3 Jan 2025 10:43:05 -0800 Subject: [PATCH 28/70] added implementation Signed-off-by: Kenrick Yap <14yapkc1@gmail.com> --- .../opensearch/sql/expression/json/JsonFunctions.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index acc0c4c064..56ea5f10c3 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -5,10 +5,20 @@ package org.opensearch.sql.expression.json; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.experimental.UtilityClass; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.BuiltinFunctionRepository; +import org.opensearch.sql.expression.function.DefaultFunctionResolver; + import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; +import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import lombok.experimental.UtilityClass; import org.opensearch.sql.expression.function.BuiltinFunctionName; From b8c6d68a4f340464c84c7c5afa2cb98b9b1c7537 Mon Sep 17 00:00:00 2001 From: Kenrick Yap <14yapkc1@gmail.com> Date: Mon, 6 Jan 2025 15:14:25 -0800 Subject: [PATCH 29/70] added doctest, integ-tests, and unit tests Signed-off-by: Kenrick Yap <14yapkc1@gmail.com> --- .../opensearch/sql/ppl/JsonFunctionIT.java | 65 +++++++++++++++++++ .../json_test_index_mappping.json | 12 ++++ 2 files changed, 77 insertions(+) create mode 100644 integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java create mode 100644 integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java new file mode 100644 index 0000000000..62e7868b41 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import org.json.JSONObject; + +import javax.json.Json; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_JSON_TEST; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +public class JsonFunctionIT extends PPLIntegTestCase { + @Override + public void init() throws IOException { + loadIndex(Index.JSON_TEST); + } + + @Test + public void test_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST + ) + ); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json object"), + rows("json array"), + rows("json scalar string"), + rows("json empty string") + ); + } + + @Test + public void test_not_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where not json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST + ) + ); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json invalid object") + ); + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json new file mode 100644 index 0000000000..b825254b11 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json @@ -0,0 +1,12 @@ +{ + "mappings": { + "properties": { + "test_name": { + "type": "text" + }, + "json_string": { + "type": "text" + } + } + } +} From afb668c74bd1a2f140ec16c18ee2b38cf8fbf1ef Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 7 Jan 2025 09:20:59 -0800 Subject: [PATCH 30/70] addressed pr comments Signed-off-by: Kenrick Yap --- .../resources/indexDefinitions/json_test_index_mappping.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json index b825254b11..fb97836d5e 100644 --- a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json +++ b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json @@ -2,7 +2,7 @@ "mappings": { "properties": { "test_name": { - "type": "text" + "type": "keyword" }, "json_string": { "type": "text" From 54ef1835c6d9cc4766aa57c9bb05a017ae5fca6d Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 7 Jan 2025 13:40:28 -0800 Subject: [PATCH 31/70] addressed PR comments Signed-off-by: Kenrick Yap --- .../sql/expression/datetime/DateTimeFunctionTest.java | 2 ++ .../org/opensearch/sql/expression/datetime/ExtractTest.java | 3 +++ .../org/opensearch/sql/expression/datetime/YearweekTest.java | 3 +++ 3 files changed, 8 insertions(+) diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index ad15dadfb7..e8a4fc81fb 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.stream.Stream; import lombok.AllArgsConstructor; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -1231,6 +1232,7 @@ public void testWeekFormats( } @Test + @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testWeekOfYearWithTimeType() { LocalDate today = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index d7635de610..02645eca06 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -11,6 +11,8 @@ import java.time.LocalDate; import java.util.stream.Stream; + +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -91,6 +93,7 @@ private void datePartWithTimeArgQuery(String part, String time, long expected) { } @Test + @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testExtractDatePartWithTimeType() { LocalDate now = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index d944f7c85c..4de5d3a341 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -16,6 +16,8 @@ import java.time.LocalDate; import java.time.temporal.ChronoUnit; import java.util.stream.Stream; + +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -100,6 +102,7 @@ public void testYearweekWithoutMode() { } @Test + @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testYearweekWithTimeType() { int expected = getYearWeekBeforeSunday(LocalDate.now(functionProperties.getQueryStartClock())); From d84139489e107edb6c1a12cafc8872b5eb90af76 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 7 Jan 2025 13:40:47 -0800 Subject: [PATCH 32/70] removed unused dependencies Signed-off-by: Kenrick Yap --- .../java/org/opensearch/sql/expression/json/JsonFunctions.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 56ea5f10c3..05d5187a14 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -7,9 +7,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.experimental.UtilityClass; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.model.ExprValueUtils; -import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.DefaultFunctionResolver; From 25fb527882104b9083e0cf0c614b24a1586336f4 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 7 Jan 2025 14:28:45 -0800 Subject: [PATCH 33/70] linting Signed-off-by: Kenrick Yap --- .../sql/expression/json/JsonFunctions.java | 5 ++ .../datetime/DateTimeFunctionTest.java | 3 +- .../sql/expression/datetime/ExtractTest.java | 4 +- .../sql/expression/datetime/YearweekTest.java | 4 +- .../opensearch/sql/ppl/JsonFunctionIT.java | 89 ++++++++----------- 5 files changed, 50 insertions(+), 55 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 05d5187a14..2f83262922 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -33,4 +33,9 @@ private DefaultFunctionResolver jsonValid() { return define( BuiltinFunctionName.JSON_VALID.getName(), impl(JsonUtils::isValidJson, BOOLEAN, STRING)); } + private DefaultFunctionResolver jsonValid() { + return define( + BuiltinFunctionName.JSON_VALID.getName(), + impl(nullMissingHandling(JsonUtils::isValidJson), BOOLEAN, STRING)); + } } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index e8a4fc81fb..eb5074f4f7 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -1232,7 +1232,8 @@ public void testWeekFormats( } @Test - @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") + @Disabled( + "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testWeekOfYearWithTimeType() { LocalDate today = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index 02645eca06..925c1f1b7c 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -11,7 +11,6 @@ import java.time.LocalDate; import java.util.stream.Stream; - import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -93,7 +92,8 @@ private void datePartWithTimeArgQuery(String part, String time, long expected) { } @Test - @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") + @Disabled( + "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testExtractDatePartWithTimeType() { LocalDate now = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index 4de5d3a341..daae8b1ff5 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -16,7 +16,6 @@ import java.time.LocalDate; import java.time.temporal.ChronoUnit; import java.util.stream.Stream; - import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -102,7 +101,8 @@ public void testYearweekWithoutMode() { } @Test - @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") + @Disabled( + "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testYearweekWithTimeType() { int expected = getYearWeekBeforeSunday(LocalDate.now(functionProperties.getQueryStartClock())); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java index 62e7868b41..f02750147d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java @@ -5,61 +5,50 @@ package org.opensearch.sql.ppl; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import org.json.JSONObject; - -import javax.json.Json; - import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_JSON_TEST; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifySchema; -public class JsonFunctionIT extends PPLIntegTestCase { - @Override - public void init() throws IOException { - loadIndex(Index.JSON_TEST); - } - - @Test - public void test_json_valid() throws IOException { - JSONObject result; - - result = - executeQuery( - String.format( - "source=%s | where json_valid(json_string) | fields test_name", - TEST_INDEX_JSON_TEST - ) - ); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows( - result, - rows("json object"), - rows("json array"), - rows("json scalar string"), - rows("json empty string") - ); - } - - @Test - public void test_not_json_valid() throws IOException { - JSONObject result; +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; - result = - executeQuery( - String.format( - "source=%s | where not json_valid(json_string) | fields test_name", - TEST_INDEX_JSON_TEST - ) - ); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows( - result, - rows("json invalid object") - ); - } +public class JsonFunctionIT extends PPLIntegTestCase { + @Override + public void init() throws IOException { + loadIndex(Index.JSON_TEST); + } + + @Test + public void test_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json object"), + rows("json array"), + rows("json scalar string"), + rows("json empty string")); + } + + @Test + public void test_not_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where not json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows(result, rows("json invalid object")); + } } From 4a20d0870a58c8273f43f837a5382aaeff80fabc Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Wed, 8 Jan 2025 14:16:31 -0800 Subject: [PATCH 34/70] addressed pr comment and rolling back disabled test case Signed-off-by: Kenrick Yap --- .../sql/expression/datetime/DateTimeFunctionTest.java | 4 ++-- .../org/opensearch/sql/expression/datetime/ExtractTest.java | 2 -- .../org/opensearch/sql/expression/datetime/YearweekTest.java | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index eb5074f4f7..ab391c6834 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -1231,9 +1231,9 @@ public void testWeekFormats( expectedInteger); } + // subtracting 1 as a temporary fix for year 2024. + // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test - @Disabled( - "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testWeekOfYearWithTimeType() { LocalDate today = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index 925c1f1b7c..1dfa3908e6 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -92,8 +92,6 @@ private void datePartWithTimeArgQuery(String part, String time, long expected) { } @Test - @Disabled( - "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testExtractDatePartWithTimeType() { LocalDate now = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index daae8b1ff5..266994c046 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -100,9 +100,9 @@ public void testYearweekWithoutMode() { assertEquals(eval(expression), eval(expressionWithoutMode)); } + // subtracting 1 as a temporary fix for year 2024. + // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test - @Disabled( - "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testYearweekWithTimeType() { int expected = getYearWeekBeforeSunday(LocalDate.now(functionProperties.getQueryStartClock())); From fdc4729de3b1c8310245cec8b469b41b96ec95d6 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 9 Jan 2025 10:58:21 -0800 Subject: [PATCH 35/70] removed disabled import Signed-off-by: Kenrick Yap --- .../opensearch/sql/expression/datetime/DateTimeFunctionTest.java | 1 - .../java/org/opensearch/sql/expression/datetime/ExtractTest.java | 1 - .../org/opensearch/sql/expression/datetime/YearweekTest.java | 1 - 3 files changed, 3 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index ab391c6834..115898e349 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -29,7 +29,6 @@ import java.util.List; import java.util.stream.Stream; import lombok.AllArgsConstructor; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index 1dfa3908e6..d7635de610 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -11,7 +11,6 @@ import java.time.LocalDate; import java.util.stream.Stream; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index 266994c046..ee4df23be4 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -16,7 +16,6 @@ import java.time.LocalDate; import java.time.temporal.ChronoUnit; import java.util.stream.Stream; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; From 707a0b9693b6293cf0261311b852ecb9f91f689f Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 9 Jan 2025 13:05:12 -0800 Subject: [PATCH 36/70] nit Signed-off-by: Kenrick Yap --- .../indexDefinitions/json_test_index_mappping.json | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json diff --git a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json deleted file mode 100644 index fb97836d5e..0000000000 --- a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "mappings": { - "properties": { - "test_name": { - "type": "keyword" - }, - "json_string": { - "type": "text" - } - } - } -} From 4f28211701c7ceb7a974abc0dd377b0acc66f500 Mon Sep 17 00:00:00 2001 From: kenrickyap <121634635+kenrickyap@users.noreply.github.com> Date: Thu, 9 Jan 2025 13:02:10 -0800 Subject: [PATCH 37/70] Update integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java Co-authored-by: Andrew Carbonetto Signed-off-by: kenrickyap <121634635+kenrickyap@users.noreply.github.com> --- .../src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java index f02750147d..501ef9448e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java @@ -15,7 +15,7 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; -public class JsonFunctionIT extends PPLIntegTestCase { +public class JsonFunctionsIT extends PPLIntegTestCase { @Override public void init() throws IOException { loadIndex(Index.JSON_TEST); From 9ec633599d14814284b368d9294263b3760213c9 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 9 Jan 2025 13:16:09 -0800 Subject: [PATCH 38/70] fixed integ test Signed-off-by: Kenrick Yap --- .../opensearch/sql/ppl/JsonFunctionIT.java | 54 ------------------- 1 file changed, 54 deletions(-) delete mode 100644 integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java deleted file mode 100644 index 501ef9448e..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.ppl; - -import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_JSON_TEST; -import static org.opensearch.sql.util.MatcherUtils.rows; -import static org.opensearch.sql.util.MatcherUtils.schema; -import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; -import static org.opensearch.sql.util.MatcherUtils.verifySchema; - -import java.io.IOException; -import org.json.JSONObject; -import org.junit.jupiter.api.Test; - -public class JsonFunctionsIT extends PPLIntegTestCase { - @Override - public void init() throws IOException { - loadIndex(Index.JSON_TEST); - } - - @Test - public void test_json_valid() throws IOException { - JSONObject result; - - result = - executeQuery( - String.format( - "source=%s | where json_valid(json_string) | fields test_name", - TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows( - result, - rows("json object"), - rows("json array"), - rows("json scalar string"), - rows("json empty string")); - } - - @Test - public void test_not_json_valid() throws IOException { - JSONObject result; - - result = - executeQuery( - String.format( - "source=%s | where not json_valid(json_string) | fields test_name", - TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows(result, rows("json invalid object")); - } -} From 3324e661953a855fc46dce474001796d4fdf33ac Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Tue, 14 Jan 2025 22:24:54 -0800 Subject: [PATCH 39/70] SQL: adding error case unit tests for json_valid Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/expression/json/JsonFunctionsTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 3228a565c2..557a2ab9e1 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -46,6 +46,9 @@ public void json_valid_returns_false() { public void json_valid_throws_ExpressionEvaluationException() { assertThrows( ExpressionEvaluationException.class, () -> execute(ExprValueUtils.booleanValue(true))); + + // caught by nullMissingHandling and returns null + assertEquals(LITERAL_NULL, execute(LITERAL_NULL)); } @Test From 7123c350f0164a0dda45f3daa8aade84a9b28abf Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 15 Jan 2025 09:12:53 -0800 Subject: [PATCH 40/70] json_valid: null and missing should return false Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/expression/json/JsonFunctions.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 2f83262922..b0554fd210 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -15,7 +15,6 @@ import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; -import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import lombok.experimental.UtilityClass; import org.opensearch.sql.expression.function.BuiltinFunctionName; @@ -33,9 +32,4 @@ private DefaultFunctionResolver jsonValid() { return define( BuiltinFunctionName.JSON_VALID.getName(), impl(JsonUtils::isValidJson, BOOLEAN, STRING)); } - private DefaultFunctionResolver jsonValid() { - return define( - BuiltinFunctionName.JSON_VALID.getName(), - impl(nullMissingHandling(JsonUtils::isValidJson), BOOLEAN, STRING)); - } } From dbca9915557246f4f4728f8b92fd5046efecec0d Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 8 Jan 2025 10:49:03 -0800 Subject: [PATCH 41/70] PPL: Add json and cast to json functions Signed-off-by: Andrew Carbonetto --- .../opensearch/sql/ast/expression/Cast.java | 2 + .../org/opensearch/sql/expression/DSL.java | 8 ++ .../function/BuiltinFunctionName.java | 6 +- .../operator/convert/TypeCastOperators.java | 9 ++ .../org/opensearch/sql/utils/JsonUtils.java | 68 ++++++++- .../expression/json/JsonFunctionsTest.java | 132 ++++++++++++++++++ .../convert/TypeCastOperatorTest.java | 103 ++++++++++++++ docs/user/ppl/functions/json.rst | 25 ++++ .../opensearch/sql/ppl/JsonFunctionsIT.java | 28 ++++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 8 ++ 11 files changed, 386 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java b/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java index 541dbedead..854ba0ed69 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java @@ -13,6 +13,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_FLOAT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_INT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_IP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_JSON; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_LONG; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_SHORT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_STRING; @@ -56,6 +57,7 @@ public class Cast extends UnresolvedExpression { .put("timestamp", CAST_TO_TIMESTAMP.getName()) .put("datetime", CAST_TO_DATETIME.getName()) .put("ip", CAST_TO_IP.getName()) + .put("json", CAST_TO_JSON.getName()) .build(); /** The source expression cast from. */ diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index dc819c8163..a50e3d1470 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -843,6 +843,10 @@ public static FunctionExpression castIp(Expression value) { return compile(FunctionProperties.None, BuiltinFunctionName.CAST_TO_IP, value); } + public static FunctionExpression castJson(Expression value) { + return compile(FunctionProperties.None, BuiltinFunctionName.CAST_TO_JSON, value); + } + public static FunctionExpression typeof(Expression value) { return compile(FunctionProperties.None, BuiltinFunctionName.TYPEOF, value); } @@ -973,6 +977,10 @@ public static FunctionExpression utc_timestamp( return compile(functionProperties, BuiltinFunctionName.UTC_TIMESTAMP, args); } + public static FunctionExpression json_function(Expression value) { + return compile(FunctionProperties.None, BuiltinFunctionName.JSON, value); + } + @SuppressWarnings("unchecked") private static T compile( FunctionProperties functionProperties, BuiltinFunctionName bfn, Expression... args) { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 43fdbf2eb7..ef9a872974 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -235,6 +235,7 @@ public enum BuiltinFunctionName { CAST_TO_TIMESTAMP(FunctionName.of("cast_to_timestamp")), CAST_TO_DATETIME(FunctionName.of("cast_to_datetime")), CAST_TO_IP(FunctionName.of("cast_to_ip")), + CAST_TO_JSON(FunctionName.of("cast_to_json")), TYPEOF(FunctionName.of("typeof")), /** Relevance Function. */ @@ -259,7 +260,10 @@ public enum BuiltinFunctionName { MULTIMATCH(FunctionName.of("multimatch")), MULTIMATCHQUERY(FunctionName.of("multimatchquery")), WILDCARDQUERY(FunctionName.of("wildcardquery")), - WILDCARD_QUERY(FunctionName.of("wildcard_query")); + WILDCARD_QUERY(FunctionName.of("wildcard_query")), + + /* Json Functions. */ + JSON(FunctionName.of("json")); private final FunctionName name; diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index b388f7d89a..d6518e47d9 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -17,10 +17,12 @@ import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; +import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import static org.opensearch.sql.expression.function.FunctionDSL.impl; import static org.opensearch.sql.expression.function.FunctionDSL.implWithProperties; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandlingWithProperties; +import static org.opensearch.sql.utils.JsonUtils.castJson; import java.util.Arrays; import java.util.stream.Collectors; @@ -57,6 +59,7 @@ public static void register(BuiltinFunctionRepository repository) { repository.register(castToDouble()); repository.register(castToBoolean()); repository.register(castToIp()); + repository.register(castToJson()); repository.register(castToDate()); repository.register(castToTime()); repository.register(castToTimestamp()); @@ -183,6 +186,12 @@ private static DefaultFunctionResolver castToIp() { impl(nullMissingHandling((v) -> v), IP, IP)); } + private static DefaultFunctionResolver castToJson() { + return FunctionDSL.define( + BuiltinFunctionName.CAST_TO_JSON.getName(), + impl(nullMissingHandling((v) -> castJson(v.stringValue())), UNDEFINED, STRING)); + } + private static DefaultFunctionResolver castToDate() { return FunctionDSL.define( BuiltinFunctionName.CAST_TO_DATE.getName(), diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 37c374286e..9835db9193 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -1,10 +1,24 @@ package org.opensearch.sql.utils; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; + import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import lombok.experimental.UtilityClass; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.exception.SemanticCheckException; @UtilityClass public class JsonUtils { @@ -23,9 +37,57 @@ public static ExprValue isValidJson(ExprValue jsonExprValue) { try { objectMapper.readTree(jsonExprValue.stringValue()); - return ExprValueUtils.LITERAL_TRUE; + return LITERAL_TRUE; } catch (JsonProcessingException e) { - return ExprValueUtils.LITERAL_FALSE; + return LITERAL_FALSE; + } + } + + /** Converts a JSON encoded string to an Expression object. */ + public static ExprValue castJson(String json) { + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode jsonNode; + try { + jsonNode = objectMapper.readTree(json); + } catch (JsonProcessingException e) { + final String errorFormat = "JSON string '%s' is not valid. Error details: %s"; + throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e); + } + + return processJsonNode(jsonNode); + } + + private static ExprValue processJsonNode(JsonNode jsonNode) { + if (jsonNode.isFloatingPointNumber()) { + return new ExprDoubleValue(jsonNode.asDouble()); + } + if (jsonNode.isIntegralNumber()) { + return new ExprIntegerValue(jsonNode.asLong()); + } + if (jsonNode.isBoolean()) { + return jsonNode.asBoolean() ? LITERAL_TRUE : LITERAL_FALSE; } + if (jsonNode.isTextual()) { + return new ExprStringValue(jsonNode.asText()); + } + if (jsonNode.isArray()) { + List elements = new LinkedList<>(); + for (var iter = jsonNode.iterator(); iter.hasNext(); ) { + jsonNode = iter.next(); + elements.add(processJsonNode(jsonNode)); + } + return new ExprCollectionValue(elements); + } + if (jsonNode.isObject()) { + Map values = new LinkedHashMap<>(); + for (var iter = jsonNode.fields(); iter.hasNext(); ) { + Map.Entry entry = iter.next(); + values.put(entry.getKey(), processJsonNode(entry.getValue())); + } + return ExprTupleValue.fromExprValueMap(values); + } + + // in all other cases, return null + return LITERAL_NULL; } } diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 557a2ab9e1..809d82941b 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -7,17 +7,60 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprBooleanValue; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprLongValue; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprBooleanValue; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprLongValue; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprBooleanValue; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprLongValue; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.ExpressionEvaluationException; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.FunctionExpression; @@ -64,4 +107,93 @@ private ExprValue execute(ExprValue jsonString) { FunctionExpression exp = DSL.jsonValid(DSL.literal(jsonString)); return exp.valueOf(); } + + @Test + void json_returnsJsonObject() { + FunctionExpression exp; + + // Setup + final String objectJson = + "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " + + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; + + LinkedHashMap objectMap = new LinkedHashMap<>(); + objectMap.put("foo", new ExprStringValue("foo")); + objectMap.put("fuzz", ExprBooleanValue.of(true)); + objectMap.put("bar", new ExprLongValue(1234)); + objectMap.put("bar2", new ExprDoubleValue(12.34)); + objectMap.put("baz", ExprNullValue.of()); + objectMap.put( + "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); + objectMap.put( + "arr", + new ExprCollectionValue( + List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = DSL.json_function(DSL.literal(objectJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + } + + @Test + void json_returnsJsonArray() { + FunctionExpression exp; + + // Setup + final String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; + ExprValue expectedArrayExpr = + new ExprCollectionValue( + List.of( + new ExprStringValue("foo"), + new ExprStringValue("fuzz"), + LITERAL_TRUE, + new ExprStringValue("bar"), + new ExprIntegerValue(1234), + new ExprDoubleValue(12.34), + LITERAL_NULL)); + + // exercise + exp = DSL.json_function(DSL.literal(arrayJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprCollectionValue); + assertEquals(expectedArrayExpr, value); + } + + @Test + void json_returnsScalar() { + assertEquals( + new ExprStringValue("foobar"), DSL.json_function(DSL.literal("\"foobar\"")).valueOf()); + + assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); + + assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); + + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); + + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); + + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), DSL.json_function(DSL.literal("{}")).valueOf()); + } + + @Test + void json_returnsSemanticCheckException() { + // invalid type + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); + + // missing bracket + assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); + + // mnissing quote + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); + } } diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java index fd579dfb47..27bd806c11 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java @@ -8,6 +8,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.BYTE; import static org.opensearch.sql.data.type.ExprCoreType.DATE; @@ -21,12 +23,16 @@ import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import java.util.stream.Stream; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.data.model.ExprByteValue; +import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDateValue; import org.opensearch.sql.data.model.ExprDoubleValue; import org.opensearch.sql.data.model.ExprFloatValue; @@ -39,6 +45,7 @@ import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTimeValue; import org.opensearch.sql.data.model.ExprTimestampValue; +import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; @@ -389,4 +396,100 @@ void castToIp() { assertEquals(IP, exp.type()); assertTrue(exp.valueOf().isMissing()); } + + @Test + void castJson_returnsJsonObject() { + FunctionExpression exp; + + // Setup + String objectJson = + "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " + + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; + + LinkedHashMap objectMap = new LinkedHashMap<>(); + objectMap.put("foo", new ExprStringValue("foo")); + objectMap.put("fuzz", ExprBooleanValue.of(true)); + objectMap.put("bar", new ExprLongValue(1234)); + objectMap.put("bar2", new ExprDoubleValue(12.34)); + objectMap.put("baz", ExprNullValue.of()); + objectMap.put( + "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); + objectMap.put( + "arr", + new ExprCollectionValue( + List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = DSL.castJson(DSL.literal(objectJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + } + + @Test + void castJson_returnsJsonArray() { + FunctionExpression exp; + + // Setup + String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; + ExprValue expectedArrayExpr = + new ExprCollectionValue( + List.of( + new ExprStringValue("foo"), + new ExprStringValue("fuzz"), + LITERAL_TRUE, + new ExprStringValue("bar"), + new ExprIntegerValue(1234), + new ExprDoubleValue(12.34), + LITERAL_NULL)); + + // exercise + exp = DSL.castJson(DSL.literal(arrayJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprCollectionValue); + assertEquals(expectedArrayExpr, value); + } + + @Test + void castJson_returnsScalar() { + String scalarStringJson = "\"foobar\""; + assertEquals( + new ExprStringValue("foobar"), DSL.castJson(DSL.literal(scalarStringJson)).valueOf()); + + String scalarNumberJson = "1234"; + assertEquals(new ExprIntegerValue(1234), DSL.castJson(DSL.literal(scalarNumberJson)).valueOf()); + + String scalarBooleanJson = "true"; + assertEquals(LITERAL_TRUE, DSL.castJson(DSL.literal(scalarBooleanJson)).valueOf()); + + String scalarNullJson = "null"; + assertEquals(LITERAL_NULL, DSL.castJson(DSL.literal(scalarNullJson)).valueOf()); + + String empty = ""; + assertEquals(LITERAL_NULL, DSL.castJson(DSL.literal(empty)).valueOf()); + + String emptyObject = "{}"; + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), + DSL.castJson(DSL.literal(emptyObject)).valueOf()); + } + + @Test + void castJson_returnsSemanticCheckException() { + // invalid type + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); + + // missing bracket + assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); + + // mnissing quote + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); + } } diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index fa704b6c65..bbc972ba73 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -34,3 +34,28 @@ Example:: | json empty string | | True | | json invalid object | {"invalid":"json", "string"} | False | +---------------------+---------------------------------+----------+ + +JSON +---------- + +Description +>>>>>>>>>>> + +Usage: `json(value)` Evaluates whether a string can be parsed as a json-encoded string and casted as an expression. Returns the JSON value if valid, null otherwise. + +Argument type: STRING + +Return type: BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY + +Example:: + + > source=json_test | where json_valid(json_string) | eval json=json(json_string) | fields test_name, json_string, json + fetched rows / total rows = 4/4 + +---------------------+------------------------------+---------------+ + | test_name | json_string | json | + |---------------------|------------------------------|---------------| + | json object | {"a":"1","b":"2"} | {a:"1",b:"2"} | + | json array | [1, 2, 3, 4] | [1,2,3,4] | + | json scalar string | "abc" | "abc" | + | json empty string | | null | + +---------------------+------------------------------+---------------+ diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 9e5ac041fb..33c54ff60b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -52,4 +52,32 @@ public void test_not_json_valid() throws IOException { verifySchema(result, schema("test_name", null, "string")); verifyDataRows(result, rows("json invalid object"), rows("json null")); } + + @Test + public void test_cast_json() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | eval json=cast(json_string to json) | fields json", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json object"), + rows("json array"), + rows("json scalar string"), + rows("json empty string")); + } + + @Test + public void test_json() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | eval json=json(json_string) | fields json", TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows(result, rows("json invalid object")); + } } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index ea7e060d9c..9d6707a872 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -143,6 +143,7 @@ FLOAT: 'FLOAT'; STRING: 'STRING'; BOOLEAN: 'BOOLEAN'; IP: 'IP'; +JSON: 'JSON'; // SPECIAL CHARACTERS AND OPERATORS PIPE: '|'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 999c5d9c87..74b05dc28b 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -409,6 +409,7 @@ convertedDataType | typeName = STRING | typeName = BOOLEAN | typeName = IP + | typeName = JSON ; evalFunctionName @@ -419,6 +420,7 @@ evalFunctionName | flowControlFunctionName | systemFunctionName | positionFunctionName + | jsonFunctionName ; functionArgs @@ -700,6 +702,10 @@ positionFunctionName : POSITION ; +jsonFunctionName + : JSON + ; + // operators comparisonOperator : EQUAL @@ -963,4 +969,6 @@ keywordsCanBeId | SPARKLINE | C | DC + // JSON + | JSON ; From 7df87cbfbd1f94986bcdaacd022309f22287ffbf Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 8 Jan 2025 10:56:50 -0800 Subject: [PATCH 42/70] PPL: Update json cast for review Signed-off-by: Andrew Carbonetto --- .../sql/expression/function/BuiltinFunctionName.java | 6 ++---- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 +- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 3 +-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index ef9a872974..cd309a712d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -206,6 +206,7 @@ public enum BuiltinFunctionName { /** Json Functions. */ JSON_VALID(FunctionName.of("json_valid")), + JSON(FunctionName.of("json")), /** NULL Test. */ IS_NULL(FunctionName.of("is null")), @@ -260,10 +261,7 @@ public enum BuiltinFunctionName { MULTIMATCH(FunctionName.of("multimatch")), MULTIMATCHQUERY(FunctionName.of("multimatchquery")), WILDCARDQUERY(FunctionName.of("wildcardquery")), - WILDCARD_QUERY(FunctionName.of("wildcard_query")), - - /* Json Functions. */ - JSON(FunctionName.of("json")); + WILDCARD_QUERY(FunctionName.of("wildcard_query")); private final FunctionName name; diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 9d6707a872..31a668be00 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -143,7 +143,6 @@ FLOAT: 'FLOAT'; STRING: 'STRING'; BOOLEAN: 'BOOLEAN'; IP: 'IP'; -JSON: 'JSON'; // SPECIAL CHARACTERS AND OPERATORS PIPE: '|'; @@ -335,6 +334,7 @@ CIDRMATCH: 'CIDRMATCH'; // JSON FUNCTIONS JSON_VALID: 'JSON_VALID'; +JSON: 'JSON'; // FLOWCONTROL FUNCTIONS IFNULL: 'IFNULL'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 74b05dc28b..5d1f29614d 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -870,6 +870,7 @@ keywordsCanBeId | mathematicalFunctionName | positionFunctionName | conditionFunctionName + | jsonFunctionName // commands | SEARCH | DESCRIBE @@ -969,6 +970,4 @@ keywordsCanBeId | SPARKLINE | C | DC - // JSON - | JSON ; From cd45fcc628f6ad57a1de1c7b479dde6fbe73cfc8 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 8 Jan 2025 17:17:51 -0800 Subject: [PATCH 43/70] Fix testes Signed-off-by: Andrew Carbonetto --- .../sql/expression/json/JsonFunctions.java | 8 ++++++ .../operator/convert/TypeCastOperators.java | 4 +-- .../org/opensearch/sql/utils/JsonUtils.java | 4 +-- .../opensearch/sql/ppl/JsonFunctionsIT.java | 25 ++++++++++++------- 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index b0554fd210..b1380eb017 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -13,6 +13,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; @@ -26,10 +27,17 @@ public class JsonFunctions { public void register(BuiltinFunctionRepository repository) { repository.register(jsonValid()); + repository.register(jsonFunction()); } private DefaultFunctionResolver jsonValid() { return define( BuiltinFunctionName.JSON_VALID.getName(), impl(JsonUtils::isValidJson, BOOLEAN, STRING)); } + + private DefaultFunctionResolver jsonFunction() { + return define( + BuiltinFunctionName.JSON.getName(), + impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING)); + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index d6518e47d9..cfd570ab89 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -22,7 +22,6 @@ import static org.opensearch.sql.expression.function.FunctionDSL.implWithProperties; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandlingWithProperties; -import static org.opensearch.sql.utils.JsonUtils.castJson; import java.util.Arrays; import java.util.stream.Collectors; @@ -44,6 +43,7 @@ import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.DefaultFunctionResolver; import org.opensearch.sql.expression.function.FunctionDSL; +import org.opensearch.sql.utils.JsonUtils; @UtilityClass public class TypeCastOperators { @@ -189,7 +189,7 @@ private static DefaultFunctionResolver castToIp() { private static DefaultFunctionResolver castToJson() { return FunctionDSL.define( BuiltinFunctionName.CAST_TO_JSON.getName(), - impl(nullMissingHandling((v) -> castJson(v.stringValue())), UNDEFINED, STRING)); + impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING)); } private static DefaultFunctionResolver castToDate() { diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 9835db9193..0579eda933 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -44,11 +44,11 @@ public static ExprValue isValidJson(ExprValue jsonExprValue) { } /** Converts a JSON encoded string to an Expression object. */ - public static ExprValue castJson(String json) { + public static ExprValue castJson(ExprValue json) { ObjectMapper objectMapper = new ObjectMapper(); JsonNode jsonNode; try { - jsonNode = objectMapper.readTree(json); + jsonNode = objectMapper.readTree(json.stringValue()); } catch (JsonProcessingException e) { final String errorFormat = "JSON string '%s' is not valid. Error details: %s"; throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 33c54ff60b..465443c096 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -12,6 +12,8 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; +import java.util.List; +import java.util.Map; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -58,15 +60,15 @@ public void test_cast_json() throws IOException { JSONObject result = executeQuery( String.format( - "source=%s | eval json=cast(json_string to json) | fields json", + "source=%s | where json_valid(json_string) | eval casted=cast(json_string as json) | fields test_name, casted", TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string")); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, - rows("json object"), - rows("json array"), - rows("json scalar string"), - rows("json empty string")); + rows("json object", Map.of("a", "1", "b", "2")), + rows("json array", List.of(1,2,3,4)), + rows("json scalar string", "abc"), + rows("json empty string", null)); } @Test @@ -76,8 +78,13 @@ public void test_json() throws IOException { result = executeQuery( String.format( - "source=%s | eval json=json(json_string) | fields json", TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows(result, rows("json invalid object")); + "source=%s | where json_valid(json_string) | eval casted=json(json_string) | fields test_name, casted", TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); + verifyDataRows( + result, + rows("json object", Map.of("a", "1", "b", "2")), + rows("json array", List.of(1,2,3,4)), + rows("json scalar string", "abc"), + rows("json empty string", null)); } } From 6f5dc07f749975bb60b6016baa25357633179e54 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Thu, 9 Jan 2025 09:33:34 -0800 Subject: [PATCH 44/70] spotless Signed-off-by: Andrew Carbonetto --- .../java/org/opensearch/sql/ppl/JsonFunctionsIT.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 465443c096..015333cdea 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -60,13 +60,14 @@ public void test_cast_json() throws IOException { JSONObject result = executeQuery( String.format( - "source=%s | where json_valid(json_string) | eval casted=cast(json_string as json) | fields test_name, casted", + "source=%s | where json_valid(json_string) | eval casted=cast(json_string as json)" + + " | fields test_name, casted", TEST_INDEX_JSON_TEST)); verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, rows("json object", Map.of("a", "1", "b", "2")), - rows("json array", List.of(1,2,3,4)), + rows("json array", List.of(1, 2, 3, 4)), rows("json scalar string", "abc"), rows("json empty string", null)); } @@ -78,12 +79,14 @@ public void test_json() throws IOException { result = executeQuery( String.format( - "source=%s | where json_valid(json_string) | eval casted=json(json_string) | fields test_name, casted", TEST_INDEX_JSON_TEST)); + "source=%s | where json_valid(json_string) | eval casted=json(json_string) | fields" + + " test_name, casted", + TEST_INDEX_JSON_TEST)); verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, rows("json object", Map.of("a", "1", "b", "2")), - rows("json array", List.of(1,2,3,4)), + rows("json array", List.of(1, 2, 3, 4)), rows("json scalar string", "abc"), rows("json empty string", null)); } From 0aae36e94f765094ea66122197885124e4462153 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Mon, 13 Jan 2025 16:13:01 -0800 Subject: [PATCH 45/70] Fix tests Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/ppl/JsonFunctionsIT.java | 13 +++++++++---- integ-test/src/test/resources/json_test.json | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 015333cdea..a636273107 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -12,8 +12,10 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -66,8 +68,9 @@ public void test_cast_json() throws IOException { verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, - rows("json object", Map.of("a", "1", "b", "2")), - rows("json array", List.of(1, 2, 3, 4)), + rows("json nested object", new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), + rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), + rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), rows("json empty string", null)); } @@ -83,10 +86,12 @@ public void test_json() throws IOException { + " test_name, casted", TEST_INDEX_JSON_TEST)); verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); + JSONObject firstRow = new JSONObject(Map.of("c", 2)); verifyDataRows( result, - rows("json object", Map.of("a", "1", "b", "2")), - rows("json array", List.of(1, 2, 3, 4)), + rows("json nested object", new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), + rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), + rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), rows("json empty string", null)); } diff --git a/integ-test/src/test/resources/json_test.json b/integ-test/src/test/resources/json_test.json index e393bfeb8e..dae01ea4ce 100644 --- a/integ-test/src/test/resources/json_test.json +++ b/integ-test/src/test/resources/json_test.json @@ -1,5 +1,5 @@ {"index":{"_id":"0"}} -{"test_name":"json nested object", "json_string":"{\"a\":\"1\",\"b\":{\"c\":\"2\",\"d\":\"3\"}}"} +{"test_name":"json nested object", "json_string":"{\"a\":\"1\", \"b\": {\"c\": \"3\"}, \"d\": [1, 2, 3]}"} {"index":{"_id":"1"}} {"test_name":"json object", "json_string":"{\"a\":\"1\",\"b\":\"2\"}"} {"index":{"_id":"2"}} From b225f2886513637cd8316471a56e800a303fa2b8 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Mon, 13 Jan 2025 16:20:17 -0800 Subject: [PATCH 46/70] SPOTLESS Signed-off-by: Andrew Carbonetto --- .../expression/json/JsonFunctionsTest.java | 176 +++++++++--------- .../opensearch/sql/ppl/JsonFunctionsIT.java | 9 +- 2 files changed, 94 insertions(+), 91 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 809d82941b..1d06e1b2bd 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -108,92 +108,92 @@ private ExprValue execute(ExprValue jsonString) { return exp.valueOf(); } - @Test - void json_returnsJsonObject() { - FunctionExpression exp; - - // Setup - final String objectJson = - "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " - + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; - - LinkedHashMap objectMap = new LinkedHashMap<>(); - objectMap.put("foo", new ExprStringValue("foo")); - objectMap.put("fuzz", ExprBooleanValue.of(true)); - objectMap.put("bar", new ExprLongValue(1234)); - objectMap.put("bar2", new ExprDoubleValue(12.34)); - objectMap.put("baz", ExprNullValue.of()); - objectMap.put( - "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); - objectMap.put( - "arr", - new ExprCollectionValue( - List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); - ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); - - // exercise - exp = DSL.json_function(DSL.literal(objectJson)); - - // Verify - var value = exp.valueOf(); - assertTrue(value instanceof ExprTupleValue); - assertEquals(expectedTupleExpr, value); - } - - @Test - void json_returnsJsonArray() { - FunctionExpression exp; - - // Setup - final String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; - ExprValue expectedArrayExpr = - new ExprCollectionValue( - List.of( - new ExprStringValue("foo"), - new ExprStringValue("fuzz"), - LITERAL_TRUE, - new ExprStringValue("bar"), - new ExprIntegerValue(1234), - new ExprDoubleValue(12.34), - LITERAL_NULL)); - - // exercise - exp = DSL.json_function(DSL.literal(arrayJson)); - - // Verify - var value = exp.valueOf(); - assertTrue(value instanceof ExprCollectionValue); - assertEquals(expectedArrayExpr, value); - } - - @Test - void json_returnsScalar() { - assertEquals( - new ExprStringValue("foobar"), DSL.json_function(DSL.literal("\"foobar\"")).valueOf()); - - assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); - - assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); - - assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); - - assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); - - assertEquals( - ExprTupleValue.fromExprValueMap(Map.of()), DSL.json_function(DSL.literal("{}")).valueOf()); - } - - @Test - void json_returnsSemanticCheckException() { - // invalid type - assertThrows( - SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); - - // missing bracket - assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); - - // mnissing quote - assertThrows( - SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); - } + @Test + void json_returnsJsonObject() { + FunctionExpression exp; + + // Setup + final String objectJson = + "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " + + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; + + LinkedHashMap objectMap = new LinkedHashMap<>(); + objectMap.put("foo", new ExprStringValue("foo")); + objectMap.put("fuzz", ExprBooleanValue.of(true)); + objectMap.put("bar", new ExprLongValue(1234)); + objectMap.put("bar2", new ExprDoubleValue(12.34)); + objectMap.put("baz", ExprNullValue.of()); + objectMap.put( + "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); + objectMap.put( + "arr", + new ExprCollectionValue( + List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = DSL.json_function(DSL.literal(objectJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + } + + @Test + void json_returnsJsonArray() { + FunctionExpression exp; + + // Setup + final String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; + ExprValue expectedArrayExpr = + new ExprCollectionValue( + List.of( + new ExprStringValue("foo"), + new ExprStringValue("fuzz"), + LITERAL_TRUE, + new ExprStringValue("bar"), + new ExprIntegerValue(1234), + new ExprDoubleValue(12.34), + LITERAL_NULL)); + + // exercise + exp = DSL.json_function(DSL.literal(arrayJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprCollectionValue); + assertEquals(expectedArrayExpr, value); + } + + @Test + void json_returnsScalar() { + assertEquals( + new ExprStringValue("foobar"), DSL.json_function(DSL.literal("\"foobar\"")).valueOf()); + + assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); + + assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); + + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); + + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); + + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), DSL.json_function(DSL.literal("{}")).valueOf()); + } + + @Test + void json_returnsSemanticCheckException() { + // invalid type + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); + + // missing bracket + assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); + + // mnissing quote + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index a636273107..eecf7fb338 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -12,7 +12,6 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import org.json.JSONArray; @@ -68,7 +67,9 @@ public void test_cast_json() throws IOException { verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, - rows("json nested object", new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), + rows( + "json nested object", + new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), @@ -89,7 +90,9 @@ public void test_json() throws IOException { JSONObject firstRow = new JSONObject(Map.of("c", 2)); verifyDataRows( result, - rows("json nested object", new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), + rows( + "json nested object", + new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), From 78af4f8f636d26ba05f1a32044fd942076ef94ba Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 15 Jan 2025 10:33:11 -0800 Subject: [PATCH 47/70] Clean up for merge Signed-off-by: Andrew Carbonetto --- .../sql/expression/json/JsonFunctions.java | 1 + .../org/opensearch/sql/utils/JsonUtils.java | 1 + .../expression/json/JsonFunctionsTest.java | 30 ------------------- 3 files changed, 2 insertions(+), 30 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index b1380eb017..64f84c44f7 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -16,6 +16,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; +import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import lombok.experimental.UtilityClass; import org.opensearch.sql.expression.function.BuiltinFunctionName; diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 0579eda933..c5f7031b13 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -18,6 +18,7 @@ import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.SemanticCheckException; @UtilityClass diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 1d06e1b2bd..f3b17241fb 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -7,24 +7,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -39,28 +27,10 @@ import org.opensearch.sql.data.model.ExprNullValue; import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTupleValue; -import org.opensearch.sql.data.model.ExprBooleanValue; -import org.opensearch.sql.data.model.ExprCollectionValue; -import org.opensearch.sql.data.model.ExprDoubleValue; -import org.opensearch.sql.data.model.ExprIntegerValue; -import org.opensearch.sql.data.model.ExprLongValue; -import org.opensearch.sql.data.model.ExprNullValue; -import org.opensearch.sql.data.model.ExprStringValue; -import org.opensearch.sql.data.model.ExprTupleValue; -import org.opensearch.sql.data.model.ExprBooleanValue; -import org.opensearch.sql.data.model.ExprCollectionValue; -import org.opensearch.sql.data.model.ExprDoubleValue; -import org.opensearch.sql.data.model.ExprIntegerValue; -import org.opensearch.sql.data.model.ExprLongValue; -import org.opensearch.sql.data.model.ExprNullValue; -import org.opensearch.sql.data.model.ExprStringValue; -import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; -import org.opensearch.sql.exception.SemanticCheckException; -import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.FunctionExpression; From b84282aa2fbf0459a13591ea85041a7ee282ef4d Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 15 Jan 2025 10:52:02 -0800 Subject: [PATCH 48/70] clean up unit tests Signed-off-by: Andrew Carbonetto --- .../expression/json/JsonFunctionsTest.java | 60 +++++++++++-------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index f3b17241fb..4ba8b69e1c 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -36,23 +36,16 @@ @ExtendWith(MockitoExtension.class) public class JsonFunctionsTest { - private static final ExprValue JsonNestedObject = - ExprValueUtils.stringValue("{\"a\":\"1\",\"b\":{\"c\":\"2\",\"d\":\"3\"}}"); - private static final ExprValue JsonObject = - ExprValueUtils.stringValue("{\"a\":\"1\",\"b\":\"2\"}"); - private static final ExprValue JsonArray = ExprValueUtils.stringValue("[1, 2, 3, 4]"); - private static final ExprValue JsonScalarString = ExprValueUtils.stringValue("\"abc\""); - private static final ExprValue JsonEmptyString = ExprValueUtils.stringValue(""); - private static final ExprValue JsonInvalidObject = - ExprValueUtils.stringValue("{\"invalid\":\"json\", \"string\"}"); - private static final ExprValue JsonInvalidScalar = ExprValueUtils.stringValue("abc"); - @Test public void json_valid_returns_false() { - assertEquals(LITERAL_FALSE, execute(JsonInvalidObject)); - assertEquals(LITERAL_FALSE, execute(JsonInvalidScalar)); - assertEquals(LITERAL_FALSE, execute(LITERAL_NULL)); - assertEquals(LITERAL_FALSE, execute(LITERAL_MISSING)); + assertEquals( + LITERAL_FALSE, + DSL.jsonValid(DSL.literal(ExprValueUtils.stringValue("{\"invalid\":\"json\", \"string\"}"))) + .valueOf()); + assertEquals( + LITERAL_FALSE, DSL.jsonValid(DSL.literal((ExprValueUtils.stringValue("abc")))).valueOf()); + assertEquals(LITERAL_FALSE, DSL.jsonValid(DSL.literal((LITERAL_NULL))).valueOf()); + assertEquals(LITERAL_FALSE, DSL.jsonValid(DSL.literal((LITERAL_MISSING))).valueOf()); } @Test @@ -66,16 +59,35 @@ public void json_valid_throws_ExpressionEvaluationException() { @Test public void json_valid_returns_true() { - assertEquals(LITERAL_TRUE, execute(JsonNestedObject)); - assertEquals(LITERAL_TRUE, execute(JsonObject)); - assertEquals(LITERAL_TRUE, execute(JsonArray)); - assertEquals(LITERAL_TRUE, execute(JsonScalarString)); - assertEquals(LITERAL_TRUE, execute(JsonEmptyString)); - } - private ExprValue execute(ExprValue jsonString) { - FunctionExpression exp = DSL.jsonValid(DSL.literal(jsonString)); - return exp.valueOf(); + List validJsonStrings = + List.of( + // test json objects are valid + "{\"a\":\"1\",\"b\":\"2\"}", + "{\"a\":1,\"b\":{\"c\":2,\"d\":3}}", + "{\"arr1\": [1,2,3], \"arr2\": [4,5,6]}", + + // test json arrays are valid + "[1, 2, 3, 4]", + "[{\"a\":1,\"b\":2}, {\"c\":3,\"d\":2}]", + + // test json scalars are valid + "\"abc\"", + "1234", + "true", + "false", + "null", + + // test empty string is valid + ""); + + validJsonStrings.stream() + .forEach( + str -> + assertEquals( + LITERAL_TRUE, + DSL.jsonValid(DSL.literal((ExprValueUtils.stringValue(str)))).valueOf(), + String.format("String %s must be valid json", str))); } @Test From 1e2328606c375b166ef2a7335d7a89e0c37e80b7 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 15 Jan 2025 11:07:48 -0800 Subject: [PATCH 49/70] Add casting from undefined Signed-off-by: Andrew Carbonetto --- .../operator/convert/TypeCastOperators.java | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index cfd570ab89..960f6743c3 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -122,7 +122,9 @@ private static DefaultFunctionResolver castToInt() { impl( nullMissingHandling((v) -> new ExprIntegerValue(v.booleanValue() ? 1 : 0)), INTEGER, - BOOLEAN)); + BOOLEAN), + impl(nullMissingHandling((v) -> v), INTEGER, UNDEFINED) + ); } private static DefaultFunctionResolver castToLong() { @@ -136,7 +138,9 @@ private static DefaultFunctionResolver castToLong() { impl( nullMissingHandling((v) -> new ExprLongValue(v.booleanValue() ? 1L : 0L)), LONG, - BOOLEAN)); + BOOLEAN), + impl(nullMissingHandling((v) -> v), LONG, UNDEFINED) + ); } private static DefaultFunctionResolver castToFloat() { @@ -150,7 +154,9 @@ private static DefaultFunctionResolver castToFloat() { impl( nullMissingHandling((v) -> new ExprFloatValue(v.booleanValue() ? 1f : 0f)), FLOAT, - BOOLEAN)); + BOOLEAN), + impl(nullMissingHandling((v) -> v), FLOAT, UNDEFINED) + ); } private static DefaultFunctionResolver castToDouble() { @@ -164,7 +170,9 @@ private static DefaultFunctionResolver castToDouble() { impl( nullMissingHandling((v) -> new ExprDoubleValue(v.booleanValue() ? 1D : 0D)), DOUBLE, - BOOLEAN)); + BOOLEAN), + impl(nullMissingHandling((v) -> v), DOUBLE, UNDEFINED) + ); } private static DefaultFunctionResolver castToBoolean() { @@ -176,7 +184,9 @@ private static DefaultFunctionResolver castToBoolean() { STRING), impl( nullMissingHandling((v) -> ExprBooleanValue.of(v.doubleValue() != 0)), BOOLEAN, DOUBLE), - impl(nullMissingHandling((v) -> v), BOOLEAN, BOOLEAN)); + impl(nullMissingHandling((v) -> v), BOOLEAN, BOOLEAN), + impl(nullMissingHandling((v) -> v), BOOLEAN, UNDEFINED) + ); } private static DefaultFunctionResolver castToIp() { From 343f5a2970948393caafd1b6b0b243fd7bf5640a Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Thu, 16 Jan 2025 14:48:51 -0800 Subject: [PATCH 50/70] Add cast to scalar from undefined expression Signed-off-by: Andrew Carbonetto --- .../operator/convert/TypeCastOperators.java | 23 +++--- .../convert/TypeCastOperatorTest.java | 53 ++++++++++++++ .../opensearch/sql/ppl/JsonFunctionsIT.java | 71 +++++++++++++++++++ integ-test/src/test/resources/json_test.json | 14 +++- 4 files changed, 145 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index 960f6743c3..1a43ef14e1 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -92,9 +92,8 @@ private static DefaultFunctionResolver castToByte() { STRING), impl(nullMissingHandling((v) -> new ExprByteValue(v.byteValue())), BYTE, DOUBLE), impl( - nullMissingHandling((v) -> new ExprByteValue(v.booleanValue() ? 1 : 0)), - BYTE, - BOOLEAN)); + nullMissingHandling((v) -> new ExprByteValue(v.booleanValue() ? 1 : 0)), BYTE, BOOLEAN), + impl(nullMissingHandling((v) -> v), BYTE, UNDEFINED)); } private static DefaultFunctionResolver castToShort() { @@ -108,7 +107,8 @@ private static DefaultFunctionResolver castToShort() { impl( nullMissingHandling((v) -> new ExprShortValue(v.booleanValue() ? 1 : 0)), SHORT, - BOOLEAN)); + BOOLEAN), + impl(nullMissingHandling((v) -> v), SHORT, UNDEFINED)); } private static DefaultFunctionResolver castToInt() { @@ -123,8 +123,7 @@ private static DefaultFunctionResolver castToInt() { nullMissingHandling((v) -> new ExprIntegerValue(v.booleanValue() ? 1 : 0)), INTEGER, BOOLEAN), - impl(nullMissingHandling((v) -> v), INTEGER, UNDEFINED) - ); + impl(nullMissingHandling((v) -> v), INTEGER, UNDEFINED)); } private static DefaultFunctionResolver castToLong() { @@ -139,8 +138,7 @@ private static DefaultFunctionResolver castToLong() { nullMissingHandling((v) -> new ExprLongValue(v.booleanValue() ? 1L : 0L)), LONG, BOOLEAN), - impl(nullMissingHandling((v) -> v), LONG, UNDEFINED) - ); + impl(nullMissingHandling((v) -> v), LONG, UNDEFINED)); } private static DefaultFunctionResolver castToFloat() { @@ -155,8 +153,7 @@ private static DefaultFunctionResolver castToFloat() { nullMissingHandling((v) -> new ExprFloatValue(v.booleanValue() ? 1f : 0f)), FLOAT, BOOLEAN), - impl(nullMissingHandling((v) -> v), FLOAT, UNDEFINED) - ); + impl(nullMissingHandling((v) -> v), FLOAT, UNDEFINED)); } private static DefaultFunctionResolver castToDouble() { @@ -171,8 +168,7 @@ private static DefaultFunctionResolver castToDouble() { nullMissingHandling((v) -> new ExprDoubleValue(v.booleanValue() ? 1D : 0D)), DOUBLE, BOOLEAN), - impl(nullMissingHandling((v) -> v), DOUBLE, UNDEFINED) - ); + impl(nullMissingHandling((v) -> v), DOUBLE, UNDEFINED)); } private static DefaultFunctionResolver castToBoolean() { @@ -185,8 +181,7 @@ private static DefaultFunctionResolver castToBoolean() { impl( nullMissingHandling((v) -> ExprBooleanValue.of(v.doubleValue() != 0)), BOOLEAN, DOUBLE), impl(nullMissingHandling((v) -> v), BOOLEAN, BOOLEAN), - impl(nullMissingHandling((v) -> v), BOOLEAN, UNDEFINED) - ); + impl(nullMissingHandling((v) -> v), BOOLEAN, UNDEFINED)); } private static DefaultFunctionResolver castToIp() { diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java index 27bd806c11..4ad28d76d0 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java @@ -165,6 +165,15 @@ void castBooleanToShort() { assertEquals(new ExprShortValue(0), expression.valueOf()); } + @Test + void castUndefinedToShort() { + Short value = 42; + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castShort(DSL.castJson(DSL.literal(value.toString()))); + assertEquals(SHORT, expression.type()); + assertEquals(new ExprShortValue(value), expression.valueOf()); + } + @Test void castBooleanToInt() { FunctionExpression expression = DSL.castInt(DSL.literal(true)); @@ -176,6 +185,15 @@ void castBooleanToInt() { assertEquals(new ExprIntegerValue(0), expression.valueOf()); } + @Test + void castUndefinedToInt() { + Integer value = 42; + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castInt(DSL.castJson(DSL.literal(value.toString()))); + assertEquals(INTEGER, expression.type()); + assertEquals(new ExprIntegerValue(value), expression.valueOf()); + } + @ParameterizedTest(name = "castToLong({0})") @MethodSource({"numberData"}) void castToLong(ExprValue value) { @@ -208,6 +226,15 @@ void castBooleanToLong() { assertEquals(new ExprLongValue(0), expression.valueOf()); } + @Test + void castUndefinedToLong() { + Long value = 42l; + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castLong(DSL.castJson(DSL.literal(value.toString()))); + assertEquals(LONG, expression.type()); + assertEquals(new ExprLongValue(value), expression.valueOf()); + } + @ParameterizedTest(name = "castToFloat({0})") @MethodSource({"numberData"}) void castToFloat(ExprValue value) { @@ -240,6 +267,15 @@ void castBooleanToFloat() { assertEquals(new ExprFloatValue(0), expression.valueOf()); } + @Test + void castUndefinedToFloat() { + Float value = 23.45f; + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castFloat(DSL.castJson(DSL.literal(value.toString()))); + assertEquals(FLOAT, expression.type()); + assertEquals(new ExprFloatValue(value), expression.valueOf()); + } + @ParameterizedTest(name = "castToDouble({0})") @MethodSource({"numberData"}) void castToDouble(ExprValue value) { @@ -272,6 +308,15 @@ void castBooleanToDouble() { assertEquals(new ExprDoubleValue(0), expression.valueOf()); } + @Test + void castUndefinedToDouble() { + Double value = 23.45e5; + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castDouble(DSL.castJson(DSL.literal(value))); + assertEquals(DOUBLE, expression.type()); + assertEquals(new ExprDoubleValue(value), expression.valueOf()); + } + @ParameterizedTest(name = "castToBoolean({0})") @MethodSource({"numberData"}) void castToBoolean(ExprValue value) { @@ -301,6 +346,14 @@ void castBooleanToBoolean() { assertEquals(ExprBooleanValue.of(true), expression.valueOf()); } + @Test + void castUndefinedToBoolean() { + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castBoolean(DSL.castJson(DSL.literal("true"))); + assertEquals(BOOLEAN, expression.type()); + assertEquals(ExprBooleanValue.of(true), expression.valueOf()); + } + @Test void castToDate() { FunctionExpression expression = DSL.castDate(DSL.literal("2012-08-07")); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index eecf7fb338..e837057b68 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -73,6 +73,11 @@ public void test_cast_json() throws IOException { rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), + rows("json scalar int", 1234), + rows("json scalar float", 12.34f), + rows("json scalar double", 2.99792458e8), + rows("json scalar boolean true", true), + rows("json scalar boolean false", false), rows("json empty string", null)); } @@ -96,6 +101,72 @@ public void test_json() throws IOException { rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), + rows("json scalar int", 1234), + rows("json scalar long", 42), + rows("json scalar double", 2.99792458e8), + rows("json scalar boolean true", true), + rows("json scalar boolean false", false), rows("json empty string", null)); } + + @Test + public void test_cast_json_scalar_to_type() throws IOException { + // cast to integer + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | " + + "where test_name='json scalar int' | " + + "eval casted=cast(json(json_string) as int) | " + + "fields test_name, casted", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "integer")); + verifyDataRows(result, rows("json scalar int", 1234)); + + result = + executeQuery( + String.format( + "source=%s | " + + "where test_name='json scalar int' | " + + "eval casted=cast(json(json_string) as long) | " + + "fields test_name, casted", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "long")); + verifyDataRows(result, rows("json scalar int", 1234l)); + + result = + executeQuery( + String.format( + "source=%s | " + + "where test_name='json scalar float' | " + + "eval casted=cast(json(json_string) as float) | " + + "fields test_name, casted", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "float")); + verifyDataRows(result, rows("json scalar float", 12.34f)); + + result = + executeQuery( + String.format( + "source=%s | " + + "where test_name='json scalar double' | " + + "eval casted=cast(json(json_string) as double) | " + + "fields test_name, casted", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "double")); + verifyDataRows(result, rows("json scalar double", 2.99792458e8)); + + result = + executeQuery( + String.format( + "source=%s | where test_name='json scalar boolean true' OR test_name='json scalar" + + " boolean false' | eval casted=cast(json(json_string) as boolean) | fields" + + " test_name, casted", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "boolean")); + verifyDataRows( + result, rows("json scalar boolean true", true), rows("json scalar boolean false", false)); + } } diff --git a/integ-test/src/test/resources/json_test.json b/integ-test/src/test/resources/json_test.json index dae01ea4ce..acab339c01 100644 --- a/integ-test/src/test/resources/json_test.json +++ b/integ-test/src/test/resources/json_test.json @@ -7,8 +7,18 @@ {"index":{"_id":"3"}} {"test_name":"json scalar string", "json_string":"\"abc\""} {"index":{"_id":"4"}} -{"test_name":"json empty string","json_string":""} +{"test_name":"json scalar int", "json_string":"1234"} {"index":{"_id":"5"}} -{"test_name":"json invalid object", "json_string":"{\"invalid\":\"json\", \"string\"}"} +{"test_name":"json scalar float", "json_string":"12.34"} {"index":{"_id":"6"}} +{"test_name":"json scalar double", "json_string":"2.99792458e8"} +{"index":{"_id":"7"}} +{"test_name":"json scalar boolean true", "json_string":"true"} +{"index":{"_id":"8"}} +{"test_name":"json scalar boolean false", "json_string":"false"} +{"index":{"_id":"9"}} +{"test_name":"json empty string", "json_string":""} +{"index":{"_id":"10"}} +{"test_name":"json invalid object", "json_string":"{\"invalid\":\"json\", \"string\"}"} +{"index":{"_id":"11"}} {"test_name":"json null", "json_string":null} From e8b6df3e6ab0ce616b0ba1c9868dca19044cdfad Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Thu, 16 Jan 2025 15:34:17 -0800 Subject: [PATCH 51/70] Add test for missing/null Signed-off-by: Andrew Carbonetto --- .../opensearch/sql/expression/json/JsonFunctionsTest.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 4ba8b69e1c..1cbd71b806 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -74,6 +74,7 @@ public void json_valid_returns_true() { // test json scalars are valid "\"abc\"", "1234", + "12.34", "true", "false", "null", @@ -155,10 +156,16 @@ void json_returnsScalar() { assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); + assertEquals(new ExprDoubleValue(12.34), DSL.json_function(DSL.literal("12.34")).valueOf()); + assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal(LITERAL_NULL)).valueOf()); + + assertEquals(LITERAL_MISSING, DSL.json_function(DSL.literal(LITERAL_MISSING)).valueOf()); + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); assertEquals( From ab9be7533c3d497b5e13c6cc36337312264247c4 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Fri, 17 Jan 2025 09:58:42 -0800 Subject: [PATCH 52/70] Clean up merge conflicts Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/expression/DSL.java | 8 ++-- .../sql/expression/json/JsonFunctions.java | 6 --- .../operator/convert/TypeCastOperators.java | 5 ++- .../org/opensearch/sql/utils/JsonUtils.java | 20 +++++++++- .../expression/json/JsonFunctionsTest.java | 38 ++++++++++--------- .../convert/TypeCastOperatorTest.java | 14 ++++--- .../opensearch/sql/ppl/JsonFunctionsIT.java | 7 +++- 7 files changed, 59 insertions(+), 39 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index a50e3d1470..966077c790 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -687,6 +687,10 @@ public static FunctionExpression jsonValid(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.JSON_VALID, expressions); } + public static FunctionExpression stringToJson(Expression value) { + return compile(FunctionProperties.None, BuiltinFunctionName.JSON, value); + } + public static Aggregator avg(Expression... expressions) { return aggregate(BuiltinFunctionName.AVG, expressions); } @@ -977,10 +981,6 @@ public static FunctionExpression utc_timestamp( return compile(functionProperties, BuiltinFunctionName.UTC_TIMESTAMP, args); } - public static FunctionExpression json_function(Expression value) { - return compile(FunctionProperties.None, BuiltinFunctionName.JSON, value); - } - @SuppressWarnings("unchecked") private static T compile( FunctionProperties functionProperties, BuiltinFunctionName bfn, Expression... args) { diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 64f84c44f7..75f134aa4e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -5,12 +5,6 @@ package org.opensearch.sql.expression.json; -import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.experimental.UtilityClass; -import org.opensearch.sql.expression.function.BuiltinFunctionName; -import org.opensearch.sql.expression.function.BuiltinFunctionRepository; -import org.opensearch.sql.expression.function.DefaultFunctionResolver; - import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index 1a43ef14e1..c1391ac9ab 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -92,8 +92,9 @@ private static DefaultFunctionResolver castToByte() { STRING), impl(nullMissingHandling((v) -> new ExprByteValue(v.byteValue())), BYTE, DOUBLE), impl( - nullMissingHandling((v) -> new ExprByteValue(v.booleanValue() ? 1 : 0)), BYTE, BOOLEAN), - impl(nullMissingHandling((v) -> v), BYTE, UNDEFINED)); + nullMissingHandling((v) -> new ExprByteValue(v.booleanValue() ? 1 : 0)), + BYTE, + BOOLEAN)); } private static DefaultFunctionResolver castToShort() { diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index c5f7031b13..63cbaf4a99 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -12,9 +12,11 @@ import java.util.List; import java.util.Map; import lombok.experimental.UtilityClass; +import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDoubleValue; import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprNullValue; import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; @@ -27,7 +29,7 @@ public class JsonUtils { * Checks if given JSON string can be parsed as valid JSON. * * @param jsonExprValue JSON string (e.g. "{\"hello\": \"world\"}"). - * @return true if the string can be parsed as valid JSON, else false. + * @return true if the string can be parsed as valid JSON, else false (including null or missing). */ public static ExprValue isValidJson(ExprValue jsonExprValue) { ObjectMapper objectMapper = new ObjectMapper(); @@ -44,7 +46,21 @@ public static ExprValue isValidJson(ExprValue jsonExprValue) { } } - /** Converts a JSON encoded string to an Expression object. */ + /** + * Converts a JSON encoded string to a {@link ExprValue}. Expression type will be UNDEFINED. + * + * @param json JSON string (e.g. "{\"hello\": \"world\"}"). + * @return ExprValue returns an expression that best represents the provided JSON-encoded string. + *
    + *
  1. {@link ExprTupleValue} if the JSON is an object + *
  2. {@link ExprCollectionValue} if the JSON is an array + *
  3. {@link ExprDoubleValue} if the JSON is a floating-point number scalar + *
  4. {@link ExprIntegerValue} if the JSON is an integral number scalar + *
  5. {@link ExprStringValue} if the JSON is a string scalar + *
  6. {@link ExprBooleanValue} if the JSON is a boolean scalar + *
  7. {@link ExprNullValue} if the JSON is null, empty, or invalid + *
+ */ public static ExprValue castJson(ExprValue json) { ObjectMapper objectMapper = new ObjectMapper(); JsonNode jsonNode; diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 1cbd71b806..26b8ed89fd 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -51,10 +51,8 @@ public void json_valid_returns_false() { @Test public void json_valid_throws_ExpressionEvaluationException() { assertThrows( - ExpressionEvaluationException.class, () -> execute(ExprValueUtils.booleanValue(true))); - - // caught by nullMissingHandling and returns null - assertEquals(LITERAL_NULL, execute(LITERAL_NULL)); + ExpressionEvaluationException.class, + () -> DSL.jsonValid(DSL.literal((ExprValueUtils.booleanValue(true)))).valueOf()); } @Test @@ -115,12 +113,16 @@ void json_returnsJsonObject() { ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); // exercise - exp = DSL.json_function(DSL.literal(objectJson)); + exp = DSL.stringToJson(DSL.literal(objectJson)); // Verify var value = exp.valueOf(); assertTrue(value instanceof ExprTupleValue); assertEquals(expectedTupleExpr, value); + + // also test the empty object case + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), DSL.stringToJson(DSL.literal("{}")).valueOf()); } @Test @@ -141,35 +143,35 @@ void json_returnsJsonArray() { LITERAL_NULL)); // exercise - exp = DSL.json_function(DSL.literal(arrayJson)); + exp = DSL.stringToJson(DSL.literal(arrayJson)); // Verify var value = exp.valueOf(); assertTrue(value instanceof ExprCollectionValue); assertEquals(expectedArrayExpr, value); + + // also test the empty-array case + assertEquals(new ExprCollectionValue(List.of()), DSL.stringToJson(DSL.literal("[]")).valueOf()); } @Test void json_returnsScalar() { assertEquals( - new ExprStringValue("foobar"), DSL.json_function(DSL.literal("\"foobar\"")).valueOf()); - - assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); + new ExprStringValue("foobar"), DSL.stringToJson(DSL.literal("\"foobar\"")).valueOf()); - assertEquals(new ExprDoubleValue(12.34), DSL.json_function(DSL.literal("12.34")).valueOf()); + assertEquals(new ExprIntegerValue(1234), DSL.stringToJson(DSL.literal("1234")).valueOf()); - assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); + assertEquals(new ExprDoubleValue(12.34), DSL.stringToJson(DSL.literal("12.34")).valueOf()); - assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); + assertEquals(LITERAL_TRUE, DSL.stringToJson(DSL.literal("true")).valueOf()); - assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal(LITERAL_NULL)).valueOf()); + assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal("null")).valueOf()); - assertEquals(LITERAL_MISSING, DSL.json_function(DSL.literal(LITERAL_MISSING)).valueOf()); + assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal(LITERAL_NULL)).valueOf()); - assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); + assertEquals(LITERAL_MISSING, DSL.stringToJson(DSL.literal(LITERAL_MISSING)).valueOf()); - assertEquals( - ExprTupleValue.fromExprValueMap(Map.of()), DSL.json_function(DSL.literal("{}")).valueOf()); + assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal("")).valueOf()); } @Test @@ -181,7 +183,7 @@ void json_returnsSemanticCheckException() { // missing bracket assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); - // mnissing quote + // missing quote assertThrows( SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); } diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java index 4ad28d76d0..ff0c8bcc01 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java @@ -312,7 +312,7 @@ void castBooleanToDouble() { void castUndefinedToDouble() { Double value = 23.45e5; // json cast is an UNDEFINED type expression - FunctionExpression expression = DSL.castDouble(DSL.castJson(DSL.literal(value))); + FunctionExpression expression = DSL.castDouble(DSL.castJson(DSL.literal(value.toString()))); assertEquals(DOUBLE, expression.type()); assertEquals(new ExprDoubleValue(value), expression.valueOf()); } @@ -480,6 +480,10 @@ void castJson_returnsJsonObject() { var value = exp.valueOf(); assertTrue(value instanceof ExprTupleValue); assertEquals(expectedTupleExpr, value); + + // also test the empty-object case + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), DSL.castJson(DSL.literal("{}")).valueOf()); } @Test @@ -506,6 +510,9 @@ void castJson_returnsJsonArray() { var value = exp.valueOf(); assertTrue(value instanceof ExprCollectionValue); assertEquals(expectedArrayExpr, value); + + // also test the empty-array case + assertEquals(new ExprCollectionValue(List.of()), DSL.castJson(DSL.literal("[]")).valueOf()); } @Test @@ -525,11 +532,6 @@ void castJson_returnsScalar() { String empty = ""; assertEquals(LITERAL_NULL, DSL.castJson(DSL.literal(empty)).valueOf()); - - String emptyObject = "{}"; - assertEquals( - ExprTupleValue.fromExprValueMap(Map.of()), - DSL.castJson(DSL.literal(emptyObject)).valueOf()); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index e837057b68..9ed61c33a4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -40,6 +40,11 @@ public void test_json_valid() throws IOException { rows("json object"), rows("json array"), rows("json scalar string"), + rows("json scalar int"), + rows("json scalar float"), + rows("json scalar double"), + rows("json scalar boolean true"), + rows("json scalar boolean false"), rows("json empty string")); } @@ -102,7 +107,7 @@ public void test_json() throws IOException { rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), rows("json scalar int", 1234), - rows("json scalar long", 42), + rows("json scalar float", 12.34), rows("json scalar double", 2.99792458e8), rows("json scalar boolean true", true), rows("json scalar boolean false", false), From 788be9d6e91e4c647cc14859e3b66fab40c5324d Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Fri, 17 Jan 2025 15:24:03 -0800 Subject: [PATCH 53/70] Fix jacoco coverage Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/expression/json/JsonFunctionsTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 26b8ed89fd..1f39b5be9b 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -164,6 +164,7 @@ void json_returnsScalar() { assertEquals(new ExprDoubleValue(12.34), DSL.stringToJson(DSL.literal("12.34")).valueOf()); assertEquals(LITERAL_TRUE, DSL.stringToJson(DSL.literal("true")).valueOf()); + assertEquals(LITERAL_FALSE, DSL.stringToJson(DSL.literal("false")).valueOf()); assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal("null")).valueOf()); From a9721bfb2269524e83ba381eb32ed3e468c8fc5f Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Fri, 17 Jan 2025 15:44:23 -0800 Subject: [PATCH 54/70] Move to Switch by json type Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/utils/JsonUtils.java | 57 +++++++++---------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 63cbaf4a99..6120d11bb5 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -75,36 +75,33 @@ public static ExprValue castJson(ExprValue json) { } private static ExprValue processJsonNode(JsonNode jsonNode) { - if (jsonNode.isFloatingPointNumber()) { - return new ExprDoubleValue(jsonNode.asDouble()); + switch (jsonNode.getNodeType()) { + case ARRAY: + List elements = new LinkedList<>(); + for (var iter = jsonNode.iterator(); iter.hasNext(); ) { + jsonNode = iter.next(); + elements.add(processJsonNode(jsonNode)); + } + return new ExprCollectionValue(elements); + case OBJECT: + Map values = new LinkedHashMap<>(); + for (var iter = jsonNode.fields(); iter.hasNext(); ) { + Map.Entry entry = iter.next(); + values.put(entry.getKey(), processJsonNode(entry.getValue())); + } + return ExprTupleValue.fromExprValueMap(values); + case STRING: + return new ExprStringValue(jsonNode.asText()); + case NUMBER: + if (jsonNode.isFloatingPointNumber()) { + return new ExprDoubleValue(jsonNode.asDouble()); + } + return new ExprIntegerValue(jsonNode.asLong()); + case BOOLEAN: + return jsonNode.asBoolean() ? LITERAL_TRUE : LITERAL_FALSE; + default: + // in all other cases, return null + return LITERAL_NULL; } - if (jsonNode.isIntegralNumber()) { - return new ExprIntegerValue(jsonNode.asLong()); - } - if (jsonNode.isBoolean()) { - return jsonNode.asBoolean() ? LITERAL_TRUE : LITERAL_FALSE; - } - if (jsonNode.isTextual()) { - return new ExprStringValue(jsonNode.asText()); - } - if (jsonNode.isArray()) { - List elements = new LinkedList<>(); - for (var iter = jsonNode.iterator(); iter.hasNext(); ) { - jsonNode = iter.next(); - elements.add(processJsonNode(jsonNode)); - } - return new ExprCollectionValue(elements); - } - if (jsonNode.isObject()) { - Map values = new LinkedHashMap<>(); - for (var iter = jsonNode.fields(); iter.hasNext(); ) { - Map.Entry entry = iter.next(); - values.put(entry.getKey(), processJsonNode(entry.getValue())); - } - return ExprTupleValue.fromExprValueMap(values); - } - - // in all other cases, return null - return LITERAL_NULL; } } From 018e462583fd26c2f333b7e554fe2ad309556eb8 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Mon, 20 Jan 2025 11:01:35 -0800 Subject: [PATCH 55/70] functionality implemented Signed-off-by: Kenrick Yap --- core/build.gradle | 1 + .../org/opensearch/sql/expression/DSL.java | 4 ++ .../function/BuiltinFunctionName.java | 1 + .../sql/expression/json/JsonFunctions.java | 7 +++ .../org/opensearch/sql/utils/JsonUtils.java | 63 +++++++++++++++++-- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 1 + 7 files changed, 74 insertions(+), 4 deletions(-) diff --git a/core/build.gradle b/core/build.gradle index c596251342..d16d1b31fb 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -54,6 +54,7 @@ dependencies { api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" + api group: 'com.jayway.jsonpath', name: 'json-path', version: '2.9.0' api group: 'com.google.code.gson', name: 'gson', version: '2.8.9' api group: 'com.tdunning', name: 't-digest', version: '3.3' api project(':common') diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index a50e3d1470..7b4aa337e0 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -687,6 +687,10 @@ public static FunctionExpression jsonValid(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.JSON_VALID, expressions); } + public static FunctionExpression jsonExtract(Expression... expressions) { + return compile(FunctionProperties.None, BuiltinFunctionName.JSON_EXTRACT, expressions); + } + public static Aggregator avg(Expression... expressions) { return aggregate(BuiltinFunctionName.AVG, expressions); } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index cd309a712d..261624a822 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -207,6 +207,7 @@ public enum BuiltinFunctionName { /** Json Functions. */ JSON_VALID(FunctionName.of("json_valid")), JSON(FunctionName.of("json")), + JSON_EXTRACT(FunctionName.of("json_extract")), /** NULL Test. */ IS_NULL(FunctionName.of("is null")), diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 75f134aa4e..dfc4d32ef6 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -23,6 +23,7 @@ public class JsonFunctions { public void register(BuiltinFunctionRepository repository) { repository.register(jsonValid()); repository.register(jsonFunction()); + repository.register(jsonExtract()); } private DefaultFunctionResolver jsonValid() { @@ -35,4 +36,10 @@ private DefaultFunctionResolver jsonFunction() { BuiltinFunctionName.JSON.getName(), impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING)); } + + private DefaultFunctionResolver jsonExtract() { + return define( + BuiltinFunctionName.JSON_EXTRACT.getName(), + impl(nullMissingHandling(JsonUtils::extractJson), UNDEFINED, STRING, STRING)); + } } diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index c5f7031b13..8471c58b3e 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -7,10 +7,22 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.jayway.jsonpath.Configuration; +import com.jayway.jsonpath.DocumentContext; +import com.jayway.jsonpath.InvalidJsonException; +import com.jayway.jsonpath.JsonPath; + +import java.io.ObjectInputFilter; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; + +import com.jayway.jsonpath.Option; +import com.jayway.jsonpath.PathNotFoundException; +import com.jayway.jsonpath.spi.json.JacksonJsonProvider; +import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; import lombok.experimental.UtilityClass; import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDoubleValue; @@ -46,16 +58,59 @@ public static ExprValue isValidJson(ExprValue jsonExprValue) { /** Converts a JSON encoded string to an Expression object. */ public static ExprValue castJson(ExprValue json) { + JsonNode jsonNode = jsonToNode(json); + return processJsonNode(jsonNode); + } + + public static ExprValue extractJson(ExprValue json, ExprValue path) { + String jsonString = json.stringValue(); + String jsonPath = path.stringValue(); + + try { + Configuration config = Configuration.builder() + .options(Option.AS_PATH_LIST) + .build(); + List resultPaths = JsonPath.using(config).parse(jsonString).read(jsonPath); + + List elements = new LinkedList<>(); + + for (String resultPath : resultPaths) { + Object result = JsonPath.parse(jsonString).read(resultPath); + String resultJsonString = new ObjectMapper().writeValueAsString(result); + try { + elements.add(processJsonNode(jsonStringToNode(resultJsonString))); + } catch (SemanticCheckException e) { + elements.add(new ExprStringValue(resultJsonString)); + } + } + + if (elements.size() == 1) { + return elements.get(0); + } else { + return new ExprCollectionValue(elements); + } + } catch (PathNotFoundException e) { + return LITERAL_NULL; + } catch (InvalidJsonException | JsonProcessingException e) { + final String errorFormat = "JSON string '%s' is not valid. Error details: %s"; + throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e); + } + } + + private static JsonNode jsonToNode(ExprValue json) { + return jsonStringToNode(json.stringValue()); + } + + private static JsonNode jsonStringToNode(String jsonString) { ObjectMapper objectMapper = new ObjectMapper(); JsonNode jsonNode; try { - jsonNode = objectMapper.readTree(json.stringValue()); + jsonNode = objectMapper.readTree(jsonString); } catch (JsonProcessingException e) { final String errorFormat = "JSON string '%s' is not valid. Error details: %s"; - throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e); + throw new SemanticCheckException(String.format(errorFormat, jsonString, e.getMessage()), e); } - - return processJsonNode(jsonNode); + return jsonNode; } private static ExprValue processJsonNode(JsonNode jsonNode) { diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 31a668be00..267698ee06 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -335,6 +335,7 @@ CIDRMATCH: 'CIDRMATCH'; // JSON FUNCTIONS JSON_VALID: 'JSON_VALID'; JSON: 'JSON'; +JSON_EXTRACT: 'JSON_EXTRACT'; // FLOWCONTROL FUNCTIONS IFNULL: 'IFNULL'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 5d1f29614d..b9a27dcd66 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -704,6 +704,7 @@ positionFunctionName jsonFunctionName : JSON + | JSON_EXTRACT ; // operators From c6c6cc1562440f412d327a91972031cab975476a Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Tue, 21 Jan 2025 10:43:36 -0800 Subject: [PATCH 56/70] Remove conflicted files Signed-off-by: Andrew Carbonetto --- .../sql/expression/datetime/DateTimeFunctionTest.java | 2 -- .../org/opensearch/sql/expression/datetime/YearweekTest.java | 2 -- 2 files changed, 4 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index 115898e349..ad15dadfb7 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -1230,8 +1230,6 @@ public void testWeekFormats( expectedInteger); } - // subtracting 1 as a temporary fix for year 2024. - // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test public void testWeekOfYearWithTimeType() { LocalDate today = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index ee4df23be4..d944f7c85c 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -99,8 +99,6 @@ public void testYearweekWithoutMode() { assertEquals(eval(expression), eval(expressionWithoutMode)); } - // subtracting 1 as a temporary fix for year 2024. - // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test public void testYearweekWithTimeType() { int expected = getYearWeekBeforeSunday(LocalDate.now(functionProperties.getQueryStartClock())); From a5652eabeb6cf42ff71851f2cb5c5cfc5b55c02d Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Tue, 21 Jan 2025 14:51:31 -0800 Subject: [PATCH 57/70] Add doctext row Signed-off-by: Andrew Carbonetto --- docs/user/ppl/functions/json.rst | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index bbc972ba73..77d9d00f45 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -50,12 +50,13 @@ Return type: BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY Example:: > source=json_test | where json_valid(json_string) | eval json=json(json_string) | fields test_name, json_string, json - fetched rows / total rows = 4/4 - +---------------------+------------------------------+---------------+ - | test_name | json_string | json | - |---------------------|------------------------------|---------------| - | json object | {"a":"1","b":"2"} | {a:"1",b:"2"} | - | json array | [1, 2, 3, 4] | [1,2,3,4] | - | json scalar string | "abc" | "abc" | - | json empty string | | null | - +---------------------+------------------------------+---------------+ + fetched rows / total rows = 5/5 + +---------------------+---------------------------------+-------------------------+ + | test_name | json_string | json | + |---------------------|---------------------------------|-------------------------| + | json nested object | {"a":"1","b":{"c":"2","d":"3"}} | {a:"1",b:{c:"2",d:"3"}} | + | json object | {"a":"1","b":"2"} | {a:"1",b:"2"} | + | json array | [1, 2, 3, 4] | [1,2,3,4] | + | json scalar string | "abc" | "abc" | + | json empty string | | null | + +---------------------+---------------------------------+-------------------------+ From 2cd10a2ca9e4780328de92706803f05183f32c3d Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 21 Jan 2025 16:19:48 -0800 Subject: [PATCH 58/70] added integ-test and doc test Signed-off-by: Kenrick Yap --- .../org/opensearch/sql/utils/JsonUtils.java | 9 ++- docs/user/ppl/functions/json.rst | 67 ++++++++++++++++--- doctest/test_data/json_test.json | 1 + .../opensearch/sql/ppl/JsonFunctionsIT.java | 36 +++++++++- integ-test/src/test/resources/json_test.json | 2 + 5 files changed, 97 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 8471c58b3e..4e149c7ad0 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -8,21 +8,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.jayway.jsonpath.Configuration; -import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.InvalidJsonException; import com.jayway.jsonpath.JsonPath; -import java.io.ObjectInputFilter; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import com.jayway.jsonpath.Option; import com.jayway.jsonpath.PathNotFoundException; -import com.jayway.jsonpath.spi.json.JacksonJsonProvider; -import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; import lombok.experimental.UtilityClass; import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDoubleValue; @@ -66,6 +61,10 @@ public static ExprValue extractJson(ExprValue json, ExprValue path) { String jsonString = json.stringValue(); String jsonPath = path.stringValue(); + if (jsonString.equals("")) { + return LITERAL_NULL; + } + try { Configuration config = Configuration.builder() .options(Option.AS_PATH_LIST) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 82c0da23a6..1f2c9e7b23 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -24,16 +24,17 @@ Example:: > source=json_test | eval is_valid = json_valid(json_string) | fields test_name, json_string, is_valid fetched rows / total rows = 6/6 - +---------------------+---------------------------------+----------+ - | test_name | json_string | is_valid | - |---------------------|---------------------------------|----------| - | json nested object | {"a":"1","b":{"c":"2","d":"3"}} | True | - | json object | {"a":"1","b":"2"} | True | - | json array | [1, 2, 3, 4] | True | - | json scalar string | "abc" | True | - | json empty string | | True | - | json invalid object | {"invalid":"json", "string"} | False | - +---------------------+---------------------------------+----------+ + +---------------------+-------------------------------------+----------+ + | test_name | json_string | is_valid | + |---------------------|-------------------------------------|----------| + | json nested object | {"a":"1","b":{"c":"2","d":"3"}} | True | + | json nested list | {"a":"1","b":[{"c":"2"},{"c":"3"}]} | True | + | json object | {"a":"1","b":"2"} | True | + | json array | [1, 2, 3, 4] | True | + | json scalar string | "abc" | True | + | json empty string | | True | + | json invalid object | {"invalid":"json", "string"} | False | + +---------------------+-------------------------------------+----------+ JSON ---------- @@ -59,3 +60,49 @@ Example:: | json scalar string | "abc" | "abc" | | json empty string | | null | +---------------------+------------------------------+---------------+ + +JSON_EXTRACT +____________ + +Description +>>>>>>>>>>> + +Usage: `json_extract(doc, path [, path]...)` Extracts a json value or scalar from a json document based on the path(s) specified. + +Argument type: STRING, STRING + +Return type: BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY + +- Returns a JSON array for multiple paths or if the path leads to an array. +- Return null if path is not valid. + +Example:: + + > source=json_test | where json_valid(json_string) | eval json_extract=json_extract(json_string, '$.b') | fields test_name, json_string, json_extract + fetched rows / total rows = 6/6 + +---------------------+-------------------------------------+-------------------+ + | test_name | json_string | json_extract | + |---------------------|-------------------------------------|-------------------| + | json nested object | {"a":"1","b":{"c":"2","d":"3"}} | {c:"2",d:"3"} | + | json nested list | {"a":"1","b":[{"c":"2"},{"c":"3"}]} | [{c:"2"},{c:"3"}] | + | json object | {"a":"1","b":"2"} | 2 | + | json array | [1, 2, 3, 4] | null | + | json scalar string | "abc" | null | + | json empty string | | null | + +---------------------+-------------------------------------+-------------------+ + + > source=json_test | where test_name="json nested list" | eval json_extract=json_extract('{"a":[{"b":1},{"b":2}]}', '$.b[1].c') + fetched rows / total rows = 1/1 + +---------------------+-------------------------------------+--------------+ + | test_name | json_string | json_extract | + |---------------------|-------------------------------------|--------------| + | json nested list | {"a":"1","b":[{"c":"2"},{"c":"3"}]} | 3 | + +---------------------+-------------------------------------+--------------+ + + > source=json_test | where test_name="json nested list" | eval json_extract=json_extract('{"a":[{"b":1},{"b":2}]}', '$.b[*].c') + fetched rows / total rows = 1/1 + +---------------------+-------------------------------------+--------------+ + | test_name | json_string | json_extract | + |---------------------|-------------------------------------|--------------| + | json nested list | {"a":"1","b":[{"c":"2"},{"c":"3"}]} | [2,3] | + +---------------------+-------------------------------------+--------------+ diff --git a/doctest/test_data/json_test.json b/doctest/test_data/json_test.json index 7494fc4aa9..63e7f15011 100644 --- a/doctest/test_data/json_test.json +++ b/doctest/test_data/json_test.json @@ -1,4 +1,5 @@ {"test_name":"json nested object", "json_string":"{\"a\":\"1\",\"b\":{\"c\":\"2\",\"d\":\"3\"}}"} +{"test_name":"json nested list", "json_string":"{\"a\":\"1\",\"b\":[{\"c\":\"2\"}, {\"c\":\"3\"}]}"} {"test_name":"json object", "json_string":"{\"a\":\"1\",\"b\":\"2\"}"} {"test_name":"json array", "json_string":"[1, 2, 3, 4]"} {"test_name":"json scalar string", "json_string":"\"abc\""} diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index eecf7fb338..ef55b51685 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -40,7 +40,9 @@ public void test_json_valid() throws IOException { rows("json object"), rows("json array"), rows("json scalar string"), - rows("json empty string")); + rows("json empty string"), + rows("json nested list") + ); } @Test @@ -73,7 +75,10 @@ public void test_cast_json() throws IOException { rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), - rows("json empty string", null)); + rows("json empty string", null), + rows("json nested list", + new JSONObject(Map.of("a", "1", "b", List.of(Map.of("c", "2"), Map.of("c", "3"))))) + ); } @Test @@ -96,6 +101,31 @@ public void test_json() throws IOException { rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), - rows("json empty string", null)); + rows("json empty string", null), + rows("json nested list", + new JSONObject(Map.of("a", "1", "b", List.of(Map.of("c", "2"), Map.of("c", "3"))))) + ); + } + + @Test + public void test_json_extract() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where json_valid(json_string) | eval json_extract=json_extract(json_string, '$.b') | fields" + + " test_name, json_extract", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("json_extract", null, "undefined")); + verifyDataRows( + result, + rows("json nested object", new JSONObject(Map.of("c", "3"))), + rows("json object", "2"), + rows("json array", null), + rows("json scalar string", null), + rows("json empty string", null), + rows("json nested list", new JSONArray(List.of(Map.of("c","2"), Map.of("c","3")))) + ); } } diff --git a/integ-test/src/test/resources/json_test.json b/integ-test/src/test/resources/json_test.json index dae01ea4ce..1a317fb587 100644 --- a/integ-test/src/test/resources/json_test.json +++ b/integ-test/src/test/resources/json_test.json @@ -12,3 +12,5 @@ {"test_name":"json invalid object", "json_string":"{\"invalid\":\"json\", \"string\"}"} {"index":{"_id":"6"}} {"test_name":"json null", "json_string":null} +{"index":{"_id":"7"}} +{"test_name":"json nested list", "json_string":"{\"a\":\"1\",\"b\":[{\"c\":\"2\"}, {\"c\":\"3\"}]}"} From cd78ddd93560ed79fb6f0d5fa815b9a468027e38 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Wed, 22 Jan 2025 14:07:31 -0800 Subject: [PATCH 59/70] fixed integ tests Signed-off-by: Kenrick Yap --- .../opensearch/sql/expression/json/JsonFunctions.java | 2 +- .../main/java/org/opensearch/sql/utils/JsonUtils.java | 10 ++-------- .../java/org/opensearch/sql/ppl/JsonFunctionsIT.java | 6 ++---- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index dfc4d32ef6..30c5cde162 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -40,6 +40,6 @@ private DefaultFunctionResolver jsonFunction() { private DefaultFunctionResolver jsonExtract() { return define( BuiltinFunctionName.JSON_EXTRACT.getName(), - impl(nullMissingHandling(JsonUtils::extractJson), UNDEFINED, STRING, STRING)); + impl(JsonUtils::extractJson, UNDEFINED, STRING, STRING)); } } diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 4e149c7ad0..5b0e0af69b 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -72,15 +72,9 @@ public static ExprValue extractJson(ExprValue json, ExprValue path) { List resultPaths = JsonPath.using(config).parse(jsonString).read(jsonPath); List elements = new LinkedList<>(); - for (String resultPath : resultPaths) { Object result = JsonPath.parse(jsonString).read(resultPath); - String resultJsonString = new ObjectMapper().writeValueAsString(result); - try { - elements.add(processJsonNode(jsonStringToNode(resultJsonString))); - } catch (SemanticCheckException e) { - elements.add(new ExprStringValue(resultJsonString)); - } + elements.add(ExprValueUtils.fromObjectValue(result)); } if (elements.size() == 1) { @@ -90,7 +84,7 @@ public static ExprValue extractJson(ExprValue json, ExprValue path) { } } catch (PathNotFoundException e) { return LITERAL_NULL; - } catch (InvalidJsonException | JsonProcessingException e) { + } catch (InvalidJsonException e) { final String errorFormat = "JSON string '%s' is not valid. Error details: %s"; throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e); } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index ef55b51685..3b15ca99e9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -110,14 +110,12 @@ public void test_json() throws IOException { @Test public void test_json_extract() throws IOException { JSONObject result; - result = executeQuery( String.format( - "source=%s | where json_valid(json_string) | eval json_extract=json_extract(json_string, '$.b') | fields" - + " test_name, json_extract", + "source=%s | where json_valid(json_string) | eval extracted=json_extract(json_string, '$.b') | fields test_name, extracted", TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string"), schema("json_extract", null, "undefined")); + verifySchema(result, schema("test_name", null, "string"), schema("extracted", null, "undefined")); verifyDataRows( result, rows("json nested object", new JSONObject(Map.of("c", "3"))), From afb385fb059933cc86c8081da7218346158f7ec3 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 23 Jan 2025 00:23:05 -0800 Subject: [PATCH 60/70] unit tests Signed-off-by: Kenrick Yap --- .../org/opensearch/sql/utils/JsonUtils.java | 4 + .../expression/json/JsonFunctionsTest.java | 104 +++++++++++++++++- 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 5b0e0af69b..c6196c0c05 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -9,6 +9,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.InvalidJsonException; +import com.jayway.jsonpath.InvalidPathException; import com.jayway.jsonpath.JsonPath; import java.util.LinkedHashMap; @@ -84,6 +85,9 @@ public static ExprValue extractJson(ExprValue json, ExprValue path) { } } catch (PathNotFoundException e) { return LITERAL_NULL; + } catch (InvalidPathException e) { + final String errorFormat = "JSON path '%s' is not valid. Error details: %s"; + throw new SemanticCheckException(String.format(errorFormat, path, e.getMessage()), e); } catch (InvalidJsonException e) { final String errorFormat = "JSON string '%s' is not valid. Error details: %s"; throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e); diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 89ea57c2e4..cede667260 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -16,12 +16,14 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; + import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprFloatValue; import org.opensearch.sql.data.model.ExprIntegerValue; import org.opensearch.sql.data.model.ExprLongValue; import org.opensearch.sql.data.model.ExprNullValue; @@ -32,6 +34,7 @@ import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.FunctionExpression; @ExtendWith(MockitoExtension.class) @@ -159,8 +162,107 @@ void json_returnsSemanticCheckException() { // missing bracket assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); - // mnissing quote + // missing quote assertThrows( SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); } + + @Test + void json_extract_return_object() { + List validJsonStrings = + List.of( + // test json objects are valid + "{\"a\":\"1\",\"b\":\"2\"}", + "{\"a\":1,\"b\":{\"c\":2,\"d\":3}}", + "{\"arr1\": [1,2,3], \"arr2\": [4,5,6]}", + + // test json arrays are valid + "[1, 2, 3, 4]", + "[{\"a\":1,\"b\":2}, {\"c\":3,\"d\":2}]", + + // test json scalars are valid + "\"abc\"", + "1234", + "12.34", + "true", + "false", + "null", + + // test empty string is valid + ""); + + validJsonStrings.stream() + .forEach( + str -> + assertEquals( + LITERAL_TRUE, + DSL.jsonValid(DSL.literal((ExprValueUtils.stringValue(str)))).valueOf(), + String.format("String %s must be valid json", str))); + } + + @Test + void json_extract_search_arrays() { + Expression jsonArray = DSL.literal(ExprValueUtils.stringValue("{\"a\":[1,2.3,\"abc\",true,null,{\"c\":1},[1,2,3]]}")); + List expectedExprValue = List.of( + new ExprIntegerValue(1), + new ExprFloatValue(2.3), + new ExprStringValue("abc"), + LITERAL_TRUE, + LITERAL_NULL, + ExprTupleValue.fromExprValueMap(Map.of("c", new ExprIntegerValue(1))), + new ExprCollectionValue(List.of(new ExprIntegerValue(1), new ExprIntegerValue(2), new ExprIntegerValue(3))) + ); + + // extract specific index from JSON list + for (int i = 0 ; i < expectedExprValue.size() ; i++ ) { + String path = String.format("$.a[%d]", i); + Expression pathExpr = DSL.literal(ExprValueUtils.stringValue(path)); + FunctionExpression expression = DSL.jsonExtract(jsonArray, pathExpr); + assertEquals(expectedExprValue.get(i), expression.valueOf()); + } + + // extract * from JSON list + Expression starPath = DSL.literal(ExprValueUtils.stringValue("$.a[*]")); + FunctionExpression starExpression = DSL.castInt(DSL.jsonExtract(jsonArray, starPath)); + assertEquals( + new ExprCollectionValue(expectedExprValue), starExpression.valueOf()); + } + + @Test + void json_extract_returns_null() { + List jsonStrings = + List.of( + "{\"a\":\"1\",\"b\":\"2\"}", + "{\"a\":1,\"b\":{\"c\":2,\"d\":3}}", + "{\"arr1\": [1,2,3], \"arr2\": [4,5,6]}", + "[1, 2, 3, 4]", + "[{\"a\":1,\"b\":2}, {\"c\":3,\"d\":2}]", + "\"abc\"", + "1234", + "12.34", + "true", + "false", + "null", + ""); + + jsonStrings.stream() + .forEach( + str -> + assertEquals( + LITERAL_NULL, + DSL.jsonExtract( + DSL.literal((ExprValueUtils.stringValue(str))), + DSL.literal("$.a.path_not_found_key")).valueOf(), + String.format("JSON string %s should return null", str))); + } + + @Test + void json_returns_SemanticCheckException() { + // invalid path + assertThrows( + SemanticCheckException.class, () -> DSL.jsonExtract(DSL.literal("invalid"), DSL.literal("invalid")).valueOf()); + + // invalid json + assertThrows(SemanticCheckException.class, () -> DSL.jsonExtract(DSL.literal("{\"invalid\":\"json\", \"string\"}"), DSL.literal("invalid")).valueOf()); + } } From 794db8ab3ff1ecb930b6e0e2a2936ccebd27347a Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 23 Jan 2025 04:45:44 -0800 Subject: [PATCH 61/70] finnished unit tests Signed-off-by: Kenrick Yap --- .../org/opensearch/sql/utils/JsonUtils.java | 10 +- .../expression/json/JsonFunctionsTest.java | 122 ++++++++++-------- .../opensearch/sql/ppl/JsonFunctionsIT.java | 24 ++-- 3 files changed, 82 insertions(+), 74 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index c6196c0c05..f3239ae25a 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -11,14 +11,12 @@ import com.jayway.jsonpath.InvalidJsonException; import com.jayway.jsonpath.InvalidPathException; import com.jayway.jsonpath.JsonPath; - +import com.jayway.jsonpath.Option; +import com.jayway.jsonpath.PathNotFoundException; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; - -import com.jayway.jsonpath.Option; -import com.jayway.jsonpath.PathNotFoundException; import lombok.experimental.UtilityClass; import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDoubleValue; @@ -67,9 +65,7 @@ public static ExprValue extractJson(ExprValue json, ExprValue path) { } try { - Configuration config = Configuration.builder() - .options(Option.AS_PATH_LIST) - .build(); + Configuration config = Configuration.builder().options(Option.AS_PATH_LIST).build(); List resultPaths = JsonPath.using(config).parse(jsonString).read(jsonPath); List elements = new LinkedList<>(); diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 5729b229a2..bb73a27fe9 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -8,7 +8,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; - import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; @@ -17,7 +16,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; - import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; @@ -168,54 +166,25 @@ void json_returnsSemanticCheckException() { SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); } - @Test - void json_extract_return_object() { - List validJsonStrings = - List.of( - // test json objects are valid - "{\"a\":\"1\",\"b\":\"2\"}", - "{\"a\":1,\"b\":{\"c\":2,\"d\":3}}", - "{\"arr1\": [1,2,3], \"arr2\": [4,5,6]}", - - // test json arrays are valid - "[1, 2, 3, 4]", - "[{\"a\":1,\"b\":2}, {\"c\":3,\"d\":2}]", - - // test json scalars are valid - "\"abc\"", - "1234", - "12.34", - "true", - "false", - "null", - - // test empty string is valid - ""); - - validJsonStrings.stream() - .forEach( - str -> - assertEquals( - LITERAL_TRUE, - DSL.jsonValid(DSL.literal((ExprValueUtils.stringValue(str)))).valueOf(), - String.format("String %s must be valid json", str))); - } - @Test void json_extract_search_arrays() { - Expression jsonArray = DSL.literal(ExprValueUtils.stringValue("{\"a\":[1,2.3,\"abc\",true,null,{\"c\":1},[1,2,3]]}")); - List expectedExprValue = List.of( - new ExprIntegerValue(1), - new ExprFloatValue(2.3), - new ExprStringValue("abc"), - LITERAL_TRUE, - LITERAL_NULL, - ExprTupleValue.fromExprValueMap(Map.of("c", new ExprIntegerValue(1))), - new ExprCollectionValue(List.of(new ExprIntegerValue(1), new ExprIntegerValue(2), new ExprIntegerValue(3))) - ); + Expression jsonArray = + DSL.literal( + ExprValueUtils.stringValue("{\"a\":[1,2.3,\"abc\",true,null,{\"c\":1},[1,2,3]]}")); + List expectedExprValue = + List.of( + new ExprIntegerValue(1), + new ExprFloatValue(2.3), + new ExprStringValue("abc"), + LITERAL_TRUE, + LITERAL_NULL, + ExprTupleValue.fromExprValueMap(Map.of("c", new ExprIntegerValue(1))), + new ExprCollectionValue( + List.of( + new ExprIntegerValue(1), new ExprIntegerValue(2), new ExprIntegerValue(3)))); // extract specific index from JSON list - for (int i = 0 ; i < expectedExprValue.size() ; i++ ) { + for (int i = 0; i < expectedExprValue.size(); i++) { String path = String.format("$.a[%d]", i); Expression pathExpr = DSL.literal(ExprValueUtils.stringValue(path)); FunctionExpression expression = DSL.jsonExtract(jsonArray, pathExpr); @@ -224,9 +193,8 @@ void json_extract_search_arrays() { // extract * from JSON list Expression starPath = DSL.literal(ExprValueUtils.stringValue("$.a[*]")); - FunctionExpression starExpression = DSL.castInt(DSL.jsonExtract(jsonArray, starPath)); - assertEquals( - new ExprCollectionValue(expectedExprValue), starExpression.valueOf()); + FunctionExpression starExpression = DSL.jsonExtract(jsonArray, starPath); + assertEquals(new ExprCollectionValue(expectedExprValue), starExpression.valueOf()); } @Test @@ -243,7 +211,6 @@ void json_extract_returns_null() { "12.34", "true", "false", - "null", ""); jsonStrings.stream() @@ -252,18 +219,63 @@ void json_extract_returns_null() { assertEquals( LITERAL_NULL, DSL.jsonExtract( - DSL.literal((ExprValueUtils.stringValue(str))), - DSL.literal("$.a.path_not_found_key")).valueOf(), + DSL.literal((ExprValueUtils.stringValue(str))), + DSL.literal("$.a.path_not_found_key")) + .valueOf(), String.format("JSON string %s should return null", str))); } @Test - void json_returns_SemanticCheckException() { + void json_extract_throws_SemanticCheckException() { // invalid path assertThrows( - SemanticCheckException.class, () -> DSL.jsonExtract(DSL.literal("invalid"), DSL.literal("invalid")).valueOf()); + SemanticCheckException.class, + () -> + DSL.jsonExtract( + DSL.literal(new ExprStringValue("{\"a\":1}")), + DSL.literal(new ExprStringValue("$a"))) + .valueOf()); // invalid json - assertThrows(SemanticCheckException.class, () -> DSL.jsonExtract(DSL.literal("{\"invalid\":\"json\", \"string\"}"), DSL.literal("invalid")).valueOf()); + assertThrows( + SemanticCheckException.class, + () -> + DSL.jsonExtract( + DSL.literal(new ExprStringValue("{\"invalid\":\"json\", \"string\"}")), + DSL.literal(new ExprStringValue("$.a"))) + .valueOf()); + } + + @Test + void json_extract_throws_ExpressionEvaluationException() { + // null json + assertThrows( + ExpressionEvaluationException.class, + () -> + DSL.jsonExtract(DSL.literal(LITERAL_NULL), DSL.literal(new ExprStringValue("$.a"))) + .valueOf()); + + // null path + assertThrows( + ExpressionEvaluationException.class, + () -> + DSL.jsonExtract( + DSL.literal(new ExprStringValue("{\"a\":1}")), DSL.literal(LITERAL_NULL)) + .valueOf()); + + // missing json + assertThrows( + ExpressionEvaluationException.class, + () -> + DSL.jsonExtract(DSL.literal(LITERAL_MISSING), DSL.literal(new ExprStringValue("$.a"))) + .valueOf()); + + // missing path + assertThrows( + ExpressionEvaluationException.class, + () -> + DSL.jsonExtract( + DSL.literal(new ExprStringValue("{\"a\":1}")), DSL.literal(LITERAL_MISSING)) + .valueOf()); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 3b15ca99e9..afaa989a74 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -41,8 +41,7 @@ public void test_json_valid() throws IOException { rows("json array"), rows("json scalar string"), rows("json empty string"), - rows("json nested list") - ); + rows("json nested list")); } @Test @@ -76,9 +75,9 @@ public void test_cast_json() throws IOException { rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), rows("json empty string", null), - rows("json nested list", - new JSONObject(Map.of("a", "1", "b", List.of(Map.of("c", "2"), Map.of("c", "3"))))) - ); + rows( + "json nested list", + new JSONObject(Map.of("a", "1", "b", List.of(Map.of("c", "2"), Map.of("c", "3")))))); } @Test @@ -102,9 +101,9 @@ public void test_json() throws IOException { rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), rows("json empty string", null), - rows("json nested list", - new JSONObject(Map.of("a", "1", "b", List.of(Map.of("c", "2"), Map.of("c", "3"))))) - ); + rows( + "json nested list", + new JSONObject(Map.of("a", "1", "b", List.of(Map.of("c", "2"), Map.of("c", "3")))))); } @Test @@ -113,9 +112,11 @@ public void test_json_extract() throws IOException { result = executeQuery( String.format( - "source=%s | where json_valid(json_string) | eval extracted=json_extract(json_string, '$.b') | fields test_name, extracted", + "source=%s | where json_valid(json_string) | eval" + + " extracted=json_extract(json_string, '$.b') | fields test_name, extracted", TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string"), schema("extracted", null, "undefined")); + verifySchema( + result, schema("test_name", null, "string"), schema("extracted", null, "undefined")); verifyDataRows( result, rows("json nested object", new JSONObject(Map.of("c", "3"))), @@ -123,7 +124,6 @@ public void test_json_extract() throws IOException { rows("json array", null), rows("json scalar string", null), rows("json empty string", null), - rows("json nested list", new JSONArray(List.of(Map.of("c","2"), Map.of("c","3")))) - ); + rows("json nested list", new JSONArray(List.of(Map.of("c", "2"), Map.of("c", "3"))))); } } From 0f0b8d4a26fa47d2f9ae190d9050d9d87b2ab37c Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 23 Jan 2025 04:59:57 -0800 Subject: [PATCH 62/70] update doctest Signed-off-by: Kenrick Yap --- docs/user/ppl/functions/json.rst | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index ffef550c9c..fb6a398b05 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -51,15 +51,17 @@ Return type: BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY Example:: > source=json_test | where json_valid(json_string) | eval json=json(json_string) | fields test_name, json_string, json - fetched rows / total rows = 4/4 - +---------------------+------------------------------+---------------+ - | test_name | json_string | json | - |---------------------|------------------------------|---------------| - | json object | {"a":"1","b":"2"} | {a:"1",b:"2"} | - | json array | [1, 2, 3, 4] | [1,2,3,4] | - | json scalar string | "abc" | "abc" | - | json empty string | | null | - +---------------------+------------------------------+---------------+ + fetched rows / total rows = 6/6 + +---------------------+-------------------------------------+-----------------------------+ + | test_name | json_string | json | + |---------------------|-------------------------------------|-----------------------------| + | json nested object | {"a":"1","b":{"c":"2","d":"3"}} | {a:"1",b:{c:"2",d:"3"} | + | json nested list | {"a":"1","b":[{"c":"2"},{"c":"3"}]} | {a:"1",b:[{c:"2"},{c:"3"}]} | + | json object | {"a":"1","b":"2"} | {a:"1",b:"2"} | + | json array | [1, 2, 3, 4] | [1,2,3,4] | + | json scalar string | "abc" | "abc" | + | json empty string | | null | + +---------------------+-------------------------------------+-----------------------------+ JSON_EXTRACT ____________ From f030057b1999ba1ddc626d45c051e38d626369f6 Mon Sep 17 00:00:00 2001 From: 14yapkc1 Date: Mon, 27 Jan 2025 11:20:46 -0500 Subject: [PATCH 63/70] addessed comments Signed-off-by: 14yapkc1 --- .../org/opensearch/sql/utils/JsonUtils.java | 20 +++---------------- docs/user/ppl/functions/json.rst | 2 +- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index f3239ae25a..b3968911f2 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -7,11 +7,9 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.InvalidJsonException; import com.jayway.jsonpath.InvalidPathException; import com.jayway.jsonpath.JsonPath; -import com.jayway.jsonpath.Option; import com.jayway.jsonpath.PathNotFoundException; import java.util.LinkedHashMap; import java.util.LinkedList; @@ -60,25 +58,13 @@ public static ExprValue extractJson(ExprValue json, ExprValue path) { String jsonString = json.stringValue(); String jsonPath = path.stringValue(); - if (jsonString.equals("")) { + if (jsonString.isEmpty()) { return LITERAL_NULL; } try { - Configuration config = Configuration.builder().options(Option.AS_PATH_LIST).build(); - List resultPaths = JsonPath.using(config).parse(jsonString).read(jsonPath); - - List elements = new LinkedList<>(); - for (String resultPath : resultPaths) { - Object result = JsonPath.parse(jsonString).read(resultPath); - elements.add(ExprValueUtils.fromObjectValue(result)); - } - - if (elements.size() == 1) { - return elements.get(0); - } else { - return new ExprCollectionValue(elements); - } + Object results = JsonPath.parse(jsonString).read(jsonPath); + return ExprValueUtils.fromObjectValue(results); } catch (PathNotFoundException e) { return LITERAL_NULL; } catch (InvalidPathException e) { diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index fb6a398b05..864073dc01 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -69,7 +69,7 @@ ____________ Description >>>>>>>>>>> -Usage: `json_extract(doc, path [, path]...)` Extracts a json value or scalar from a json document based on the path(s) specified. +Usage: `json_extract(doc, path)` Extracts a json value or scalar from a json document based on the path(s) specified. Argument type: STRING, STRING From 2b08007279eaccb3d9d5a145338825d54013d675 Mon Sep 17 00:00:00 2001 From: 14yapkc1 Date: Tue, 28 Jan 2025 11:37:49 -0500 Subject: [PATCH 64/70] added addition edge cases for unit tests Signed-off-by: 14yapkc1 --- .../org/opensearch/sql/utils/JsonUtils.java | 7 ++++ .../expression/json/JsonFunctionsTest.java | 37 ++++++++++++++++++- docs/user/ppl/functions/json.rst | 4 +- 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index b3968911f2..c4c8a5073c 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -54,6 +54,13 @@ public static ExprValue castJson(ExprValue json) { return processJsonNode(jsonNode); } + /** + * Extract value of JSON string at given JSON path. + * + * @param json JSON string (e.g. "{\"hello\": \"world\"}"). + * @param path JSON path (e.g. "$.hello") + * @return ExprValue of value at given path of json string. + */ public static ExprValue extractJson(ExprValue json, ExprValue path) { String jsonString = json.stringValue(); String jsonPath = path.stringValue(); diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index bb73a27fe9..89acba7cbe 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -166,11 +166,36 @@ void json_returnsSemanticCheckException() { SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); } + @Test + void json_extract_search() { + Expression jsonArray = DSL.literal(ExprValueUtils.stringValue("{\"a\":1}")); + ExprValue expectedExprValue = new ExprIntegerValue(1); + Expression pathExpr = DSL.literal(ExprValueUtils.stringValue("$.a")); + FunctionExpression expression = DSL.jsonExtract(jsonArray, pathExpr); + assertEquals(expectedExprValue, expression.valueOf()); + } + + @Test + void json_extract_search_arrays_out_of_bound() { + Expression jsonArray = DSL.literal(ExprValueUtils.stringValue("{\"a\":[1,2,3}")); + + // index out of bounds + assertThrows( + SemanticCheckException.class, + () -> DSL.jsonExtract(jsonArray, DSL.literal(new ExprStringValue("$.a[3]"))).valueOf()); + + // negative index + assertThrows( + SemanticCheckException.class, + () -> DSL.jsonExtract(jsonArray, DSL.literal(new ExprStringValue("$.a[-1]"))).valueOf()); + } + @Test void json_extract_search_arrays() { Expression jsonArray = DSL.literal( - ExprValueUtils.stringValue("{\"a\":[1,2.3,\"abc\",true,null,{\"c\":1},[1,2,3]]}")); + ExprValueUtils.stringValue( + "{\"a\":[1,2.3,\"abc\",true,null,{\"c\":{\"d\":1}},[1,2,3]]}")); List expectedExprValue = List.of( new ExprIntegerValue(1), @@ -178,7 +203,8 @@ void json_extract_search_arrays() { new ExprStringValue("abc"), LITERAL_TRUE, LITERAL_NULL, - ExprTupleValue.fromExprValueMap(Map.of("c", new ExprIntegerValue(1))), + ExprTupleValue.fromExprValueMap( + Map.of("c", ExprTupleValue.fromExprValueMap(Map.of("d", new ExprIntegerValue(1))))), new ExprCollectionValue( List.of( new ExprIntegerValue(1), new ExprIntegerValue(2), new ExprIntegerValue(3)))); @@ -191,6 +217,13 @@ void json_extract_search_arrays() { assertEquals(expectedExprValue.get(i), expression.valueOf()); } + // extract nested object + ExprValue nestedExpected = + ExprTupleValue.fromExprValueMap(Map.of("d", new ExprIntegerValue(1))); + Expression nestedPath = DSL.literal(ExprValueUtils.stringValue("$.a[5].c")); + FunctionExpression nestedExpression = DSL.jsonExtract(jsonArray, nestedPath); + assertEquals(nestedExpected, nestedExpression.valueOf()); + // extract * from JSON list Expression starPath = DSL.literal(ExprValueUtils.stringValue("$.a[*]")); FunctionExpression starExpression = DSL.jsonExtract(jsonArray, starPath); diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 864073dc01..febe606ffe 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -69,7 +69,7 @@ ____________ Description >>>>>>>>>>> -Usage: `json_extract(doc, path)` Extracts a json value or scalar from a json document based on the path(s) specified. +Usage: `json_extract(doc, path)` Extracts a json value or scalar from a json document based on the path specified. Argument type: STRING, STRING @@ -77,6 +77,8 @@ Return type: BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY - Returns a JSON array for multiple paths or if the path leads to an array. - Return null if path is not valid. +- Throws error if `doc` or `path` is malformed. +- Throws error if `doc` or `path` is MISSING or NULL. Example:: From 6678be4bfbb0939ead3145dbe2a47a66cc4951d8 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Wed, 29 Jan 2025 11:11:36 -0500 Subject: [PATCH 65/70] addressed PR comments Signed-off-by: Kenrick Yap --- .../org/opensearch/sql/utils/JsonUtils.java | 10 ++- .../expression/json/JsonFunctionsTest.java | 88 +++++++------------ docs/user/ppl/functions/json.rst | 8 +- 3 files changed, 43 insertions(+), 63 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 5fa148d7c1..e1cbd3a03a 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -1,9 +1,5 @@ package org.opensearch.sql.utils; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; - import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -27,6 +23,8 @@ import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.SemanticCheckException; +import static org.opensearch.sql.data.model.ExprValueUtils.*; + @UtilityClass public class JsonUtils { /** @@ -86,6 +84,10 @@ public static ExprValue castJson(ExprValue json) { * @return ExprValue of value at given path of json string. */ public static ExprValue extractJson(ExprValue json, ExprValue path) { + if (json == LITERAL_NULL || json == LITERAL_MISSING) { + return json; + } + String jsonString = json.stringValue(); String jsonPath = path.stringValue(); diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 3e57355ed7..1aef2080ed 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -193,34 +193,18 @@ void json_returnsSemanticCheckException() { @Test void json_extract_search() { - Expression jsonArray = DSL.literal(ExprValueUtils.stringValue("{\"a\":1}")); - ExprValue expectedExprValue = new ExprIntegerValue(1); - Expression pathExpr = DSL.literal(ExprValueUtils.stringValue("$.a")); - FunctionExpression expression = DSL.jsonExtract(jsonArray, pathExpr); - assertEquals(expectedExprValue, expression.valueOf()); + ExprValue expected = new ExprIntegerValue(1); + execute_extract_json(expected, "{\"a\":1}", "$.a"); } @Test void json_extract_search_arrays_out_of_bound() { - Expression jsonArray = DSL.literal(ExprValueUtils.stringValue("{\"a\":[1,2,3}")); - - // index out of bounds - assertThrows( - SemanticCheckException.class, - () -> DSL.jsonExtract(jsonArray, DSL.literal(new ExprStringValue("$.a[3]"))).valueOf()); - - // negative index - assertThrows( - SemanticCheckException.class, - () -> DSL.jsonExtract(jsonArray, DSL.literal(new ExprStringValue("$.a[-1]"))).valueOf()); + execute_extract_json(LITERAL_NULL, "{\"a\":[1,2,3]}", "$.a[4]"); } @Test void json_extract_search_arrays() { - Expression jsonArray = - DSL.literal( - ExprValueUtils.stringValue( - "{\"a\":[1,2.3,\"abc\",true,null,{\"c\":{\"d\":1}},[1,2,3]]}")); + String jsonArray = "{\"a\":[1,2.3,\"abc\",true,null,{\"c\":{\"d\":1}},[1,2,3]]}"; List expectedExprValue = List.of( new ExprIntegerValue(1), @@ -237,22 +221,17 @@ void json_extract_search_arrays() { // extract specific index from JSON list for (int i = 0; i < expectedExprValue.size(); i++) { String path = String.format("$.a[%d]", i); - Expression pathExpr = DSL.literal(ExprValueUtils.stringValue(path)); - FunctionExpression expression = DSL.jsonExtract(jsonArray, pathExpr); - assertEquals(expectedExprValue.get(i), expression.valueOf()); + execute_extract_json(expectedExprValue.get(i), jsonArray, path); } // extract nested object ExprValue nestedExpected = ExprTupleValue.fromExprValueMap(Map.of("d", new ExprIntegerValue(1))); - Expression nestedPath = DSL.literal(ExprValueUtils.stringValue("$.a[5].c")); - FunctionExpression nestedExpression = DSL.jsonExtract(jsonArray, nestedPath); - assertEquals(nestedExpected, nestedExpression.valueOf()); + execute_extract_json(nestedExpected, jsonArray, "$.a[5].c"); // extract * from JSON list - Expression starPath = DSL.literal(ExprValueUtils.stringValue("$.a[*]")); - FunctionExpression starExpression = DSL.jsonExtract(jsonArray, starPath); - assertEquals(new ExprCollectionValue(expectedExprValue), starExpression.valueOf()); + ExprValue starExpected = new ExprCollectionValue(expectedExprValue); + execute_extract_json(starExpected, jsonArray, "$.a[*]"); } @Test @@ -271,48 +250,47 @@ void json_extract_returns_null() { "false", ""); - jsonStrings.stream() - .forEach( - str -> - assertEquals( - LITERAL_NULL, - DSL.jsonExtract( - DSL.literal((ExprValueUtils.stringValue(str))), - DSL.literal("$.a.path_not_found_key")) - .valueOf(), - String.format("JSON string %s should return null", str))); + jsonStrings.forEach( + str -> execute_extract_json(LITERAL_NULL, str, "$.a.path_not_found_key") + ); + + // null json + assertEquals(LITERAL_NULL, DSL.jsonExtract(DSL.literal(LITERAL_NULL), DSL.literal(new ExprStringValue("$.a"))).valueOf()); + + // missing json + assertEquals(LITERAL_MISSING, DSL.jsonExtract(DSL.literal(LITERAL_MISSING), DSL.literal(new ExprStringValue("$.a"))).valueOf()); } @Test void json_extract_throws_SemanticCheckException() { // invalid path - assertThrows( + SemanticCheckException invalidPathError = assertThrows( SemanticCheckException.class, () -> DSL.jsonExtract( DSL.literal(new ExprStringValue("{\"a\":1}")), DSL.literal(new ExprStringValue("$a"))) .valueOf()); + assertEquals( + "JSON path '\"$a\"' is not valid. Error details: Illegal character at position 1 expected '.' or '['", + invalidPathError.getMessage()); + // invalid json - assertThrows( + SemanticCheckException invalidJsonError = assertThrows( SemanticCheckException.class, () -> DSL.jsonExtract( DSL.literal(new ExprStringValue("{\"invalid\":\"json\", \"string\"}")), DSL.literal(new ExprStringValue("$.a"))) .valueOf()); + assertEquals( + "JSON string '\"{\"invalid\":\"json\", \"string\"}\"' is not valid. Error details: net.minidev.json.parser.ParseException: Unexpected character (}) at position 26.", + invalidJsonError.getMessage()); } @Test void json_extract_throws_ExpressionEvaluationException() { - // null json - assertThrows( - ExpressionEvaluationException.class, - () -> - DSL.jsonExtract(DSL.literal(LITERAL_NULL), DSL.literal(new ExprStringValue("$.a"))) - .valueOf()); - // null path assertThrows( ExpressionEvaluationException.class, @@ -321,13 +299,6 @@ void json_extract_throws_ExpressionEvaluationException() { DSL.literal(new ExprStringValue("{\"a\":1}")), DSL.literal(LITERAL_NULL)) .valueOf()); - // missing json - assertThrows( - ExpressionEvaluationException.class, - () -> - DSL.jsonExtract(DSL.literal(LITERAL_MISSING), DSL.literal(new ExprStringValue("$.a"))) - .valueOf()); - // missing path assertThrows( ExpressionEvaluationException.class, @@ -336,4 +307,11 @@ void json_extract_throws_ExpressionEvaluationException() { DSL.literal(new ExprStringValue("{\"a\":1}")), DSL.literal(LITERAL_MISSING)) .valueOf()); } + + private static void execute_extract_json(ExprValue expected, String json, String path) { + Expression pathExpr = DSL.literal(ExprValueUtils.stringValue(path)); + Expression jsonExpr = DSL.literal(ExprValueUtils.stringValue(json)); + ExprValue actual = DSL.jsonExtract(jsonExpr, pathExpr).valueOf(); + assertEquals(expected, actual); + } } diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index febe606ffe..c9260529bb 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -75,10 +75,10 @@ Argument type: STRING, STRING Return type: BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY -- Returns a JSON array for multiple paths or if the path leads to an array. -- Return null if path is not valid. -- Throws error if `doc` or `path` is malformed. -- Throws error if `doc` or `path` is MISSING or NULL. +- Returns a JSON array if path points to multiple results (e.g. $.a[*]) or if the path points to an array. +- Return null if path is not valid is MISSING or NULL. +- Throws SemanticCheckException if `doc` or `path` is malformed. +- Throws ExpressionEvaluationException if `path` is missing. Example:: From e57fa21a67db9a039b8cead077df6985fe447b4e Mon Sep 17 00:00:00 2001 From: Kenrick Yap <14yapkc1@gmail.com> Date: Thu, 30 Jan 2025 11:07:35 -0500 Subject: [PATCH 66/70] fix code coverage Signed-off-by: Kenrick Yap <14yapkc1@gmail.com> --- .../org/opensearch/sql/expression/DSL.java | 4 -- .../org/opensearch/sql/utils/JsonUtils.java | 4 +- .../expression/json/JsonFunctionsTest.java | 55 +++++++++++-------- docs/user/ppl/functions/json.rst | 4 +- 4 files changed, 35 insertions(+), 32 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index 98afc9cbb4..58bc9390df 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -985,10 +985,6 @@ public static FunctionExpression utc_timestamp( return compile(functionProperties, BuiltinFunctionName.UTC_TIMESTAMP, args); } - public static FunctionExpression json_function(Expression value) { - return compile(FunctionProperties.None, BuiltinFunctionName.JSON, value); - } - @SuppressWarnings("unchecked") private static T compile( FunctionProperties functionProperties, BuiltinFunctionName bfn, Expression... args) { diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index e1cbd3a03a..b898026a08 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -1,5 +1,7 @@ package org.opensearch.sql.utils; +import static org.opensearch.sql.data.model.ExprValueUtils.*; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -23,8 +25,6 @@ import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.SemanticCheckException; -import static org.opensearch.sql.data.model.ExprValueUtils.*; - @UtilityClass public class JsonUtils { /** diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 1aef2080ed..beac451a6e 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -250,43 +250,50 @@ void json_extract_returns_null() { "false", ""); - jsonStrings.forEach( - str -> execute_extract_json(LITERAL_NULL, str, "$.a.path_not_found_key") - ); + jsonStrings.forEach(str -> execute_extract_json(LITERAL_NULL, str, "$.a.path_not_found_key")); // null json - assertEquals(LITERAL_NULL, DSL.jsonExtract(DSL.literal(LITERAL_NULL), DSL.literal(new ExprStringValue("$.a"))).valueOf()); + assertEquals( + LITERAL_NULL, + DSL.jsonExtract(DSL.literal(LITERAL_NULL), DSL.literal(new ExprStringValue("$.a"))) + .valueOf()); // missing json - assertEquals(LITERAL_MISSING, DSL.jsonExtract(DSL.literal(LITERAL_MISSING), DSL.literal(new ExprStringValue("$.a"))).valueOf()); + assertEquals( + LITERAL_MISSING, + DSL.jsonExtract(DSL.literal(LITERAL_MISSING), DSL.literal(new ExprStringValue("$.a"))) + .valueOf()); } @Test void json_extract_throws_SemanticCheckException() { // invalid path - SemanticCheckException invalidPathError = assertThrows( - SemanticCheckException.class, - () -> - DSL.jsonExtract( - DSL.literal(new ExprStringValue("{\"a\":1}")), - DSL.literal(new ExprStringValue("$a"))) - .valueOf()); + SemanticCheckException invalidPathError = + assertThrows( + SemanticCheckException.class, + () -> + DSL.jsonExtract( + DSL.literal(new ExprStringValue("{\"a\":1}")), + DSL.literal(new ExprStringValue("$a"))) + .valueOf()); assertEquals( - "JSON path '\"$a\"' is not valid. Error details: Illegal character at position 1 expected '.' or '['", - invalidPathError.getMessage()); - + "JSON path '\"$a\"' is not valid. Error details: Illegal character at position 1 expected" + + " '.' or '['", + invalidPathError.getMessage()); // invalid json - SemanticCheckException invalidJsonError = assertThrows( - SemanticCheckException.class, - () -> - DSL.jsonExtract( - DSL.literal(new ExprStringValue("{\"invalid\":\"json\", \"string\"}")), - DSL.literal(new ExprStringValue("$.a"))) - .valueOf()); + SemanticCheckException invalidJsonError = + assertThrows( + SemanticCheckException.class, + () -> + DSL.jsonExtract( + DSL.literal(new ExprStringValue("{\"invalid\":\"json\", \"string\"}")), + DSL.literal(new ExprStringValue("$.a"))) + .valueOf()); assertEquals( - "JSON string '\"{\"invalid\":\"json\", \"string\"}\"' is not valid. Error details: net.minidev.json.parser.ParseException: Unexpected character (}) at position 26.", - invalidJsonError.getMessage()); + "JSON string '\"{\"invalid\":\"json\", \"string\"}\"' is not valid. Error details:" + + " net.minidev.json.parser.ParseException: Unexpected character (}) at position 26.", + invalidJsonError.getMessage()); } @Test diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index c9260529bb..b75c241a53 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -69,11 +69,11 @@ ____________ Description >>>>>>>>>>> -Usage: `json_extract(doc, path)` Extracts a json value or scalar from a json document based on the path specified. +Usage: `json_extract(doc, path)` Extracts a JSON value from a json document based on the path specified. Argument type: STRING, STRING -Return type: BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY +Return type: STRING/BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY - Returns a JSON array if path points to multiple results (e.g. $.a[*]) or if the path points to an array. - Return null if path is not valid is MISSING or NULL. From 112be65095daef4fd4100a0cb173ea190922f7a6 Mon Sep 17 00:00:00 2001 From: kenrickyap <121634635+kenrickyap@users.noreply.github.com> Date: Thu, 30 Jan 2025 12:34:17 -0500 Subject: [PATCH 67/70] Update core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java Co-authored-by: Taylor Curran Signed-off-by: kenrickyap <121634635+kenrickyap@users.noreply.github.com> --- .../org/opensearch/sql/expression/json/JsonFunctionsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index beac451a6e..9e96ff5554 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -205,7 +205,7 @@ void json_extract_search_arrays_out_of_bound() { @Test void json_extract_search_arrays() { String jsonArray = "{\"a\":[1,2.3,\"abc\",true,null,{\"c\":{\"d\":1}},[1,2,3]]}"; - List expectedExprValue = + List expectedExprValues = List.of( new ExprIntegerValue(1), new ExprFloatValue(2.3), From 306ac97d2831743bd03467e204ae52d772165173 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 30 Jan 2025 12:51:48 -0500 Subject: [PATCH 68/70] address comments Signed-off-by: Kenrick Yap --- .../sql/expression/json/JsonFunctionsTest.java | 10 ++++++---- docs/user/ppl/functions/json.rst | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 9e96ff5554..ba7d03ff16 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -290,10 +290,12 @@ void json_extract_throws_SemanticCheckException() { DSL.literal(new ExprStringValue("{\"invalid\":\"json\", \"string\"}")), DSL.literal(new ExprStringValue("$.a"))) .valueOf()); - assertEquals( - "JSON string '\"{\"invalid\":\"json\", \"string\"}\"' is not valid. Error details:" - + " net.minidev.json.parser.ParseException: Unexpected character (}) at position 26.", - invalidJsonError.getMessage()); + assertTrue( + invalidJsonError + .getMessage() + .startsWith( + "JSON string '\"{\"invalid\":\"json\", \"string\"}\"' is not valid. Error" + + " details:")); } @Test diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index b75c241a53..3e8c21a9e4 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -75,8 +75,8 @@ Argument type: STRING, STRING Return type: STRING/BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY -- Returns a JSON array if path points to multiple results (e.g. $.a[*]) or if the path points to an array. -- Return null if path is not valid is MISSING or NULL. +- Returns a JSON array if `path` points to multiple results (e.g. $.a[*]) or if the `path` points to an array. +- Return null if `path` is not valid, or if JSON `doc` is MISSING or NULL. - Throws SemanticCheckException if `doc` or `path` is malformed. - Throws ExpressionEvaluationException if `path` is missing. From 75e9cc3426a1ae8e15f74ca85e86871906d51f17 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 30 Jan 2025 14:15:23 -0500 Subject: [PATCH 69/70] fix build error Signed-off-by: Kenrick Yap --- .../opensearch/sql/expression/json/JsonFunctionsTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index ba7d03ff16..1f37e1f0db 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -219,9 +219,9 @@ void json_extract_search_arrays() { new ExprIntegerValue(1), new ExprIntegerValue(2), new ExprIntegerValue(3)))); // extract specific index from JSON list - for (int i = 0; i < expectedExprValue.size(); i++) { + for (int i = 0; i < expectedExprValues.size(); i++) { String path = String.format("$.a[%d]", i); - execute_extract_json(expectedExprValue.get(i), jsonArray, path); + execute_extract_json(expectedExprValues.get(i), jsonArray, path); } // extract nested object @@ -230,7 +230,7 @@ void json_extract_search_arrays() { execute_extract_json(nestedExpected, jsonArray, "$.a[5].c"); // extract * from JSON list - ExprValue starExpected = new ExprCollectionValue(expectedExprValue); + ExprValue starExpected = new ExprCollectionValue(expectedExprValues); execute_extract_json(starExpected, jsonArray, "$.a[*]"); } From 80f44e28e9f4e395d4c108f760264d91a306fc41 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Fri, 31 Jan 2025 10:19:30 -0500 Subject: [PATCH 70/70] add header Signed-off-by: Kenrick Yap --- core/src/main/java/org/opensearch/sql/utils/JsonUtils.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index b898026a08..433abe7673 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.utils; import static org.opensearch.sql.data.model.ExprValueUtils.*;