|
| 1 | +/* |
| 2 | + * Copyright OpenSearch Contributors |
| 3 | + * SPDX-License-Identifier: Apache-2.0 |
| 4 | + */ |
| 5 | + |
| 6 | +package org.opensearch.sql.calcite.standalone; |
| 7 | + |
| 8 | +import static org.opensearch.sql.legacy.TestUtils.createIndexByRestClient; |
| 9 | +import static org.opensearch.sql.legacy.TestUtils.isIndexExist; |
| 10 | +import static org.opensearch.sql.util.MatcherUtils.rows; |
| 11 | +import static org.opensearch.sql.util.MatcherUtils.schema; |
| 12 | +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; |
| 13 | +import static org.opensearch.sql.util.MatcherUtils.verifySchema; |
| 14 | + |
| 15 | +import java.io.IOException; |
| 16 | +import org.json.JSONObject; |
| 17 | +import org.junit.jupiter.api.Test; |
| 18 | +import org.opensearch.client.Request; |
| 19 | + |
| 20 | +public class CalciteDynamicFieldsCommandIT extends CalcitePPLPermissiveIntegTestCase { |
| 21 | + |
| 22 | + private static final String TEST_INDEX_DYNAMIC = "test_dynamic_fields"; |
| 23 | + |
| 24 | + @Override |
| 25 | + public void init() throws IOException { |
| 26 | + super.init(); |
| 27 | + createTestIndexWithUnmappedFields(); |
| 28 | + enableCalcite(); |
| 29 | + } |
| 30 | + |
| 31 | + @Test |
| 32 | + public void testBasicProjection() throws IOException { |
| 33 | + String query = |
| 34 | + source(TEST_INDEX_DYNAMIC, "fields firstname, lastname, department, salary | head 1"); |
| 35 | + assertExplainYaml( |
| 36 | + query, |
| 37 | + "calcite:\n" |
| 38 | + + " logical: |\n" |
| 39 | + + " LogicalSystemLimit(fetch=[200], type=[QUERY_SIZE_LIMIT])\n" |
| 40 | + + " LogicalSort(fetch=[1])\n" |
| 41 | + + " LogicalProject(firstname=[$0], lastname=[$2], department=[ITEM($9," |
| 42 | + + " 'department')], salary=[ITEM($9, 'salary')])\n" |
| 43 | + + " CalciteLogicalIndexScan(table=[[OpenSearch, test_dynamic_fields]])\n" |
| 44 | + + " physical: |\n" |
| 45 | + + " EnumerableLimit(fetch=[200])\n" |
| 46 | + + " EnumerableCalc(expr#0..9=[{inputs}], expr#10=['department']," |
| 47 | + + " expr#11=[ITEM($t9, $t10)], expr#12=['salary'], expr#13=[ITEM($t9, $t12)]," |
| 48 | + + " firstname=[$t0], lastname=[$t2], department=[$t11], salary=[$t13])\n" |
| 49 | + + " EnumerableLimit(fetch=[1])\n" |
| 50 | + + " CalciteEnumerableIndexScan(table=[[OpenSearch, test_dynamic_fields]])\n"); |
| 51 | + |
| 52 | + JSONObject result = executeQuery(query); |
| 53 | + verifySchema( |
| 54 | + result, |
| 55 | + schema("firstname", "string"), |
| 56 | + schema("lastname", "string"), |
| 57 | + schema("department", "string"), |
| 58 | + schema("salary", "int")); |
| 59 | + verifyDataRows(result, rows("John", "Doe", "Engineering", 75000)); |
| 60 | + } |
| 61 | + |
| 62 | + @Test |
| 63 | + public void testEval() throws IOException { |
| 64 | + String query = |
| 65 | + source( |
| 66 | + TEST_INDEX_DYNAMIC, |
| 67 | + "eval salary = cast(salary as int) * 2 | fields firstname," |
| 68 | + + " lastname, salary | head 1"); |
| 69 | + |
| 70 | + assertExplainYaml( |
| 71 | + query, |
| 72 | + "calcite:\n" |
| 73 | + + " logical: |\n" |
| 74 | + + " LogicalSystemLimit(fetch=[200], type=[QUERY_SIZE_LIMIT])\n" |
| 75 | + + " LogicalSort(fetch=[1])\n" |
| 76 | + + " LogicalProject(firstname=[$0], lastname=[$2], salary=[*(SAFE_CAST(ITEM($9," |
| 77 | + + " 'salary')), 2)])\n" |
| 78 | + + " CalciteLogicalIndexScan(table=[[OpenSearch, test_dynamic_fields]])\n" |
| 79 | + + " physical: |\n" |
| 80 | + + " EnumerableLimit(fetch=[200])\n" |
| 81 | + + " EnumerableCalc(expr#0..9=[{inputs}], expr#10=['salary'], expr#11=[ITEM($t9," |
| 82 | + + " $t10)], expr#12=[SAFE_CAST($t11)], expr#13=[2], expr#14=[*($t12, $t13)]," |
| 83 | + + " firstname=[$t0], lastname=[$t2], salary=[$t14])\n" |
| 84 | + + " EnumerableLimit(fetch=[1])\n" |
| 85 | + + " CalciteEnumerableIndexScan(table=[[OpenSearch, test_dynamic_fields]])\n" |
| 86 | + + ""); |
| 87 | + |
| 88 | + JSONObject result = executeQuery(query); |
| 89 | + |
| 90 | + verifySchema( |
| 91 | + result, |
| 92 | + schema("firstname", "string"), |
| 93 | + schema("lastname", "string"), |
| 94 | + schema("salary", "int")); |
| 95 | + } |
| 96 | + |
| 97 | + private void createTestIndexWithUnmappedFields() throws IOException { |
| 98 | + if (isIndexExist(client(), TEST_INDEX_DYNAMIC)) { |
| 99 | + return; |
| 100 | + } |
| 101 | + |
| 102 | + String mapping = |
| 103 | + "{" |
| 104 | + + "\"mappings\": {" |
| 105 | + // Disable dynamic mapping - extra fields won't be indexed but will be stored |
| 106 | + + " \"dynamic\": false," |
| 107 | + + " \"properties\": {" |
| 108 | + + " \"firstname\": {\"type\": \"text\"}," |
| 109 | + + " \"lastname\": {\"type\": \"text\"}," |
| 110 | + + " \"accountnumber\": {\"type\": \"long\"}" |
| 111 | + + " }" |
| 112 | + + "}" |
| 113 | + + "}"; |
| 114 | + |
| 115 | + createIndexByRestClient(client(), TEST_INDEX_DYNAMIC, mapping); |
| 116 | + |
| 117 | + String bulkData = |
| 118 | + "{\"index\":{\"_id\":\"1\"}}\n" |
| 119 | + + "{\"firstname\":\"John\",\"lastname\":\"Doe\",\"account_number\":1,\"city\":\"NYC\",\"department\":\"Engineering\",\"salary\":75000}\n" |
| 120 | + + "{\"index\":{\"_id\":\"2\"}}\n" |
| 121 | + + "{\"firstname\":\"Jane\",\"lastname\":\"Smith\",\"account_number\":2,\"city\":\"LA\",\"department\":\"Marketing\",\"salary\":65000}\n" |
| 122 | + + "{\"index\":{\"_id\":\"3\"}}\n" |
| 123 | + + "{\"firstname\":\"Bob\",\"lastname\":\"Johnson\",\"account_number\":3,\"city\":\"Chicago\",\"department\":\"Sales\",\"salary\":55000}\n" |
| 124 | + + "{\"index\":{\"_id\":\"4\"}}\n" |
| 125 | + + "{\"firstname\":\"Alice\",\"lastname\":\"Brown\",\"account_number\":4,\"city\":\"Seattle\",\"department\":\"Engineering\",\"salary\":80000}\n" |
| 126 | + + "{\"index\":{\"_id\":\"5\"}}\n" |
| 127 | + + "{\"firstname\":\"Charlie\",\"lastname\":\"Wilson\",\"account_number\":5,\"city\":\"Boston\",\"department\":\"HR\",\"salary\":60000}\n"; |
| 128 | + |
| 129 | + Request request = new Request("POST", "/" + TEST_INDEX_DYNAMIC + "/_bulk?refresh=true"); |
| 130 | + request.setJsonEntity(bulkData); |
| 131 | + client().performRequest(request); |
| 132 | + } |
| 133 | +} |
0 commit comments