Skip to content

Commit 254cdbf

Browse files
penghuonormanj-bitquill
authored and
noCharger
committed
Array values are preserved (#1300) (#3095) (#3120)
Signed-off-by: Norman Jordan <norman.jordan@improving.com> (cherry picked from commit e109417) Co-authored-by: normanj-bitquill <78755797+normanj-bitquill@users.noreply.github.com> Signed-off-by: Louis Chu <clingzhi@amazon.com>
1 parent 0d2a4aa commit 254cdbf

File tree

19 files changed

+285
-63
lines changed

19 files changed

+285
-63
lines changed

‎common/src/main/java/org/opensearch/sql/common/setting/Settings.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@ public enum Key {
2727
/** PPL Settings. */
2828
PPL_ENABLED("plugins.ppl.enabled"),
2929

30+
/** Query Settings. */
31+
FIELD_TYPE_TOLERANCE("plugins.query.field_type_tolerance"),
32+
3033
/** Common Settings for SQL and PPL. */
3134
QUERY_MEMORY_LIMIT("plugins.query.memory_limit"),
3235
QUERY_SIZE_LIMIT("plugins.query.size_limit"),

‎docs/user/beyond/partiql.rst‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -202,11 +202,11 @@ Selecting top level for object fields, object fields of array value and nested f
202202

203203
os> SELECT city, accounts, projects FROM people;
204204
fetched rows / total rows = 1/1
205-
+-----------------------------------------------------+------------+----------------------------------------------------------------------------------------------------------------+
206-
| city | accounts | projects |
207-
|-----------------------------------------------------+------------+----------------------------------------------------------------------------------------------------------------|
208-
| {'name': 'Seattle', 'location': {'latitude': 10.5}} | {'id': 1} | [{'name': 'AWS Redshift Spectrum querying'},{'name': 'AWS Redshift security'},{'name': 'AWS Aurora security'}] |
209-
+-----------------------------------------------------+------------+----------------------------------------------------------------------------------------------------------------+
205+
+-----------------------------------------------------+-----------------------+----------------------------------------------------------------------------------------------------------------+
206+
| city | accounts | projects |
207+
|-----------------------------------------------------+-----------------------+----------------------------------------------------------------------------------------------------------------|
208+
| {'name': 'Seattle', 'location': {'latitude': 10.5}} | [{'id': 1},{'id': 2}] | [{'name': 'AWS Redshift Spectrum querying'},{'name': 'AWS Redshift security'},{'name': 'AWS Aurora security'}] |
209+
+-----------------------------------------------------+-----------------------+----------------------------------------------------------------------------------------------------------------+
210210

211211
Example 2: Selecting Deeper Levels
212212
----------------------------------

‎docs/user/ppl/general/datatypes.rst‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -412,8 +412,8 @@ Select deeper level for object fields of array value which returns the first ele
412412

413413
os> source = people | fields accounts, accounts.id;
414414
fetched rows / total rows = 1/1
415-
+------------+---------------+
416-
| accounts | accounts.id |
417-
|------------+---------------|
418-
| {'id': 1} | 1 |
419-
+------------+---------------+
415+
+-----------------------+-------------+
416+
| accounts | accounts.id |
417+
|-----------------------+-------------|
418+
| [{'id': 1},{'id': 2}] | 1 |
419+
+-----------------------+-------------+

‎integ-test/src/test/java/org/opensearch/sql/legacy/ObjectFieldSelectIT.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import org.json.JSONArray;
1515
import org.json.JSONObject;
1616
import org.junit.Test;
17+
import org.opensearch.sql.common.setting.Settings;
1718
import org.opensearch.sql.legacy.utils.StringUtils;
1819

1920
/**
@@ -79,9 +80,20 @@ public void testSelectNestedFieldItself() {
7980
@Test
8081
public void testSelectObjectFieldOfArrayValuesItself() {
8182
JSONObject response = new JSONObject(query("SELECT accounts FROM %s"));
83+
verifyDataRows(response, rows(new JSONArray("[{\"id\":1},{\"id\":2}]")));
84+
}
8285

83-
// Only the first element of the list of is returned.
84-
verifyDataRows(response, rows(new JSONObject("{\"id\": 1}")));
86+
@Test
87+
public void testSelectObjectFieldOfArrayValuesItselfNoFieldTypeTolerance() throws Exception {
88+
updateClusterSettings(
89+
new ClusterSetting(PERSISTENT, Settings.Key.FIELD_TYPE_TOLERANCE.getKeyValue(), "false"));
90+
try {
91+
JSONObject response = new JSONObject(query("SELECT accounts FROM %s"));
92+
verifyDataRows(response, rows(new JSONObject("{\"id\":1}")));
93+
} finally {
94+
updateClusterSettings(
95+
new ClusterSetting(PERSISTENT, Settings.Key.FIELD_TYPE_TOLERANCE.getKeyValue(), "true"));
96+
}
8597
}
8698

8799
@Test

‎integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -149,8 +149,11 @@ public void onFailure(Exception e) {
149149

150150
private Settings defaultSettings() {
151151
return new Settings() {
152-
private final Map<Key, Integer> defaultSettings =
153-
new ImmutableMap.Builder<Key, Integer>().put(Key.QUERY_SIZE_LIMIT, 200).build();
152+
private final Map<Key, Object> defaultSettings =
153+
new ImmutableMap.Builder<Key, Object>()
154+
.put(Key.QUERY_SIZE_LIMIT, 200)
155+
.put(Key.FIELD_TYPE_TOLERANCE, true)
156+
.build();
154157

155158
@Override
156159
public <T> T getSettingValue(Key key) {

‎integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -423,7 +423,7 @@ public void test_nested_in_where_as_predicate_expression_with_multiple_condition
423423
+ " nested(message.dayOfWeek) >= 4";
424424
JSONObject result = executeJdbcRequest(query);
425425
assertEquals(2, result.getInt("total"));
426-
verifyDataRows(result, rows("c", "ab", 4), rows("zz", "aa", 6));
426+
verifyDataRows(result, rows("c", "ab", 4), rows("zz", new JSONArray(List.of("aa", "bb")), 6));
427427
}
428428

429429
@Test

‎integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,7 @@ private Settings defaultSettings() {
166166
new ImmutableMap.Builder<Key, Object>()
167167
.put(Key.QUERY_SIZE_LIMIT, 200)
168168
.put(Key.SQL_CURSOR_KEEP_ALIVE, TimeValue.timeValueMinutes(1))
169+
.put(Key.FIELD_TYPE_TOLERANCE, true)
169170
.build();
170171

171172
@Override

‎opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ public class OpenSearchExprValueFactory {
7575
/** The Mapping of Field and ExprType. */
7676
private final Map<String, OpenSearchDataType> typeMapping;
7777

78+
/** Whether to support nested value types (such as arrays) */
79+
private final boolean fieldTypeTolerance;
80+
7881
/**
7982
* Extend existing mapping by new data without overwrite. Called from aggregation only {@see
8083
* AggregationQueryBuilder#buildTypeMapping}.
@@ -143,8 +146,10 @@ public void extendTypeMapping(Map<String, OpenSearchDataType> typeMapping) {
143146
.build();
144147

145148
/** Constructor of OpenSearchExprValueFactory. */
146-
public OpenSearchExprValueFactory(Map<String, OpenSearchDataType> typeMapping) {
149+
public OpenSearchExprValueFactory(
150+
Map<String, OpenSearchDataType> typeMapping, boolean fieldTypeTolerance) {
147151
this.typeMapping = OpenSearchDataType.traverseAndFlatten(typeMapping);
152+
this.fieldTypeTolerance = fieldTypeTolerance;
148153
}
149154

150155
/**
@@ -160,7 +165,7 @@ public ExprValue construct(String jsonString, boolean supportArrays) {
160165
new OpenSearchJsonContent(OBJECT_MAPPER.readTree(jsonString)),
161166
TOP_PATH,
162167
Optional.of(STRUCT),
163-
supportArrays);
168+
fieldTypeTolerance || supportArrays);
164169
} catch (JsonProcessingException e) {
165170
throw new IllegalStateException(String.format("invalid json: %s.", jsonString), e);
166171
}

‎opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,8 @@ public OpenSearchScrollRequest(StreamInput in, OpenSearchStorageEngine engine)
178178
includes = in.readStringList();
179179
indexName = new IndexName(in);
180180
OpenSearchIndex index = (OpenSearchIndex) engine.getTable(null, indexName.toString());
181-
exprValueFactory = new OpenSearchExprValueFactory(index.getFieldOpenSearchTypes());
181+
exprValueFactory =
182+
new OpenSearchExprValueFactory(
183+
index.getFieldOpenSearchTypes(), index.isFieldTypeTolerance());
182184
}
183185
}

‎opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,13 @@ public class OpenSearchSettings extends Settings {
222222
Setting.Property.NodeScope,
223223
Setting.Property.Dynamic);
224224

225+
public static final Setting<?> FIELD_TYPE_TOLERANCE_SETTING =
226+
Setting.boolSetting(
227+
Key.FIELD_TYPE_TOLERANCE.getKeyValue(),
228+
true,
229+
Setting.Property.NodeScope,
230+
Setting.Property.Dynamic);
231+
225232
/** Construct OpenSearchSetting. The OpenSearchSetting must be singleton. */
226233
@SuppressWarnings("unchecked")
227234
public OpenSearchSettings(ClusterSettings clusterSettings) {
@@ -359,13 +366,19 @@ public OpenSearchSettings(ClusterSettings clusterSettings) {
359366
clusterSettings,
360367
Key.SESSION_INACTIVITY_TIMEOUT_MILLIS,
361368
SESSION_INACTIVITY_TIMEOUT_MILLIS_SETTING,
362-
new Updater((Key.SESSION_INACTIVITY_TIMEOUT_MILLIS)));
369+
new Updater(Key.SESSION_INACTIVITY_TIMEOUT_MILLIS));
363370
register(
364371
settingBuilder,
365372
clusterSettings,
366373
Key.STREAMING_JOB_HOUSEKEEPER_INTERVAL,
367374
STREAMING_JOB_HOUSEKEEPER_INTERVAL_SETTING,
368-
new Updater((Key.STREAMING_JOB_HOUSEKEEPER_INTERVAL)));
375+
new Updater(Key.STREAMING_JOB_HOUSEKEEPER_INTERVAL));
376+
register(
377+
settingBuilder,
378+
clusterSettings,
379+
Key.FIELD_TYPE_TOLERANCE,
380+
FIELD_TYPE_TOLERANCE_SETTING,
381+
new Updater(Key.FIELD_TYPE_TOLERANCE));
369382
defaultSettings = settingBuilder.build();
370383
}
371384

@@ -441,6 +454,7 @@ public static List<Setting<?>> pluginSettings() {
441454
.add(DATASOURCES_LIMIT_SETTING)
442455
.add(SESSION_INACTIVITY_TIMEOUT_MILLIS_SETTING)
443456
.add(STREAMING_JOB_HOUSEKEEPER_INTERVAL_SETTING)
457+
.add(FIELD_TYPE_TOLERANCE_SETTING)
444458
.build();
445459
}
446460

0 commit comments

Comments
 (0)