Skip to content

Commit ccd3060

Browse files
CaideyipiCaideyipi
authored
Fixed the reserved word introduced by tree view in tree model SQL & Enhanced the tree view IT & Handled the cases in field / measurement names & Temporarily banned source measurements with the same name (apache#15475)
1 parent ea6a2d0 commit ccd3060

File tree

21 files changed

+248
-97
lines changed

21 files changed

+248
-97
lines changed

‎integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java‎

Lines changed: 87 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
import java.util.Collections;
5252
import java.util.HashSet;
5353
import java.util.List;
54+
import java.util.Set;
5455

5556
import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.describeTableColumnHeaders;
5657
import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.describeTableDetailsColumnHeaders;
@@ -761,33 +762,108 @@ public void testTreeViewTable() throws Exception {
761762
try (final Connection connection = EnvFactory.getEnv().getConnection();
762763
final Statement statement = connection.createStatement()) {
763764
statement.execute("create database root.a.b");
764-
statement.execute("create timeSeries root.a.b.c.s1 int32");
765+
statement.execute("create timeSeries root.a.b.c.S1 int32");
765766
statement.execute("create timeSeries root.a.b.c.s2 string");
766-
statement.execute("create timeSeries root.a.b.s1 int32");
767-
statement.execute("create timeSeries root.a.b.d.s1 boolean");
768-
statement.execute("create timeSeries root.a.b.c.f.g.h.s1 int32");
767+
statement.execute("create timeSeries root.a.b.S1 int32");
768+
} catch (SQLException e) {
769+
fail(e.getMessage());
770+
}
771+
772+
try (final Connection connection =
773+
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
774+
final Statement statement = connection.createStatement()) {
775+
statement.execute("create database tree_view_db");
776+
statement.execute("use tree_view_db");
777+
statement.execute("create table view tree_table (tag1 tag, tag2 tag) as root.a.**");
778+
statement.execute("drop view tree_table");
779+
}
780+
781+
try (final Connection connection = EnvFactory.getEnv().getConnection();
782+
final Statement statement = connection.createStatement()) {
783+
statement.execute("create timeSeries root.a.b.d.s1 int32");
784+
} catch (SQLException e) {
785+
fail(e.getMessage());
786+
}
787+
788+
try (final Connection connection =
789+
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
790+
final Statement statement = connection.createStatement()) {
791+
statement.execute("use tree_view_db");
792+
793+
try {
794+
statement.execute("create table view tree_table (tag1 tag, tag2 tag) as root.a.**");
795+
fail();
796+
} catch (final SQLException e) {
797+
final Set<String> result =
798+
new HashSet<>(
799+
Arrays.asList(
800+
"617: The measurements s1 and S1 share the same lower case when auto detecting type, please check",
801+
"617: The measurements S1 and s1 share the same lower case when auto detecting type, please check"));
802+
assertTrue(result.contains(e.getMessage()));
803+
}
804+
}
805+
806+
try (final Connection connection = EnvFactory.getEnv().getConnection();
807+
final Statement statement = connection.createStatement()) {
808+
statement.execute("drop timeSeries root.a.b.d.s1");
809+
statement.execute("create timeSeries root.a.b.d.S1 boolean");
810+
statement.execute("create timeSeries root.a.b.c.f.g.h.S1 int32");
769811

770812
// Put schema cache
771-
statement.execute("select s1, s2 from root.a.b.c");
813+
statement.execute("select S1, s2 from root.a.b.c");
772814
} catch (SQLException e) {
773815
fail(e.getMessage());
774816
}
775817

776818
try (final Connection connection =
777819
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
778820
final Statement statement = connection.createStatement()) {
779-
statement.execute("create database tree_view_db");
780821
statement.execute("use tree_view_db");
822+
781823
try {
782824
statement.execute("create table view tree_table (tag1 tag, tag2 tag) as root.a.**");
783825
fail();
784826
} catch (final SQLException e) {
785827
assertEquals(
786-
"614: Multiple types encountered when auto detecting type of measurement 's1', please check",
828+
"614: Multiple types encountered when auto detecting type of measurement 'S1', please check",
787829
e.getMessage());
788830
}
831+
832+
try {
833+
statement.execute(
834+
"create table view tree_table (tag1 tag, tag2 tag, S1 field) as root.a.**");
835+
fail();
836+
} catch (final SQLException e) {
837+
assertEquals(
838+
"614: Multiple types encountered when auto detecting type of measurement 'S1', please check",
839+
e.getMessage());
840+
}
841+
}
842+
843+
try (final Connection connection = EnvFactory.getEnv().getConnection();
844+
final Statement statement = connection.createStatement()) {
845+
statement.execute("create timeSeries root.a.b.e.s1 int32");
846+
} catch (SQLException e) {
847+
fail(e.getMessage());
848+
}
849+
850+
try (final Connection connection =
851+
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
852+
final Statement statement = connection.createStatement()) {
853+
statement.execute("use tree_view_db");
854+
855+
// Temporary
856+
try {
857+
statement.execute(
858+
"create or replace table view tree_table (tag1 tag, tag2 tag, S1 int32 field, s3 boolean from S1) as root.a.**");
859+
fail();
860+
} catch (final SQLException e) {
861+
assertEquals(
862+
"701: The duplicated source measurement S1 is unsupported yet.", e.getMessage());
863+
}
864+
789865
statement.execute(
790-
"create or replace table view tree_table (tag1 tag, tag2 tag, s1 int32 field, s3 from s2) as root.a.**");
866+
"create or replace table view tree_table (tag1 tag, tag2 tag, S1 int32 field, s3 from s2) as root.a.**");
791867
statement.execute("alter view tree_table rename to view_table");
792868
statement.execute("alter view view_table rename column s1 to s11");
793869
statement.execute("alter view view_table set properties ttl=100");
@@ -808,21 +884,21 @@ public void testTreeViewTable() throws Exception {
808884
"tag2,STRING,TAG,",
809885
"s11,INT32,FIELD,",
810886
"s3,STRING,FIELD,")));
811-
// Currently we show the device even if all of its measurements does not match the type,
887+
// Currently we show the device even if all of its measurements does not match,
812888
// the handling logic at query because validate it at fetching will potentially cause a
813889
// lot of time
814890
TestUtils.assertResultSetEqual(
815891
statement.executeQuery("show devices from view_table where tag1 = 'b'"),
816892
"tag1,tag2,",
817-
new HashSet<>(Arrays.asList("b,c,", "b,null,", "b,d,")));
893+
new HashSet<>(Arrays.asList("b,c,", "b,null,", "b,d,", "b,e,")));
818894
TestUtils.assertResultSetEqual(
819895
statement.executeQuery("show devices from view_table where tag1 = 'b' and tag2 is null"),
820896
"tag1,tag2,",
821897
Collections.singleton("b,null,"));
822898
TestUtils.assertResultSetEqual(
823899
statement.executeQuery("count devices from view_table"),
824900
"count(devices),",
825-
Collections.singleton("3,"));
901+
Collections.singleton("4,"));
826902
}
827903

828904
// Test tree session

‎iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,8 @@ public enum TSStatusCode {
111111
DATA_TYPE_MISMATCH(614),
112112
COLUMN_CATEGORY_MISMATCH(615),
113113
COLUMN_NOT_EXISTS(616),
114+
MEASUREMENT_NAME_CONFLICT(617),
115+
114116
WAL_ENTRY_TOO_LARGE(620),
115117

116118
// Query Engine

‎iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ keyWords
6060
| CLEAR
6161
| CLUSTER
6262
| CLUSTERID
63+
| COMMENT
6364
| CONCAT
6465
| CONDITION
6566
| CONFIGNODE
@@ -87,6 +88,7 @@ keyWords
8788
| DATASET
8889
| DEACTIVATE
8990
| DEBUG
91+
| DEFAULT
9092
| DELETE
9193
| DESC
9294
| DESCRIBE
@@ -105,6 +107,7 @@ keyWords
105107
| EXPLAIN
106108
| EXTRACTOR
107109
| FALSE
110+
| FIELD
108111
| FILL
109112
| FILE
110113
| FIRST
@@ -198,6 +201,7 @@ keyWords
198201
| RESOURCE
199202
| REPAIR
200203
| REPLACE
204+
| RESTRICT
201205
| REVOKE
202206
| ROLE
203207
| ROUND
@@ -229,6 +233,7 @@ keyWords
229233
| SUBSTRING
230234
| SYSTEM
231235
| TABLE
236+
| TAG
232237
| TAGS
233238
| TAIL
234239
| TASK

‎iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -789,7 +789,8 @@ createTableView
789789
;
790790

791791
viewColumnDefinition
792-
: identifier (type)? (columnCategory=(TAG | TIME | FIELD))? comment?
792+
: identifier columnCategory=(TAG | TIME | FIELD) comment?
793+
| identifier type (columnCategory=(TAG | TIME | FIELD))? comment?
793794
| identifier (type)? (columnCategory=FIELD)? FROM original_measurement=identifier comment?
794795
;
795796

‎iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java‎

Lines changed: 62 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.apache.iotdb.commons.schema.table.TreeViewSchema;
2929
import org.apache.iotdb.commons.schema.table.TsTable;
3030
import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema;
31+
import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema;
3132
import org.apache.iotdb.commons.utils.StatusUtils;
3233
import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType;
3334
import org.apache.iotdb.confignode.manager.ConfigManager;
@@ -44,28 +45,30 @@
4445
import java.util.ArrayList;
4546
import java.util.Arrays;
4647
import java.util.Collections;
48+
import java.util.HashMap;
49+
import java.util.HashSet;
4750
import java.util.List;
51+
import java.util.Locale;
4852
import java.util.Map;
4953
import java.util.Objects;
5054
import java.util.Set;
5155
import java.util.concurrent.ConcurrentHashMap;
52-
import java.util.stream.Collectors;
5356

5457
public class TreeDeviceViewFieldDetector {
5558

5659
private static final Logger LOGGER = LoggerFactory.getLogger(TreeDeviceViewFieldDetector.class);
57-
private static final int MEASUREMENT_TRIMMING_THRESHOLD = 1000;
5860
private final ConfigManager configManager;
5961
private final PartialPath path;
6062
private final TsTable table;
61-
private final Map<String, FieldColumnSchema> fields;
63+
private final Map<String, Set<FieldColumnSchema>> fields;
6264

6365
private TDeviceViewResp result = new TDeviceViewResp(StatusUtils.OK, new ConcurrentHashMap<>());
66+
private final Map<String, String> lowerCase2OriginalMap = new HashMap<>();
6467

6568
public TreeDeviceViewFieldDetector(
6669
final ConfigManager configManager,
6770
final TsTable table,
68-
final Map<String, FieldColumnSchema> fields) {
71+
final Map<String, Set<FieldColumnSchema>> fields) {
6972
this.configManager = configManager;
7073
this.path = TreeViewSchema.getPrefixPattern(table);
7174
this.table = table;
@@ -77,7 +80,7 @@ public TSStatus detectMissingFieldTypes() {
7780
new TreeDeviceViewFieldDetectionTaskExecutor(
7881
configManager,
7982
getLatestSchemaRegionMap(),
80-
table.getIdNums(),
83+
table.getTagNum(),
8184
TreeViewSchema.isRestrict(table))
8285
.execute();
8386
if (result.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
@@ -86,47 +89,56 @@ public TSStatus detectMissingFieldTypes() {
8689
result
8790
.getDeviewViewFieldTypeMap()
8891
.forEach(
89-
(field, type) ->
90-
table.addColumnSchema(
91-
new FieldColumnSchema(field, TSDataType.getTsDataType(type))));
92+
(field, type) -> {
93+
final FieldColumnSchema columnSchema =
94+
new FieldColumnSchema(field, TSDataType.getTsDataType(type));
95+
if (!field.equals(lowerCase2OriginalMap.get(field))) {
96+
TreeViewSchema.setOriginalName(columnSchema, lowerCase2OriginalMap.get(field));
97+
}
98+
table.addColumnSchema(columnSchema);
99+
});
92100
} else {
93-
final Map<String, FieldColumnSchema> unknownFields =
94-
Objects.isNull(fields)
95-
? table.getColumnList().stream()
96-
.filter(
97-
columnSchema ->
98-
columnSchema instanceof FieldColumnSchema
99-
&& columnSchema.getDataType() == TSDataType.UNKNOWN)
100-
.collect(
101-
Collectors.toMap(
102-
fieldColumnSchema ->
103-
Objects.nonNull(TreeViewSchema.getOriginalName(fieldColumnSchema))
104-
? TreeViewSchema.getOriginalName(fieldColumnSchema)
105-
: fieldColumnSchema.getColumnName(),
106-
FieldColumnSchema.class::cast))
107-
: fields;
101+
final Map<String, Set<FieldColumnSchema>> unknownFields;
102+
if (Objects.isNull(fields)) {
103+
unknownFields = new HashMap<>();
104+
for (final TsTableColumnSchema schema : table.getColumnList()) {
105+
if (!(schema instanceof FieldColumnSchema)
106+
|| schema.getDataType() != TSDataType.UNKNOWN) {
107+
continue;
108+
}
109+
final String key = TreeViewSchema.getSourceName(schema);
110+
if (!unknownFields.containsKey(key)) {
111+
unknownFields.put(key, new HashSet<>());
112+
}
113+
unknownFields.get(key).add((FieldColumnSchema) schema);
114+
}
115+
} else {
116+
unknownFields = fields;
117+
}
118+
108119
if (unknownFields.isEmpty()) {
109120
return StatusUtils.OK;
110121
}
111122
new TreeDeviceViewFieldDetectionTaskExecutor(
112123
configManager,
113124
getLatestSchemaRegionMap(),
114-
table.getIdNums(),
125+
table.getTagNum(),
115126
TreeViewSchema.isRestrict(table),
116-
unknownFields.size() <= MEASUREMENT_TRIMMING_THRESHOLD
117-
? unknownFields.keySet()
118-
: null)
127+
unknownFields.keySet())
119128
.execute();
120129
if (result.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
121130
return result.getStatus();
122131
}
123-
for (final Map.Entry<String, FieldColumnSchema> unknownField : unknownFields.entrySet()) {
132+
for (final Map.Entry<String, Set<FieldColumnSchema>> unknownField :
133+
unknownFields.entrySet()) {
124134
if (result.getDeviewViewFieldTypeMap().containsKey(unknownField.getKey())) {
125135
unknownField
126136
.getValue()
127-
.setDataType(
128-
TSDataType.getTsDataType(
129-
result.getDeviewViewFieldTypeMap().get(unknownField.getKey())));
137+
.forEach(
138+
field ->
139+
field.setDataType(
140+
TSDataType.getTsDataType(
141+
result.getDeviewViewFieldTypeMap().get(unknownField.getKey()))));
130142
} else {
131143
return new TSStatus(TSStatusCode.TYPE_NOT_FOUND.getStatusCode())
132144
.setMessage(
@@ -226,16 +238,32 @@ private void mergeDeviceViewResp(final TDeviceViewResp resp) {
226238
resp.getDeviewViewFieldTypeMap()
227239
.forEach(
228240
(measurement, type) -> {
229-
if (!result.getDeviewViewFieldTypeMap().containsKey(measurement)) {
230-
result.getDeviewViewFieldTypeMap().put(measurement, type);
231-
} else {
241+
final String fieldName = measurement.toLowerCase(Locale.ENGLISH);
242+
243+
// Field type collection
244+
if (!result.getDeviewViewFieldTypeMap().containsKey(fieldName)) {
245+
result.getDeviewViewFieldTypeMap().put(fieldName, type);
246+
} else if (!Objects.equals(
247+
result.getDeviewViewFieldTypeMap().get(fieldName), type)) {
232248
result.setStatus(
233249
RpcUtils.getStatus(
234250
TSStatusCode.DATA_TYPE_MISMATCH,
235251
String.format(
236252
"Multiple types encountered when auto detecting type of measurement '%s', please check",
237253
measurement)));
238254
}
255+
256+
// Field name detection
257+
if (!lowerCase2OriginalMap.containsKey(fieldName)) {
258+
lowerCase2OriginalMap.put(fieldName, measurement);
259+
} else if (!Objects.equals(lowerCase2OriginalMap.get(fieldName), measurement)) {
260+
result.setStatus(
261+
RpcUtils.getStatus(
262+
TSStatusCode.MEASUREMENT_NAME_CONFLICT,
263+
String.format(
264+
"The measurements %s and %s share the same lower case when auto detecting type, please check",
265+
lowerCase2OriginalMap.get(fieldName), measurement)));
266+
}
239267
});
240268
}
241269

0 commit comments

Comments
 (0)