Skip to content

Commit 01d88b7

Browse files
shuwenweishuwenwei
authored
Fix some bugs in AlignedTVListIterator
1 parent 182a78a commit 01d88b7

File tree

4 files changed

+153
-46
lines changed

4 files changed

+153
-46
lines changed

‎integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedMemQueryIT.java‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.db.it.aligned;
21+
22+
import org.apache.iotdb.it.env.EnvFactory;
23+
24+
import org.junit.After;
25+
import org.junit.Before;
26+
import org.junit.Test;
27+
28+
import java.sql.Connection;
29+
import java.sql.ResultSet;
30+
import java.sql.SQLException;
31+
import java.sql.Statement;
32+
import java.util.Locale;
33+
34+
import static org.junit.Assert.assertEquals;
35+
36+
public class IoTDBAlignedMemQueryIT {
37+
38+
@Before
39+
public void setUp() throws Exception {
40+
Locale.setDefault(Locale.ENGLISH);
41+
EnvFactory.getEnv()
42+
.getConfig()
43+
.getCommonConfig()
44+
.setPartitionInterval(1000)
45+
.setMemtableSizeThreshold(10000);
46+
EnvFactory.getEnv().initClusterEnvironment();
47+
}
48+
49+
@After
50+
public void tearDown() throws Exception {
51+
EnvFactory.getEnv().cleanClusterEnvironment();
52+
}
53+
54+
@Test
55+
public void test() throws SQLException {
56+
try (Connection connection = EnvFactory.getEnv().getConnection();
57+
Statement statement = connection.createStatement()) {
58+
59+
statement.execute("insert into root.vehicle.d0(time,s0) aligned values (10,310)");
60+
statement.execute("insert into root.vehicle.d0(time,s3) aligned values (10,'text')");
61+
statement.execute("insert into root.vehicle.d0(time,s4) aligned values (10,true)");
62+
63+
try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle.d0")) {
64+
int cnt = 0;
65+
while (set.next()) {
66+
cnt++;
67+
}
68+
assertEquals(1, cnt);
69+
}
70+
71+
statement.execute("insert into root.vehicle.d1(time,s0,s1) aligned values (1,1,1)");
72+
statement.execute("insert into root.vehicle.d1(time,s0) aligned values (2,2)");
73+
statement.execute("insert into root.vehicle.d1(time,s1) aligned values (3,3)");
74+
75+
try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle.d1")) {
76+
int cnt = 0;
77+
while (set.next()) {
78+
cnt++;
79+
}
80+
assertEquals(2, cnt);
81+
}
82+
statement.execute("flush");
83+
84+
statement.execute("insert into root.vehicle.d1(time,s0,s1) aligned values (1,1,1)");
85+
statement.execute("insert into root.vehicle.d1(time,s0) aligned values (2,2)");
86+
statement.execute("insert into root.vehicle.d1(time,s1) aligned values (3,3)");
87+
88+
try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle.d1")) {
89+
int cnt = 0;
90+
while (set.next()) {
91+
cnt++;
92+
}
93+
assertEquals(2, cnt);
94+
}
95+
}
96+
}
97+
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java‎

Lines changed: 28 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1705,17 +1705,14 @@ protected void prepareNext() {
17051705
probeNext = true;
17061706
return;
17071707
}
1708-
if (scanOrder.isAscending()) {
1709-
// When traversing in ASC order, we only need to overwrite the previous non-null value
1710-
// with the non-null value encountered later
1711-
Arrays.fill(selectedIndices, index);
1712-
} else {
1713-
// When traversing in DESC order, we need to keep the non-null value encountered first,
1714-
// and only overwrite it if the previous value is null and current value is non-null. In
1715-
// order to identify the previous null, we use index -1 here to represent
1716-
for (int i = 0; i < selectedIndices.length; i++) {
1717-
selectedIndices[i] = isNullValue(index, i) ? -1 : index;
1718-
}
1708+
// When traversing in ASC order, we only need to overwrite the previous non-null value
1709+
// with the non-null value encountered later.
1710+
// When traversing in DESC order, we need to keep the non-null value encountered first,
1711+
// and only overwrite it if the previous value is null and current value is non-null.
1712+
for (int i = 0; i < selectedIndices.length; i++) {
1713+
// In order to identify the previous null, we use index -1 here to represent.
1714+
// The -1 here is also used for checking all null rows
1715+
selectedIndices[i] = isNullValue(index, i) ? -1 : index;
17191716
}
17201717
findValidRow = true;
17211718

@@ -1740,35 +1737,36 @@ && getTime(getScanOrderIndex(index + 1)) == getTime(getScanOrderIndex(index))) {
17401737
}
17411738
}
17421739
}
1743-
// For DESC traversal, we previously set some -1. If these values are still -1 in the end,
1744-
// it means that each index is invalid. At this time, we can use any one at random.
1745-
if (!scanOrder.isAscending()) {
1746-
for (int i = 0; i < selectedIndices.length; i++) {
1747-
if (selectedIndices[i] == -1) {
1748-
selectedIndices[i] = index;
1749-
}
1750-
}
1751-
}
17521740

17531741
// valueColumnsDeletionList is set when AlignedTVList iterator is created by
1754-
// MemPointIterator.single method. Otherwise, all-null rows is checked by
1742+
// MemPointIterator.single method. Otherwise, it is checked by
17551743
// MergeSortMultiAlignedTVListIterator or OrderedMultiAlignedTVListIterator.
1756-
if (valueColumnsDeletionList != null) {
1757-
BitMap bitMap = new BitMap(dataTypeList.size());
1744+
if (ignoreAllNullRows) {
1745+
BitMap bitMap = null;
17581746
time = getTime(getScanOrderIndex(index));
17591747
for (int columnIndex = 0; columnIndex < dataTypeList.size(); columnIndex++) {
1760-
if (isNullValue(index, columnIndex)
1761-
|| isPointDeleted(
1762-
time,
1763-
valueColumnsDeletionList.get(columnIndex),
1764-
valueColumnDeleteCursor.get(columnIndex),
1765-
scanOrder)) {
1748+
if (selectedIndices[columnIndex] == -1
1749+
|| (valueColumnsDeletionList != null
1750+
&& isPointDeleted(
1751+
time,
1752+
valueColumnsDeletionList.get(columnIndex),
1753+
valueColumnDeleteCursor.get(columnIndex),
1754+
scanOrder))) {
1755+
bitMap = bitMap == null ? new BitMap(dataTypeList.size()) : bitMap;
17661756
bitMap.mark(columnIndex);
17671757
}
17681758
}
1769-
if (ignoreAllNullRows && bitMap.isAllMarked()) {
1759+
if (bitMap != null && bitMap.isAllMarked()) {
17701760
findValidRow = false;
17711761
index++;
1762+
continue;
1763+
}
1764+
}
1765+
// We previously set some -1. If these values are still -1 in the end,
1766+
// it means that each index is invalid. At this time, we can use any one at random.
1767+
for (int i = 0; i < selectedIndices.length; i++) {
1768+
if (selectedIndices[i] == -1) {
1769+
selectedIndices[i] = index;
17721770
}
17731771
}
17741772
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiAlignedTVListIterator.java‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -115,22 +115,35 @@ protected void prepareNext() {
115115
.isNullValue(rowIndices[columnIndex], columnIndex)) {
116116
bitMap.mark(columnIndex);
117117
}
118+
// check valueColumnsDeletionList
119+
if (valueColumnsDeletionList != null
120+
&& isPointDeleted(
121+
currentTime,
122+
valueColumnsDeletionList.get(columnIndex),
123+
valueColumnDeleteCursor.get(columnIndex),
124+
scanOrder)) {
125+
iteratorIndices[columnIndex] = -1;
126+
bitMap.mark(columnIndex);
127+
}
118128
}
119129
hasNext = true;
120130

121131
// duplicated timestamps
122-
boolean[] valueDeleted = new boolean[tsDataTypeList.size()];
123132
while (!heap.isEmpty() && heap.peek().left == currentTime) {
124133
Pair<Long, Integer> element = heap.poll();
125134
probeIterators.add(element.right);
126135

127136
for (int columnIndex = 0; columnIndex < tsDataTypeList.size(); columnIndex++) {
128137
// if current column null, it needs update
129138
int iteratorIndex = currentIteratorIndex(columnIndex);
130-
if (iteratorIndex == -1
131-
|| alignedTvListIterators
132-
.get(iteratorIndex)
133-
.isNullValue(rowIndices[columnIndex], columnIndex)) {
139+
if (iteratorIndex == -1) {
140+
// -1 means all point of this timestamp was deleted by Deletion and no further
141+
// processing is required.
142+
continue;
143+
}
144+
if (alignedTvListIterators
145+
.get(iteratorIndex)
146+
.isNullValue(rowIndices[columnIndex], columnIndex)) {
134147
iteratorIndices[columnIndex] = element.right;
135148
rowIndices[columnIndex] =
136149
alignedTvListIterators.get(element.right).getSelectedIndex(columnIndex);

‎iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedTVListIteratorTest.java‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -109,17 +109,16 @@ public static void tearDown() {
109109
public void otherTest() throws IOException {
110110
Map<TVList, Integer> tvListMap =
111111
buildAlignedSingleTvListMap(Collections.singletonList(new TimeRange(1, 1)));
112-
// testAligned(
113-
// tvListMap,
114-
// Ordering.ASC,
115-
// null,
116-
// null,
117-
// PaginationController.UNLIMITED_PAGINATION_CONTROLLER,
118-
// Collections.emptyList(),
119-
// Arrays.asList(Collections.emptyList(), Collections.emptyList(),
120-
// Collections.emptyList()),
121-
// false,
122-
// 1);
112+
testAligned(
113+
tvListMap,
114+
Ordering.ASC,
115+
null,
116+
null,
117+
PaginationController.UNLIMITED_PAGINATION_CONTROLLER,
118+
Collections.emptyList(),
119+
Arrays.asList(Collections.emptyList(), Collections.emptyList(), Collections.emptyList()),
120+
false,
121+
1);
123122
testAligned(
124123
tvListMap,
125124
Ordering.DESC,

0 commit comments

Comments
 (0)