Skip to content

Commit 013b409

Browse files
shuwenweishuwenwei
authored
[to dev/1.3] Further optimize the efficiency of memtable region scan #16979 
1 parent b48b46e commit 013b409

File tree

4 files changed

+167
-53
lines changed

4 files changed

+167
-53
lines changed

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java‎

Lines changed: 55 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import java.util.concurrent.BlockingQueue;
5656
import java.util.concurrent.atomic.AtomicInteger;
5757

58+
import static org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager.ARRAY_SIZE;
5859
import static org.apache.iotdb.db.utils.ModificationUtils.isPointDeleted;
5960

6061
public class AlignedWritableMemChunk extends AbstractWritableMemChunk {
@@ -780,54 +781,67 @@ private void getAnySatisfiedTimestamp(
780781
// timestamp: 1 bitmap: 011
781782
// timestamp: 2 bitmap: 101
782783
// timestamp: 4 bitmap: 110
783-
for (int row = 0; row < rowCount; row++) {
784-
// the row is deleted
785-
if (allValueColDeletedMap != null && allValueColDeletedMap.isMarked(row)) {
786-
continue;
787-
}
788-
long timestamp = alignedTVList.getTime(row);
789-
if (globalTimeFilter != null && !globalTimeFilter.satisfy(timestamp, null)) {
790-
continue;
791-
}
792-
793-
// Note that this method will only perform bitmap unmarking on the first occurrence of a
794-
// non-null value in multiple timestamps for the same column.
795-
BitMap currentRowNullValueBitmap = null;
796-
797-
for (int column = 0; column < schemaList.size(); column++) {
798-
if (alignedTVList.isNullValue(alignedTVList.getValueIndex(row), column)) {
784+
List<long[]> timestampsList = alignedTVList.getTimestamps();
785+
List<int[]> indicesList = alignedTVList.getIndices();
786+
int row = -1;
787+
788+
for (int i = 0; i < timestampsList.size(); i++) {
789+
long[] timestamps = timestampsList.get(i);
790+
int[] indices = indicesList == null ? null : indicesList.get(i);
791+
int limit = (i == timestampsList.size() - 1) ? rowCount - i * ARRAY_SIZE : ARRAY_SIZE;
792+
for (int j = 0; j < limit; j++) {
793+
row++;
794+
// the row is deleted
795+
if (allValueColDeletedMap != null && allValueColDeletedMap.isMarked(row)) {
799796
continue;
800797
}
801-
802-
// skip deleted row
803-
if (valueColumnsDeletionList != null
804-
&& !valueColumnsDeletionList.isEmpty()
805-
&& isPointDeleted(
806-
timestamp,
807-
valueColumnsDeletionList.get(column),
808-
valueColumnDeleteCursor.get(column))) {
798+
long timestamp = timestamps[j];
799+
if (globalTimeFilter != null && !globalTimeFilter.satisfy(timestamp, null)) {
809800
continue;
810801
}
811-
if (!columnHasNonNullValue.isMarked(column)) {
812-
hasNonNullValueColumnCount.incrementAndGet();
813-
columnHasNonNullValue.mark(column);
814-
currentRowNullValueBitmap =
815-
currentRowNullValueBitmap != null
816-
? currentRowNullValueBitmap
817-
: timestampWithBitmap.computeIfAbsent(
818-
timestamp, k -> getAllMarkedBitmap(schemaList.size()));
819-
currentRowNullValueBitmap.unmark(column);
802+
803+
// Note that this method will only perform bitmap unmarking on the first occurrence of a
804+
// non-null value in multiple timestamps for the same column.
805+
BitMap currentRowNullValueBitmap = null;
806+
for (int column = 0; column < schemaList.size(); column++) {
807+
if (alignedTVList.isNullValue(indices == null ? row : indices[j], column)) {
808+
continue;
809+
}
810+
811+
// skip deleted row
812+
if (valueColumnsDeletionList != null && !valueColumnsDeletionList.isEmpty()) {
813+
List<TimeRange> columnDeletionList = valueColumnsDeletionList.get(column);
814+
int[] deleteCursor = valueColumnDeleteCursor.get(column);
815+
if (columnDeletionList != null && !columnDeletionList.isEmpty()) {
816+
if (!alignedTVList.isSorted()) {
817+
deleteCursor[0] = 0;
818+
}
819+
if (isPointDeleted(timestamp, columnDeletionList, deleteCursor)) {
820+
continue;
821+
}
822+
}
823+
}
824+
if (!columnHasNonNullValue.isMarked(column)) {
825+
hasNonNullValueColumnCount.incrementAndGet();
826+
columnHasNonNullValue.mark(column);
827+
currentRowNullValueBitmap =
828+
currentRowNullValueBitmap != null
829+
? currentRowNullValueBitmap
830+
: timestampWithBitmap.computeIfAbsent(
831+
timestamp, k -> getAllMarkedBitmap(schemaList.size()));
832+
currentRowNullValueBitmap.unmark(column);
833+
}
820834
}
821-
}
822835

823-
if (currentRowNullValueBitmap == null) {
824-
continue;
825-
}
826-
// found new column with non-null value
827-
timestampWithBitmap.put(timestamp, currentRowNullValueBitmap);
836+
if (currentRowNullValueBitmap == null) {
837+
continue;
838+
}
839+
// found new column with non-null value
840+
timestampWithBitmap.put(timestamp, currentRowNullValueBitmap);
828841

829-
if (hasNonNullValueColumnCount.get() == schemaList.size()) {
830-
return;
842+
if (hasNonNullValueColumnCount.get() == schemaList.size()) {
843+
return;
844+
}
831845
}
832846
}
833847
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java‎

Lines changed: 32 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.apache.iotdb.db.conf.IoTDBConfig;
2222
import org.apache.iotdb.db.conf.IoTDBDescriptor;
2323
import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView;
24+
import org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager;
2425
import org.apache.iotdb.db.utils.ModificationUtils;
2526
import org.apache.iotdb.db.utils.datastructure.BatchEncodeInfo;
2627
import org.apache.iotdb.db.utils.datastructure.MemPointIterator;
@@ -578,19 +579,38 @@ private Optional<Long> getAnySatisfiedTimestamp(
578579
&& !globalTimeFilter.satisfyStartEndTime(tvlist.getMinTime(), tvlist.getMaxTime())) {
579580
return Optional.empty();
580581
}
581-
for (int i = 0; i < rowCount; i++) {
582-
if (tvlist.getBitMap() != null && tvlist.isNullValue(tvlist.getValueIndex(i))) {
583-
continue;
584-
}
585-
long curTime = tvlist.getTime(i);
586-
if (deletionList != null
587-
&& ModificationUtils.isPointDeleted(curTime, deletionList, deletionCursor)) {
588-
continue;
589-
}
590-
if (globalTimeFilter != null && !globalTimeFilter.satisfy(curTime, null)) {
591-
continue;
582+
583+
List<long[]> timestampsList = tvlist.getTimestamps();
584+
List<BitMap> bitMaps = tvlist.getBitMap();
585+
List<int[]> indicesList = tvlist.getIndices();
586+
for (int i = 0; i < timestampsList.size(); i++) {
587+
long[] timestamps = timestampsList.get(i);
588+
BitMap bitMap = bitMaps == null ? null : bitMaps.get(i);
589+
int[] indices = indicesList == null ? null : indicesList.get(i);
590+
int limit =
591+
(i == timestampsList.size() - 1)
592+
? rowCount - i * PrimitiveArrayManager.ARRAY_SIZE
593+
: PrimitiveArrayManager.ARRAY_SIZE;
594+
for (int j = 0; j < limit; j++) {
595+
if (bitMap != null
596+
&& (indices == null ? bitMap.isMarked(j) : tvlist.isNullValue(indices[j]))) {
597+
continue;
598+
}
599+
long curTime = timestamps[j];
600+
if (deletionList != null && !deletionList.isEmpty()) {
601+
if (!tvlist.isSorted()) {
602+
deletionCursor[0] = 0;
603+
}
604+
if (ModificationUtils.isPointDeleted(curTime, deletionList, deletionCursor)) {
605+
continue;
606+
}
607+
}
608+
if (globalTimeFilter != null && !globalTimeFilter.satisfy(curTime, null)) {
609+
continue;
610+
}
611+
612+
return Optional.of(curTime);
592613
}
593-
return Optional.of(curTime);
594614
}
595615
return Optional.empty();
596616
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -736,6 +736,10 @@ public List<long[]> getTimestamps() {
736736
return timestamps;
737737
}
738738

739+
public List<int[]> getIndices() {
740+
return indices;
741+
}
742+
739743
public void setOwnerQuery(QueryContext queryCtx) {
740744
this.ownerQuery = queryCtx;
741745
}

‎iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,48 @@ public void testAlignedWritableMemChunkRegionScan() throws IllegalPathException
232232
}
233233
}
234234

235+
@Test
236+
public void testAlignedWritableMemChunkRegionScan2() throws IllegalPathException {
237+
PrimitiveMemTable memTable = new PrimitiveMemTable("root.test", "0");
238+
try {
239+
List<IMeasurementSchema> measurementSchemas =
240+
Arrays.asList(
241+
new MeasurementSchema("s1", TSDataType.INT32),
242+
new MeasurementSchema("s2", TSDataType.INT32),
243+
new MeasurementSchema("s3", TSDataType.INT32));
244+
AlignedWritableMemChunk writableMemChunk = null;
245+
for (int i = 1000; i < 2000; i++) {
246+
memTable.writeAlignedRow(
247+
new PlainDeviceID("root.test.d1"), measurementSchemas, i, new Object[] {i, i, i});
248+
}
249+
for (int i = 1; i < 100; i++) {
250+
memTable.writeAlignedRow(
251+
new PlainDeviceID("root.test.d1"), measurementSchemas, i, new Object[] {i, null, i});
252+
}
253+
254+
MeasurementPath originalPath = new MeasurementPath("root.test.d1.s1");
255+
memTable.delete(originalPath, originalPath.getDevicePath(), 1, 1500);
256+
257+
writableMemChunk =
258+
(AlignedWritableMemChunk)
259+
memTable.getWritableMemChunk(new PlainDeviceID("root.test.d1"), "");
260+
writableMemChunk.sortTvListForFlush();
261+
List<BitMap> bitMaps = new ArrayList<>();
262+
long[] timestamps =
263+
writableMemChunk.getAnySatisfiedTimestamp(
264+
Arrays.asList(
265+
Collections.emptyList(), Collections.emptyList(), Collections.emptyList()),
266+
bitMaps,
267+
null);
268+
Assert.assertEquals(3, timestamps.length);
269+
Assert.assertEquals(1, timestamps[0]);
270+
Assert.assertEquals(1000, timestamps[1]);
271+
Assert.assertEquals(1501, timestamps[2]);
272+
} finally {
273+
memTable.release();
274+
}
275+
}
276+
235277
@Test
236278
public void testNonAlignedWritableMemChunkRegionScan() throws IllegalPathException {
237279
PrimitiveMemTable memTable = new PrimitiveMemTable("root.test", "0");
@@ -296,4 +338,38 @@ public void testNonAlignedWritableMemChunkRegionScan() throws IllegalPathExcepti
296338
memTable.release();
297339
}
298340
}
341+
342+
@Test
343+
public void testNonAlignedWritableMemChunkRegionScan2() throws IllegalPathException {
344+
PrimitiveMemTable memTable = new PrimitiveMemTable("root.test", "0");
345+
try {
346+
MeasurementSchema measurementSchema = new MeasurementSchema("s1", TSDataType.INT32);
347+
for (int i = 1000; i < 2000; i++) {
348+
memTable.write(
349+
new PlainDeviceID("root.test.d1"),
350+
Collections.singletonList(measurementSchema),
351+
i,
352+
new Object[] {i});
353+
}
354+
for (int i = 1; i < 100; i++) {
355+
memTable.write(
356+
new PlainDeviceID("root.test.d1"),
357+
Collections.singletonList(measurementSchema),
358+
i,
359+
new Object[] {i});
360+
}
361+
362+
MeasurementPath originalPath = new MeasurementPath("root.test.d1.s1");
363+
memTable.delete(originalPath, originalPath.getDevicePath(), 1, 1500);
364+
WritableMemChunk writableMemChunk =
365+
(WritableMemChunk) memTable.getWritableMemChunk(new PlainDeviceID("root.test.d1"), "s1");
366+
writableMemChunk.sortTvListForFlush();
367+
Optional<Long> timestamp = writableMemChunk.getAnySatisfiedTimestamp(null, null);
368+
Assert.assertTrue(timestamp.isPresent());
369+
Assert.assertEquals(1501, timestamp.get().longValue());
370+
371+
} finally {
372+
memTable.release();
373+
}
374+
}
299375
}

0 commit comments

Comments
 (0)